solve most of the issues with failing async-gen tests, some asyncio tests are still failing

Cython/Compiler/ExprNodes.py

@@ -9484,12 +9484,15 @@ class YieldExprNode(ExprNode):
             if type.is_pyobject:
                 code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname))
                 code.put_xgotref(cname)
-        code.putln(code.error_goto_if_null(Naming.sent_value_cname, self.pos))
+        self.generate_sent_value_handling_code(code, Naming.sent_value_cname)
         if self.result_is_used:
             self.allocate_temp_result(code)
             code.put('%s = %s; ' % (self.result(), Naming.sent_value_cname))
             code.put_incref(self.result(), py_object_type)
 
+    def generate_sent_value_handling_code(self, code, value_cname):
+        code.putln(code.error_goto_if_null(value_cname, self.pos))
+
 
 class _YieldDelegationExprNode(YieldExprNode):
     def yield_from_func(self, code):
@@ -9582,8 +9585,7 @@ class AwaitIterNextExprNode(AwaitExprNode):
     #
     # Breaks out of loop on StopAsyncIteration exception.
 
-    def fetch_iteration_result(self, code):
-        assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop"
+    def _generate_break(self, code):
         code.globalstate.use_utility_code(UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
         code.putln("PyObject* exc_type = PyErr_Occurred();")
         code.putln("if (exc_type && likely(exc_type == __Pyx_PyExc_StopAsyncIteration ||"
@@ -9591,8 +9593,20 @@ class AwaitIterNextExprNode(AwaitExprNode):
         code.putln("PyErr_Clear();")
         code.putln("break;")
         code.putln("}")
+
+    def fetch_iteration_result(self, code):
+        assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop"
+        self._generate_break(code)
         super(AwaitIterNextExprNode, self).fetch_iteration_result(code)
 
+    def generate_sent_value_handling_code(self, code, value_cname):
+        assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop"
+        code.putln("if (unlikely(!%s)) {" % value_cname)
+        self._generate_break(code)
+        # all non-break exceptions are errors, as in parent class
+        code.putln(code.error_goto(self.pos))
+        code.putln("}")
+
 
 class GlobalsExprNode(AtomicExprNode):
     type = dict_type

Cython/Compiler/Nodes.py

@@ -4025,19 +4025,19 @@ class AsyncGenNode(AsyncDefNode):
     is_asyncgen = True
 
 
-
 class GeneratorBodyDefNode(DefNode):
     # Main code body of a generator implemented as a DefNode.
     #
 
     is_generator_body = True
     is_inlined = False
+    is_async_gen = False
     inlined_comprehension_type = None  # container type for inlined comprehensions
 
-    def __init__(self, pos=None, name=None, body=None):
+    def __init__(self, pos=None, name=None, body=None, is_async_gen=False):
         super(GeneratorBodyDefNode, self).__init__(
-            pos=pos, body=body, name=name, doc=None,
-            args=[], star_arg=None, starstar_arg=None)
+            pos=pos, body=body, name=name, is_async_gen=is_async_gen,
+            doc=None, args=[], star_arg=None, starstar_arg=None)
 
     def declare_generator_body(self, env):
         prefix = env.next_id(env.scope_prefix)
@@ -4134,9 +4134,10 @@ class GeneratorBodyDefNode(DefNode):
         # on normal generator termination, we do not take the exception propagation
         # path: no traceback info is required and not creating it is much faster
         if not self.is_inlined and not self.body.is_terminator:
-            code.putln('PyErr_SetNone(PyExc_StopIteration);')
+            code.putln('PyErr_SetNone(%s);' % (
+                '__Pyx_PyExc_StopAsyncIteration' if self.is_async_gen else 'PyExc_StopIteration'))
         # ----- Error cleanup
-        if code.error_label in code.labels_used:
+        if code.label_used(code.error_label):
             if not self.body.is_terminator:
                 code.put_goto(code.return_label)
             code.put_label(code.error_label)
@@ -4145,8 +4146,7 @@ class GeneratorBodyDefNode(DefNode):
             if Future.generator_stop in env.global_scope().context.future_directives:
                 # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
                 code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Coroutine.c"))
-                code.putln("if (unlikely(PyErr_ExceptionMatches(PyExc_StopIteration))) "
-                           "__Pyx_Generator_Replace_StopIteration();")
+                code.putln("__Pyx_Generator_Replace_StopIteration(%d);" % bool(self.is_async_gen))
             for cname, type in code.funcstate.all_managed_temps():
                 code.put_xdecref(cname, type)
             code.put_add_traceback(self.entry.qualified_name)
@@ -5583,10 +5583,12 @@ class ReturnStatNode(StatNode):
     #  value         ExprNode or None
     #  return_type   PyrexType
     #  in_generator  return inside of generator => raise StopIteration
+    #  in_async_gen  return inside of async generator
 
     child_attrs = ["value"]
     is_terminator = True
     in_generator = False
+    in_async_gen = False
 
     # Whether we are in a parallel section
     in_parallel = False
@@ -5598,6 +5600,8 @@ class ReturnStatNode(StatNode):
             error(self.pos, "Return not inside a function body")
             return self
         if self.value:
+            if self.in_async_gen:
+                error(self.pos, "Return with value in async generator")
             self.value = self.value.analyse_types(env)
             if return_type.is_void or return_type.is_returncode:
                 error(self.value.pos, "Return with value in void function")
@@ -5654,6 +5658,8 @@ class ReturnStatNode(StatNode):
         else:
             if self.return_type.is_pyobject:
                 if self.in_generator:
+                    if self.in_async_gen:
+                        code.put("PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); ")
                     code.putln("%s = NULL;" % Naming.retval_cname)
                 else:
                     code.put_init_to_py_none(Naming.retval_cname, self.return_type)

Cython/Compiler/ParseTreeTransforms.py

@@ -2518,7 +2518,7 @@ class MarkClosureVisitor(CythonTransform):
         if node.is_async_def:
             coroutine_type = Nodes.AsyncGenNode if collector.has_yield else Nodes.AsyncDefNode
             if collector.has_yield:
-                for yield_expr in collector.yields:
+                for yield_expr in collector.yields + collector.returns:
                     yield_expr.in_async_gen = True
         elif collector.has_await:
             found = next(y for y in collector.yields if y.is_await)
@@ -2535,7 +2535,8 @@ class MarkClosureVisitor(CythonTransform):
             retnode.in_generator = True
 
         gbody = Nodes.GeneratorBodyDefNode(
-            pos=node.pos, name=node.name, body=node.body)
+            pos=node.pos, name=node.name, body=node.body,
+            is_async_gen=node.is_async_def and collector.has_yield)
         coroutine = coroutine_type(
             pos=node.pos, name=node.name, args=node.args,
             star_arg=node.star_arg, starstar_arg=node.starstar_arg,

Cython/Utility/AsyncGen.c

 // This is copied from genobject.c in CPython 3.6.
-// Try to keep it in sync.
+// Try to keep it in sync by doing this from time to time:
+//    sed -e 's|__pyx_||ig'  Cython/Utility/AsyncGen.c | diff -udw - cpython/Objects/genobject.c | less
 
 //////////////////// AsyncGenerator.proto ////////////////////
 //@requires: Coroutine.c::Coroutine
@@ -12,14 +13,20 @@ typedef struct {
     int ag_closed;
 } __pyx_PyAsyncGenObject;
 
+typedef struct __pyx_PyAsyncGenASend_struct __pyx_PyAsyncGenASend;
 
 static PyTypeObject *__pyx__PyAsyncGenWrappedValueType = 0;
 static PyTypeObject *__pyx__PyAsyncGenASendType = 0;
 static PyTypeObject *__pyx__PyAsyncGenAThrowType = 0;
 static PyTypeObject *__pyx_AsyncGenType = 0;
+
 #define __Pyx_AsyncGen_CheckExact(obj) (Py_TYPE(obj) == __pyx_AsyncGenType)
+#define __pyx_PyAsyncGenASend_CheckExact(o) \
+                    (Py_TYPE(o) == __pyx__PyAsyncGenASendType)
 
-static PyObject *__Pyx_async_gen_anext(PyObject *o);
+static PyObject *__Pyx_async_gen_anext(__pyx_PyAsyncGenObject *o);
+static PyObject *__Pyx_async_gen_asend_iternext(__pyx_PyAsyncGenASend *o);
+static PyObject *__Pyx_async_gen_asend_send(__pyx_PyAsyncGenASend *o, PyObject *arg);
 
 static PyObject *__Pyx__PyAsyncGenValueWrapperNew(PyObject *val);
 
@@ -120,7 +127,7 @@ typedef enum {
     __PYX_AWAITABLE_STATE_CLOSED, /* closed */
 } __pyx_AwaitableState;
 
-typedef struct {
+struct __pyx_PyAsyncGenASend_struct {
     PyObject_HEAD
     __pyx_PyAsyncGenObject *ags_gen;
 
@@ -128,7 +135,7 @@ typedef struct {
     PyObject *ags_sendval;
 
     __pyx_AwaitableState ags_state;
-} __pyx_PyAsyncGenASend;
+};
 
 
 typedef struct {
@@ -167,9 +174,6 @@ static int __Pyx_ag_asend_freelist_free = 0;
 #define __pyx__PyAsyncGenWrappedValue_CheckExact(o) \
                     (Py_TYPE(o) == __pyx__PyAsyncGenWrappedValueType)
 
-#define __pyx_PyAsyncGenASend_CheckExact(o) \
-                    (Py_TYPE(o) == __pyx__PyAsyncGenASendType)
-
 
 static int
 __Pyx_async_gen_traverse(__pyx_PyAsyncGenObject *gen, visitproc visit, void *arg)
@@ -303,7 +307,7 @@ static PyMethodDef __Pyx_async_gen_methods[] = {
 };
 
 
-static PyAsyncMethods __Pyx_async_gen_as_async = {
+static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_as_async = {
     0,                                          /* am_await */
     PyObject_SelfIter,                          /* am_aiter */
     (unaryfunc)__Pyx_async_gen_anext             /* am_anext */
@@ -475,12 +479,12 @@ __Pyx_async_gen_asend_send(__pyx_PyAsyncGenASend *o, PyObject *arg)
 
     if (o->ags_state == __PYX_AWAITABLE_STATE_INIT) {
         if (arg == NULL || arg == Py_None) {
-            arg = o->ags_sendval;
+            arg = o->ags_sendval ? o->ags_sendval : Py_None;
         }
         o->ags_state = __PYX_AWAITABLE_STATE_ITER;
     }
 
-    result = __Pyx_Coroutine_SendEx((__pyx_CoroutineObject*)o->ags_gen, arg);
+    result = __Pyx_Coroutine_SendEx((__pyx_CoroutineObject*)o->ags_gen, arg, 0);
     result = __Pyx_async_gen_unwrap_value(o->ags_gen, result);
 
     if (result == NULL) {
@@ -494,7 +498,7 @@ __Pyx_async_gen_asend_send(__pyx_PyAsyncGenASend *o, PyObject *arg)
 static PyObject *
 __Pyx_async_gen_asend_iternext(__pyx_PyAsyncGenASend *o)
 {
-    return __Pyx_async_gen_asend_send(o, NULL);
+    return __Pyx_async_gen_asend_send(o, Py_None);
 }
 
 
@@ -535,7 +539,7 @@ static PyMethodDef __Pyx_async_gen_asend_methods[] = {
 };
 
 
-static PyAsyncMethods __Pyx_async_gen_asend_as_async = {
+static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_asend_as_async = {
     PyObject_SelfIter,                          /* am_await */
     0,                                          /* am_aiter */
     0                                           /* am_anext */
@@ -824,7 +828,7 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg)
 
     assert (o->agt_state == __PYX_AWAITABLE_STATE_ITER);
 
-    retval = __Pyx_Coroutine_SendEx((__pyx_CoroutineObject *)gen, arg);
+    retval = __Pyx_Coroutine_SendEx((__pyx_CoroutineObject *)gen, arg, 0);
     if (o->agt_args) {
         return __Pyx_async_gen_unwrap_value(o->agt_gen, retval);
     } else {
@@ -921,7 +925,7 @@ static PyMethodDef __Pyx_async_gen_athrow_methods[] = {
 };
 
 
-static PyAsyncMethods __Pyx_async_gen_athrow_as_async = {
+static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_athrow_as_async = {
     PyObject_SelfIter,                          /* am_await */
     0,                                          /* am_aiter */
     0                                           /* am_anext */

Cython/Utility/Coroutine.c

@@ -85,6 +85,16 @@ static CYTHON_INLINE PyObject* __Pyx__Coroutine_Yield_From(__pyx_CoroutineObject
             gen->yieldfrom = source;
             return retval;
         }
+#ifdef __Pyx_AsyncGen_USED
+    // inlined "__pyx_PyAsyncGenASend" handling to avoid the series of generic calls below
+    } else if (__pyx_PyAsyncGenASend_CheckExact(source)) {
+        retval = __Pyx_async_gen_asend_iternext((__pyx_PyAsyncGenASend *)source);
+        if (retval) {
+            Py_INCREF(source);
+            gen->yieldfrom = source;
+            return retval;
+        }
+#endif
     } else {
         PyObject *source_gen = __Pyx__Coroutine_GetAwaitableIter(source);
         if (unlikely(!source_gen))
@@ -292,7 +302,7 @@ static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAsyncIter(PyObject *obj) {
 static CYTHON_INLINE PyObject *__Pyx_Coroutine_AsyncIterNext(PyObject *obj) {
 #ifdef __Pyx_AsyncGen_USED
     if (__Pyx_AsyncGen_CheckExact(obj)) {
-        return __Pyx_async_gen_anext(obj);
+        return __Pyx_async_gen_anext((__pyx_PyAsyncGenObject*) obj);
     }
 #endif
 #if CYTHON_USE_ASYNC_SLOTS
@@ -320,22 +330,41 @@ static CYTHON_INLINE PyObject *__Pyx_Coroutine_AsyncIterNext(PyObject *obj) {
 
 //////////////////// pep479.proto ////////////////////
 
-static void __Pyx_Generator_Replace_StopIteration(void); /*proto*/
+static void __Pyx_Generator_Replace_StopIteration(int in_async_gen); /*proto*/
 
 //////////////////// pep479 ////////////////////
 //@requires: Exceptions.c::GetException
 
-static void __Pyx_Generator_Replace_StopIteration(void) {
-    PyObject *exc, *val, *tb;
-    // Chain exceptions by moving StopIteration to exc_info before creating the RuntimeError.
-    // In Py2.x, no chaining happens, but the exception still stays visible in exc_info.
+static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen) {
+    PyObject *exc, *val, *tb, *cur_exc;
     __Pyx_PyThreadState_declare
+    #ifdef __Pyx_StopAsyncIteration_USED
+    int is_async_stopiteration = 0;
+    #endif
+
+    cur_exc = PyErr_Occurred();
+    if (likely(!PyErr_GivenExceptionMatches(cur_exc, PyExc_StopIteration))) {
+        #ifdef __Pyx_StopAsyncIteration_USED
+        if (in_async_gen && unlikely(PyErr_GivenExceptionMatches(cur_exc, __Pyx_PyExc_StopAsyncIteration))) {
+            is_async_stopiteration = 1;
+        } else
+        #endif
+            return;
+    }
+
     __Pyx_PyThreadState_assign
+    // Chain exceptions by moving Stop(Async)Iteration to exc_info before creating the RuntimeError.
+    // In Py2.x, no chaining happens, but the exception still stays visible in exc_info.
     __Pyx_GetException(&exc, &val, &tb);
     Py_XDECREF(exc);
     Py_XDECREF(val);
     Py_XDECREF(tb);
-    PyErr_SetString(PyExc_RuntimeError, "generator raised StopIteration");
+    PyErr_SetString(PyExc_RuntimeError,
+        #ifdef __Pyx_StopAsyncIteration_USED
+        is_async_stopiteration ? "async generator raised StopAsyncIteration" :
+        in_async_gen ? "async generator raised StopIteration" :
+        #endif
+        "generator raised StopIteration");
 }
 
 
@@ -567,7 +596,7 @@ int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) {
 }
 
 static CYTHON_INLINE
-PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) {
+PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, int closing) {
     PyObject *retval;
     __Pyx_PyThreadState_declare
 
@@ -594,12 +623,22 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) {
     }
 
     if (unlikely(self->resume_label == -1)) {
+        if (!closing && __Pyx_Coroutine_CheckExact((PyObject*)self)) {
+            // `self` is an exhausted coroutine: raise an error,
+            // except when called from gen_close(), which should
+            // always be a silent method.
+            PyErr_SetString(PyExc_RuntimeError, "cannot reuse already awaited coroutine");
+        }
+        else if (value) {
+            // `gen` is an exhausted generator:
+            // only set exception if called from send().
             #ifdef __Pyx_AsyncGen_USED
             if (__Pyx_AsyncGen_CheckExact((PyObject*)self))
                 PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration);
             else
             #endif
             PyErr_SetNone(PyExc_StopIteration);
+        }
         return NULL;
     }
 
@@ -652,9 +691,14 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) {
 }
 
 static CYTHON_INLINE
-PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) {
+PyObject *__Pyx_Coroutine_MethodReturn(PyObject* gen, PyObject *retval) {
     if (unlikely(!retval && !PyErr_Occurred())) {
         // method call must not terminate with NULL without setting an exception
+        #ifdef __Pyx_AsyncGen_USED
+        if (__Pyx_AsyncGen_CheckExact(gen)) {
+            PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration);
+        } else
+        #endif
         PyErr_SetNone(PyExc_StopIteration);
     }
     return retval;
@@ -667,7 +711,7 @@ PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) {
     __Pyx_Coroutine_Undelegate(gen);
     __Pyx_PyGen_FetchStopIterationValue(&val);
     // val == NULL on failure => pass on exception
-    ret = __Pyx_Coroutine_SendEx(gen, val);
+    ret = __Pyx_Coroutine_SendEx(gen, val, 0);
     Py_XDECREF(val);
     return ret;
 }
@@ -693,6 +737,11 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) {
             ret = __Pyx_Coroutine_Send(yf, value);
         } else
         #endif
+        #ifdef __Pyx_AsyncGen_USED
+        if (__pyx_PyAsyncGenASend_CheckExact(yf)) {
+            ret = __Pyx_async_gen_asend_send((__pyx_PyAsyncGenASend *)yf, value);
+        } else
+        #endif
         {
             if (value == Py_None)
                 ret = Py_TYPE(yf)->tp_iternext(yf);
@@ -706,9 +755,9 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) {
         }
         retval = __Pyx_Coroutine_FinishDelegation(gen);
     } else {
-        retval = __Pyx_Coroutine_SendEx(gen, value);
+        retval = __Pyx_Coroutine_SendEx(gen, value, 0);
     }
-    return __Pyx_Coroutine_MethodReturn(retval);
+    return __Pyx_Coroutine_MethodReturn(self, retval);
 }
 
 //   This helper function is used by gen_close and gen_throw to
@@ -776,7 +825,7 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) {
         }
         return __Pyx_Coroutine_FinishDelegation(gen);
     }
-    return __Pyx_Coroutine_SendEx(gen, Py_None);
+    return __Pyx_Coroutine_SendEx(gen, Py_None, 0);
 }
 
 static PyObject *__Pyx_Coroutine_Close(PyObject *self) {
@@ -796,7 +845,7 @@ static PyObject *__Pyx_Coroutine_Close(PyObject *self) {
     }
     if (err == 0)
         PyErr_SetNone(PyExc_GeneratorExit);
-    retval = __Pyx_Coroutine_SendEx(gen, NULL);
+    retval = __Pyx_Coroutine_SendEx(gen, NULL, 1);
     if (retval) {
         const char *msg;
         Py_DECREF(retval);
@@ -841,12 +890,15 @@ static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject
     if (yf) {
         PyObject *ret;
         Py_INCREF(yf);
-        if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) {
+        if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && close_on_genexit) {
+            // Asynchronous generators *should not* be closed right away.
+            // We have to allow some awaits to work it through, hence the
+            // `close_on_genexit` parameter here.
             int err = __Pyx_Coroutine_CloseIter(gen, yf);
             Py_DECREF(yf);
             __Pyx_Coroutine_Undelegate(gen);
             if (err < 0)
-                return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL));
+                return __Pyx_Coroutine_MethodReturn(self, __Pyx_Coroutine_SendEx(gen, NULL, 0));
             goto throw_here;
         }
         gen->is_running = 1;
@@ -888,11 +940,11 @@ static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject
         if (!ret) {
             ret = __Pyx_Coroutine_FinishDelegation(gen);
         }
-        return __Pyx_Coroutine_MethodReturn(ret);
+        return __Pyx_Coroutine_MethodReturn(self, ret);
     }
 throw_here:
     __Pyx_Raise(typ, val, tb, NULL);
-    return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL));
+    return __Pyx_Coroutine_MethodReturn(self, __Pyx_Coroutine_SendEx(gen, NULL, 0));
 }
 
 static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) {
@@ -958,6 +1010,14 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) {
         PyObject_GC_UnTrack(self);
     }
 
+#ifdef __Pyx_AsyncGen_USED
+    if (__Pyx_AsyncGen_CheckExact(self)) {
+        /* We have to handle this case for asynchronous generators
+           right here, because this code has to be between UNTRACK
+           and GC_Del. */
+        Py_CLEAR(((__pyx_PyAsyncGenObject*)self)->ag_finalizer);
+    }
+#endif
     __Pyx_Coroutine_clear(self);
     PyObject_GC_Del(gen);
 }
@@ -977,8 +1037,31 @@ static void __Pyx_Coroutine_del(PyObject *self) {
     self->ob_refcnt = 1;
 #endif
 
-    // Save the current exception, if any.
     __Pyx_PyThreadState_assign
+
+#ifdef __Pyx_AsyncGen_USED
+    if (__Pyx_AsyncGen_CheckExact(self)) {
+        __pyx_PyAsyncGenObject *agen = (__pyx_PyAsyncGenObject*)self;
+        PyObject *finalizer = agen->ag_finalizer;
+        if (finalizer && !agen->ag_closed) {
+            /* Save the current exception, if any. */
+            __Pyx_ErrFetch(&error_type, &error_value, &error_traceback);
+
+            res = __Pyx_PyObject_CallOneArg(finalizer, self);
+
+            if (res == NULL) {
+                PyErr_WriteUnraisable(self);
+            } else {
+                Py_DECREF(res);
+            }
+            /* Restore the saved exception. */
+            __Pyx_ErrRestore(error_type, error_value, error_traceback);
+            return;
+        }
+    }
+#endif
+
+    // Save the current exception, if any.
     __Pyx_ErrFetch(&error_type, &error_value, &error_traceback);
 
     res = __Pyx_Coroutine_Close(self);

Cython/Utility/ModuleSetupCode.c

@@ -387,15 +387,17 @@
     #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods
     #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async)
   #else
+    #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved))
+  #endif
+#else
+  #define __Pyx_PyType_AsAsync(obj) NULL
+#endif
+#ifndef __Pyx_PyAsyncMethodsStruct
     typedef struct {
         unaryfunc am_await;
         unaryfunc am_aiter;
         unaryfunc am_anext;
     } __Pyx_PyAsyncMethodsStruct;
-    #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved))
-  #endif
-#else
-  #define __Pyx_PyType_AsAsync(obj) NULL
 #endif
 
 // restrict

tests/run/test_asyncgen.py

+# cython: language_level=3, binding=True
+# mode: run
+# tag: pep525, asyncfor, await
+
+from __future__ import generator_stop
+
+import os
+import sys
+import inspect
+import types
+import unittest
+import contextlib
+
+#from unittest import mock
+
+#from test.support import import_module
+
+#asyncio = import_module("asyncio")
+
+try:
+    import asyncio
+except ImportError:
+    try:
+        from unittest import skip
+    except ImportError:
+        def requires_asyncio(c):
+            return None
+    else:
+        requires_asyncio = skip("tests require asyncio")
+    asyncio = None
+else:
+    def requires_asyncio(c):
+        return c
+
+
+# compiled exec()
+def exec(code_string, l, g):
+    from Cython.Compiler.Errors import CompileError
+    from Cython.Shadow import inline
+    try:
+        from StringIO import StringIO
+    except ImportError:
+        from io import StringIO
+
+    old_stderr = sys.stderr
+    try:
+        sys.stderr = StringIO()
+        ns = inline(code_string, locals=l, globals=g, lib_dir=os.path.dirname(__file__))
+    except CompileError as exc:
+        raise SyntaxError(str(exc))
+    finally:
+        sys.stderr = old_stderr
+    g.update(ns)
+
+
+class AwaitException(Exception):
+    pass
+
+
+@types.coroutine
+def awaitable(*, throw=False):
+    if throw:
+        yield ('throw',)
+    else:
+        yield ('result',)
+
+
+def run_until_complete(coro):
+    exc = False
+    while True:
+        try:
+            if exc:
+                exc = False
+                fut = coro.throw(AwaitException)
+            else:
+                fut = coro.send(None)
+        except StopIteration as ex:
+            return ex.args[0]
+
+        if fut == ('throw',):
+            exc = True
+
+
+def to_list(gen):
+    async def iterate():
+        res = []
+        async for i in gen:
+            res.append(i)
+        return res
+
+    return run_until_complete(iterate())
+
+
+class AsyncGenSyntaxTest(unittest.TestCase):
+
+    @contextlib.contextmanager
+    def assertRaisesRegex(self, exc_type, regex):
+        # the error messages usually don't match, so we just ignore them
+        try:
+            yield
+        except exc_type:
+            self.assertTrue(True)
+        else:
+            self.assertTrue(False)
+
+    def test_async_gen_syntax_01(self):
+        code = '''async def foo():
+            await abc
+            yield from 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_02(self):
+        code = '''async def foo():
+            yield from 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_03(self):
+        code = '''async def foo():
+            await abc
+            yield
+            return 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_04(self):
+        code = '''async def foo():
+            yield
+            return 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_05(self):
+        code = '''async def foo():
+            if 0:
+                yield
+            return 12
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+
+class AsyncGenTest(unittest.TestCase):
+
+    def compare_generators(self, sync_gen, async_gen):
+        def sync_iterate(g):
+            res = []
+            while True:
+                try:
+                    res.append(g.__next__())
+                except StopIteration:
+                    res.append('STOP')
+                    break
+                except Exception as ex:
+                    res.append(str(type(ex)))
+            return res
+
+        def async_iterate(g):
+            res = []
+            while True:
+                try:
+                    g.__anext__().__next__()
+                except StopAsyncIteration:
+                    res.append('STOP')
+                    break
+                except StopIteration as ex:
+                    if ex.args:
+                        res.append(ex.args[0])
+                    else:
+                        res.append('EMPTY StopIteration')
+                        break
+                except Exception as ex:
+                    res.append(str(type(ex)))
+            return res
+
+        sync_gen_result = sync_iterate(sync_gen)
+        async_gen_result = async_iterate(async_gen)
+        self.assertEqual(sync_gen_result, async_gen_result)
+        return async_gen_result
+
+    def test_async_gen_iteration_01(self):
+        async def gen():
+            await awaitable()
+            a = yield 123
+            self.assertIs(a, None)
+            await awaitable()
+            yield 456
+            await awaitable()
+            yield 789
+
+        self.assertEqual(to_list(gen()), [123, 456, 789])
+
+    def test_async_gen_iteration_02(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            await awaitable()
+
+        g = gen()
+        ai = g.__aiter__()
+        self.assertEqual(ai.__anext__().__next__(), ('result',))
+
+        try:
+            ai.__anext__().__next__()
+        except StopIteration as ex:
+            self.assertEqual(ex.args[0], 123)
+        else:
+            self.fail('StopIteration was not raised')
+
+        self.assertEqual(ai.__anext__().__next__(), ('result',))
+
+        try:
+            ai.__anext__().__next__()
+        except StopAsyncIteration as ex:
+            self.assertFalse(ex.args)
+        else:
+            self.fail('StopAsyncIteration was not raised')
+
+    def test_async_gen_exception_03(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            await awaitable(throw=True)
+            yield 456
+
+        with self.assertRaises(AwaitException):
+            to_list(gen())
+
+    def test_async_gen_exception_04(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            1 / 0
+
+        g = gen()
+        ai = g.__aiter__()
+        self.assertEqual(ai.__anext__().__next__(), ('result',))
+
+        try:
+            ai.__anext__().__next__()
+        except StopIteration as ex:
+            self.assertEqual(ex.args[0], 123)
+        else:
+            self.fail('StopIteration was not raised')
+
+        with self.assertRaises(ZeroDivisionError):
+            ai.__anext__().__next__()
+
+    def test_async_gen_exception_05(self):
+        async def gen():
+            yield 123
+            raise StopAsyncIteration
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'async generator.*StopAsyncIteration'):
+            to_list(gen())
+
+    def test_async_gen_exception_06(self):
+        async def gen():
+            yield 123
+            raise StopIteration
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'async generator.*StopIteration'):
+            to_list(gen())
+
+    def test_async_gen_exception_07(self):
+        def sync_gen():
+            try:
+                yield 1
+                1 / 0
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                yield 1
+                1 / 0
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_08(self):
+        def sync_gen():
+            try:
+                yield 1
+            finally:
+                yield 2
+                1 / 0
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                yield 1
+                await awaitable()
+            finally:
+                await awaitable()
+                yield 2
+                1 / 0
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_09(self):
+        def sync_gen():
+            try:
+                yield 1
+                1 / 0
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                await awaitable()
+                yield 1
+                1 / 0
+            finally:
+                yield 2
+                await awaitable()
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_10(self):
+        async def gen():
+            yield 123
+        with self.assertRaisesRegex(TypeError,
+                                    "non-None value .* async generator"):
+            gen().__anext__().send(100)
+
+    def test_async_gen_api_01(self):
+        async def gen():
+            yield 123
+
+        g = gen()
+
+        self.assertEqual(g.__name__, 'gen')
+        g.__name__ = '123'
+        self.assertEqual(g.__name__, '123')
+
+        self.assertIn('.gen', g.__qualname__)
+        g.__qualname__ = '123'
+        self.assertEqual(g.__qualname__, '123')
+
+        #self.assertIsNone(g.ag_await)
+        #self.assertIsInstance(g.ag_frame, types.FrameType)
+        self.assertFalse(g.ag_running)
+        #self.assertIsInstance(g.ag_code, types.CodeType)
+
+        self.assertTrue(inspect.isawaitable(g.aclose()))
+
+
+@requires_asyncio
+class AsyncGenAsyncioTest(unittest.TestCase):
+
+    def setUp(self):
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(None)
+
+    def tearDown(self):
+        self.loop.close()
+        self.loop = None
+
+    async def to_list(self, gen):
+        res = []
+        async for i in gen:
+            res.append(i)
+        return res
+
+    def test_async_gen_asyncio_01(self):
+        async def gen():
+            yield 1
+            await asyncio.sleep(0.01, loop=self.loop)
+            yield 2
+            await asyncio.sleep(0.01, loop=self.loop)
+            return
+            yield 3
+
+        res = self.loop.run_until_complete(self.to_list(gen()))
+        self.assertEqual(res, [1, 2])
+
+    def test_async_gen_asyncio_02(self):
+        async def gen():
+            yield 1
+            await asyncio.sleep(0.01, loop=self.loop)
+            yield 2
+            1 / 0
+            yield 3
+
+        with self.assertRaises(ZeroDivisionError):
+            self.loop.run_until_complete(self.to_list(gen()))
+
+    def test_async_gen_asyncio_03(self):
+        loop = self.loop
+
+        class Gen:
+            async def __aiter__(self):
+                yield 1
+                await asyncio.sleep(0.01, loop=loop)
+                yield 2
+
+        res = loop.run_until_complete(self.to_list(Gen()))
+        self.assertEqual(res, [1, 2])
+
+    def test_async_gen_asyncio_anext_04(self):
+        async def foo():
+            yield 1
+            await asyncio.sleep(0.01, loop=self.loop)
+            try:
+                yield 2
+                yield 3
+            except ZeroDivisionError:
+                yield 1000
+            await asyncio.sleep(0.01, loop=self.loop)
+            yield 4
+
+        async def run1():
+            it = foo().__aiter__()
+
+            self.assertEqual(await it.__anext__(), 1)
+            self.assertEqual(await it.__anext__(), 2)
+            self.assertEqual(await it.__anext__(), 3)
+            self.assertEqual(await it.__anext__(), 4)
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        async def run2():
+            it = foo().__aiter__()
+
+            self.assertEqual(await it.__anext__(), 1)
+            self.assertEqual(await it.__anext__(), 2)
+            try:
+                it.__anext__().throw(ZeroDivisionError)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1000)
+            else:
+                self.fail('StopIteration was not raised')
+            self.assertEqual(await it.__anext__(), 4)
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run1())
+        self.loop.run_until_complete(run2())
+
+    def test_async_gen_asyncio_anext_05(self):
+        async def foo():
+            v = yield 1
+            v = yield v
+            yield v * 100
+
+        async def run():
+            it = foo().__aiter__()
+
+            try:
+                it.__anext__().send(None)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1)
+            else:
+                self.fail('StopIteration was not raised')
+
+            try:
+                it.__anext__().send(10)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 10)
+            else:
+                self.fail('StopIteration was not raised')
+
+            try:
+                it.__anext__().send(12)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1200)
+            else:
+                self.fail('StopIteration was not raised')
+
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_anext_06(self):
+        DONE = 0
+
+        # test synchronous generators
+        def foo():
+            try:
+                yield
+            except:
+                pass
+        g = foo()
+        g.send(None)
+        with self.assertRaises(StopIteration):
+            g.send(None)
+
+        # now with asynchronous generators
+
+        async def gen():
+            nonlocal DONE
+            try:
+                yield
+            except:
+                pass
+            DONE = 1
+
+        async def run():
+            nonlocal DONE
+            g = gen()
+            await g.asend(None)
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+            DONE += 10
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 11)
+
+    def test_async_gen_asyncio_anext_tuple(self):
+        async def foo():
+            try:
+                yield (1,)
+            except ZeroDivisionError:
+                yield (2,)
+
+        async def run():
+            it = foo().__aiter__()
+
+            self.assertEqual(await it.__anext__(), (1,))
+            with self.assertRaises(StopIteration) as cm:
+                it.__anext__().throw(ZeroDivisionError)
+            self.assertEqual(cm.exception.args[0], (2,))
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_anext_stopiteration(self):
+        async def foo():
+            try:
+                yield StopIteration(1)
+            except ZeroDivisionError:
+                yield StopIteration(3)
+
+        async def run():
+            it = foo().__aiter__()
+
+            v = await it.__anext__()
+            self.assertIsInstance(v, StopIteration)
+            self.assertEqual(v.value, 1)
+            with self.assertRaises(StopIteration) as cm:
+                it.__anext__().throw(ZeroDivisionError)
+            v = cm.exception.args[0]
+            self.assertIsInstance(v, StopIteration)
+            self.assertEqual(v.value, 3)
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_aclose_06(self):
+        async def foo():
+            try:
+                yield 1
+                1 / 0
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                yield 12
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            await it.__anext__()
+            await gen.aclose()
+
+        with self.assertRaisesRegex(
+                RuntimeError,
+                "async generator ignored GeneratorExit"):
+            self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_aclose_07(self):
+        DONE = 0
+
+        async def foo():
+            nonlocal DONE
+            try:
+                yield 1
+                1 / 0
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE += 1
+            DONE += 1000
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            await it.__anext__()
+            await gen.aclose()
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_aclose_08(self):
+        DONE = 0
+
+        fut = asyncio.Future(loop=self.loop)
+
+        async def foo():
+            nonlocal DONE
+            try:
+                yield 1
+                await fut
+                DONE += 1000
+                yield 2
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE += 1
+            DONE += 1000
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            self.assertEqual(await it.__anext__(), 1)
+            t = self.loop.create_task(it.__anext__())
+            await asyncio.sleep(0.01, loop=self.loop)
+            await gen.aclose()
+            return t
+
+        t = self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+        # Silence ResourceWarnings
+        fut.cancel()
+        t.cancel()
+        self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop))
+
+    def test_async_gen_asyncio_gc_aclose_09(self):
+        DONE = 0
+
+        async def gen():
+            nonlocal DONE
+            try:
+                while True:
+                    yield 1
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+            await g.__anext__()
+            await g.__anext__()
+            del g
+
+            await asyncio.sleep(0.1, loop=self.loop)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_aclose_10(self):
+        DONE = 0
+
+        # test synchronous generators
+        def foo():
+            try:
+                yield
+            except:
+                pass
+        g = foo()
+        g.send(None)
+        g.close()
+
+        # now with asynchronous generators
+
+        async def gen():
+            nonlocal DONE
+            try:
+                yield
+            except:
+                pass
+            DONE = 1
+
+        async def run():
+            nonlocal DONE
+            g = gen()
+            await g.asend(None)
+            await g.aclose()
+            DONE += 10
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 11)
+
+    def test_async_gen_asyncio_aclose_11(self):
+        DONE = 0
+
+        # test synchronous generators
+        def foo():
+            try:
+                yield
+            except:
+                pass
+            yield
+        g = foo()
+        g.send(None)
+        with self.assertRaisesRegex(RuntimeError, 'ignored GeneratorExit'):
+            g.close()
+
+        # now with asynchronous generators
+
+        async def gen():
+            nonlocal DONE
+            try:
+                yield
+            except:
+                pass
+            yield
+            DONE += 1
+
+        async def run():
+            nonlocal DONE
+            g = gen()
+            await g.asend(None)
+            with self.assertRaisesRegex(RuntimeError, 'ignored GeneratorExit'):
+                await g.aclose()
+            DONE += 10
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 10)
+
+    def test_async_gen_asyncio_asend_01(self):
+        DONE = 0
+
+        # Sanity check:
+        def sgen():
+            v = yield 1
+            yield v * 2
+        sg = sgen()
+        v = sg.send(None)
+        self.assertEqual(v, 1)
+        v = sg.send(100)
+        self.assertEqual(v, 200)
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                v = yield 1
+                await asyncio.sleep(0.01, loop=self.loop)
+                yield v * 2
+                await asyncio.sleep(0.01, loop=self.loop)
+                return
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            v = await g.asend(100)
+            self.assertEqual(v, 200)
+
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_asend_02(self):
+        DONE = 0
+
+        async def sleep_n_crash(delay):
+            await asyncio.sleep(delay, loop=self.loop)
+            1 / 0
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                v = yield 1
+                await sleep_n_crash(0.01)
+                DONE += 1000
+                yield v * 2
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            await g.asend(100)
+
+        with self.assertRaises(ZeroDivisionError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_asend_03(self):
+        DONE = 0
+
+        async def sleep_n_crash(delay):
+            fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop),
+                                        loop=self.loop)
+            self.loop.call_later(delay / 2, lambda: fut.cancel())
+            return await fut
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                v = yield 1
+                await sleep_n_crash(0.01)
+                DONE += 1000
+                yield v * 2
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            await g.asend(100)
+
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_athrow_01(self):
+        DONE = 0
+
+        class FooEr(Exception):
+            pass
+
+        # Sanity check:
+        def sgen():
+            try:
+                v = yield 1
+            except FooEr:
+                v = 1000
+            yield v * 2
+        sg = sgen()
+        v = sg.send(None)
+        self.assertEqual(v, 1)
+        v = sg.throw(FooEr)
+        self.assertEqual(v, 2000)
+        with self.assertRaises(StopIteration):
+            sg.send(None)
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                try:
+                    v = yield 1
+                except FooEr:
+                    v = 1000
+                    await asyncio.sleep(0.01, loop=self.loop)
+                yield v * 2
+                await asyncio.sleep(0.01, loop=self.loop)
+                # return
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            v = await g.athrow(FooEr)
+            self.assertEqual(v, 2000)
+
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_athrow_02(self):
+        DONE = 0
+
+        class FooEr(Exception):
+            pass
+
+        async def sleep_n_crash(delay):
+            fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop),
+                                        loop=self.loop)
+            self.loop.call_later(delay / 2, lambda: fut.cancel())
+            return await fut
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                try:
+                    v = yield 1
+                except FooEr:
+                    await sleep_n_crash(0.01)
+                yield v * 2
+                await asyncio.sleep(0.01, loop=self.loop)
+                # return
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            try:
+                await g.athrow(FooEr)
+            except asyncio.CancelledError:
+                self.assertEqual(DONE, 1)
+                raise
+            else:
+                self.fail('CancelledError was not raised')
+
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_athrow_03(self):
+        DONE = 0
+
+        # test synchronous generators
+        def foo():
+            try:
+                yield
+            except:
+                pass
+        g = foo()
+        g.send(None)
+        with self.assertRaises(StopIteration):
+            g.throw(ValueError)
+
+        # now with asynchronous generators
+
+        async def gen():
+            nonlocal DONE
+            try:
+                yield
+            except:
+                pass
+            DONE = 1
+
+        async def run():
+            nonlocal DONE
+            g = gen()
+            await g.asend(None)
+            with self.assertRaises(StopAsyncIteration):
+                await g.athrow(ValueError)
+            DONE += 10
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 11)
+
+    def test_async_gen_asyncio_athrow_tuple(self):
+        async def gen():
+            try:
+                yield 1
+            except ZeroDivisionError:
+                yield (2,)
+
+        async def run():
+            g = gen()
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+            v = await g.athrow(ZeroDivisionError)
+            self.assertEqual(v, (2,))
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_athrow_stopiteration(self):
+        async def gen():
+            try:
+                yield 1
+            except ZeroDivisionError:
+                yield StopIteration(2)
+
+        async def run():
+            g = gen()
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+            v = await g.athrow(ZeroDivisionError)
+            self.assertIsInstance(v, StopIteration)
+            self.assertEqual(v.value, 2)
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_shutdown_01(self):
+        finalized = 0
+
+        async def waiter(timeout):
+            nonlocal finalized
+            try:
+                await asyncio.sleep(timeout, loop=self.loop)
+                yield 1
+            finally:
+                await asyncio.sleep(0, loop=self.loop)
+                finalized += 1
+
+        async def wait():
+            async for _ in waiter(1):
+                pass
+
+        t1 = self.loop.create_task(wait())
+        t2 = self.loop.create_task(wait())
+
+        self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+
+        self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+        self.assertEqual(finalized, 2)
+
+        # Silence warnings
+        t1.cancel()
+        t2.cancel()
+        self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+
+    def test_async_gen_asyncio_shutdown_02(self):
+        logged = 0
+
+        def logger(loop, context):
+            nonlocal logged
+            self.assertIn('asyncgen', context)
+            expected = 'an error occurred during closing of asynchronous'
+            if expected in context['message']:
+                logged += 1
+
+        async def waiter(timeout):
+            try:
+                await asyncio.sleep(timeout, loop=self.loop)
+                yield 1
+            finally:
+                1 / 0
+
+        async def wait():
+            async for _ in waiter(1):
+                pass
+
+        t = self.loop.create_task(wait())
+        self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+
+        self.loop.set_exception_handler(logger)
+        self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+
+        self.assertEqual(logged, 1)
+
+        # Silence warnings
+        t.cancel()
+        self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+
+if __name__ == "__main__":
+    unittest.main()