implement "async def" statement and "await" expression (PEP 492)

Cython/Compiler/Code.py

@@ -49,7 +49,8 @@ non_portable_builtins_map = {
     'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
     'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
     'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
-    }
+    'StopAsyncIteration': ('PY_VERSION_HEX < 0x030500B1', 'StopIteration'),
+}
 
 basicsize_builtins_map = {
     # builtins whose type has a different tp_basicsize than sizeof(...)

Cython/Compiler/ExprNodes.py

@@ -8593,10 +8593,12 @@ class YieldExprNode(ExprNode):
     type = py_object_type
     label_num = 0
     is_yield_from = False
+    is_await = False
+    expr_keyword = 'yield'
 
     def analyse_types(self, env):
         if not self.label_num:
-            error(self.pos, "'yield' not supported here")
+            error(self.pos, "'%s' not supported here" % self.expr_keyword)
         self.is_temp = 1
         if self.arg is not None:
             self.arg = self.arg.analyse_types(env)
@@ -8661,6 +8663,7 @@ class YieldExprNode(ExprNode):
 class YieldFromExprNode(YieldExprNode):
     # "yield from GEN" expression
     is_yield_from = True
+    expr_keyword = 'yield from'
 
     def coerce_yield_argument(self, env):
         if not self.arg.type.is_string:
@@ -8668,14 +8671,17 @@ class YieldFromExprNode(YieldExprNode):
             error(self.pos, "yielding from non-Python object not supported")
         self.arg = self.arg.coerce_to_pyobject(env)
 
-    def generate_evaluation_code(self, code):
-        code.globalstate.use_utility_code(UtilityCode.load_cached("YieldFrom", "Coroutine.c"))
+    def yield_from_func(self, code):
+        code.globalstate.use_utility_code(UtilityCode.load_cached("GeneratorYieldFrom", "Coroutine.c"))
+        return "__Pyx_Generator_Yield_From"
 
+    def generate_evaluation_code(self, code):
         self.arg.generate_evaluation_code(code)
-        code.putln("%s = __Pyx_Generator_Yield_From(%s, %s);" % (
+        code.putln("%s = %s(%s, %s);" % (
             Naming.retval_cname,
+            self.yield_from_func(code),
             Naming.generator_cname,
-            self.arg.result_as(py_object_type)))
+            self.arg.py_result()))
         self.arg.generate_disposal_code(code)
         self.arg.free_temps(code)
         code.put_xgotref(Naming.retval_cname)
@@ -8687,9 +8693,7 @@ class YieldFromExprNode(YieldExprNode):
         if self.result_is_used:
             # YieldExprNode has allocated the result temp for us
             code.putln("%s = NULL;" % self.result())
-            code.putln("if (unlikely(__Pyx_PyGen_FetchStopIterationValue(&%s) < 0)) %s" % (
-                self.result(),
-                code.error_goto(self.pos)))
+            code.put_error_if_neg(self.pos, "__Pyx_PyGen_FetchStopIterationValue(&%s)" % self.result())
             code.put_gotref(self.result())
         else:
             code.putln("PyObject* exc_type = PyErr_Occurred();")
@@ -8700,6 +8704,25 @@ class YieldFromExprNode(YieldExprNode):
             code.putln("}")
         code.putln("}")
 
+
+class AwaitExprNode(YieldFromExprNode):
+    # 'await' expression node
+    #
+    # arg         ExprNode   the Awaitable value to await
+    # label_num   integer    yield label number
+    # is_yield_from  boolean is a YieldFromExprNode to delegate to another generator
+    is_await = True
+    expr_keyword = 'await'
+
+    def coerce_yield_argument(self, env):
+        # FIXME: use same check as in YieldFromExprNode.coerce_yield_argument() ?
+        self.arg = self.arg.coerce_to_pyobject(env)
+
+    def yield_from_func(self, code):
+        code.globalstate.use_utility_code(UtilityCode.load_cached("CoroutineYieldFrom", "Coroutine.c"))
+        return "__Pyx_Coroutine_Yield_From"
+
+
 class GlobalsExprNode(AtomicExprNode):
     type = dict_type
     is_temp = 1

Cython/Compiler/Nodes.py

@@ -1576,7 +1576,9 @@ class FuncDefNode(StatNode, BlockNode):
     #  directive_returns [ExprNode] type defined by cython.returns(...)
     #  star_arg      PyArgDeclNode or None  * argument
     #  starstar_arg  PyArgDeclNode or None  ** argument
-
+    #
+    #  is_async_def  boolean          is a Coroutine function
+    #
     #  has_fused_arguments  boolean
     #       Whether this cdef function has fused parameters. This is needed
     #       by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
@@ -1588,6 +1590,7 @@ class FuncDefNode(StatNode, BlockNode):
     pymethdef_required = False
     is_generator = False
     is_generator_body = False
+    is_async_def = False
     modifiers = []
     has_fused_arguments = False
     star_arg = None
@@ -3936,6 +3939,7 @@ class GeneratorDefNode(DefNode):
     #
 
     is_generator = True
+    is_coroutine = False
     needs_closure = True
 
     child_attrs = DefNode.child_attrs + ["gbody"]
@@ -3956,8 +3960,9 @@ class GeneratorDefNode(DefNode):
         qualname = code.intern_identifier(self.qualname)
 
         code.putln('{')
-        code.putln('__pyx_CoroutineObject *gen = __Pyx_Generator_New('
+        code.putln('__pyx_CoroutineObject *gen = __Pyx_%s_New('
                    '(__pyx_coroutine_body_t) %s, (PyObject *) %s, %s, %s); %s' % (
+                       'Coroutine' if self.is_coroutine else 'Generator',
                        body_cname, Naming.cur_scope_cname, name, qualname,
                        code.error_goto_if_null('gen', self.pos)))
         code.put_decref(Naming.cur_scope_cname, py_object_type)
@@ -3972,13 +3977,18 @@ class GeneratorDefNode(DefNode):
         code.putln('}')
 
     def generate_function_definitions(self, env, code):
-        env.use_utility_code(UtilityCode.load_cached("Generator", "Coroutine.c"))
+        env.use_utility_code(UtilityCode.load_cached(
+            'Coroutine' if self.is_coroutine else 'Generator', "Coroutine.c"))
 
         self.gbody.generate_function_header(code, proto=True)
         super(GeneratorDefNode, self).generate_function_definitions(env, code)
         self.gbody.generate_function_definitions(env, code)
 
 
+class AsyncDefNode(GeneratorDefNode):
+    is_coroutine = True
+
+
 class GeneratorBodyDefNode(DefNode):
     # Main code body of a generator implemented as a DefNode.
     #
@@ -7108,7 +7118,7 @@ class GILStatNode(NogilTryFinallyStatNode):
         from .ParseTreeTransforms import YieldNodeCollector
         collector = YieldNodeCollector()
         collector.visitchildren(body)
-        if not collector.yields:
+        if not collector.yields and not collector.awaits:
             return
 
         if state == 'gil':

Cython/Compiler/ParseTreeTransforms.py

@@ -200,7 +200,7 @@ class PostParse(ScopeTrackingTransform):
         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
         collector = YieldNodeCollector()
         collector.visitchildren(node.result_expr)
-        if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
+        if collector.yields or collector.awaits or isinstance(node.result_expr, ExprNodes.YieldExprNode):
             body = Nodes.ExprStatNode(
                 node.result_expr.pos, expr=node.result_expr)
         else:
@@ -2205,6 +2205,7 @@ class YieldNodeCollector(TreeVisitor):
     def __init__(self):
         super(YieldNodeCollector, self).__init__()
         self.yields = []
+        self.awaits = []
         self.returns = []
         self.has_return_value = False
 
@@ -2215,6 +2216,10 @@ class YieldNodeCollector(TreeVisitor):
         self.yields.append(node)
         self.visitchildren(node)
 
+    def visit_AwaitExprNode(self, node):
+        self.awaits.append(node)
+        self.visitchildren(node)
+
     def visit_ReturnStatNode(self, node):
         self.visitchildren(node)
         if node.value:
@@ -2250,27 +2255,36 @@ class MarkClosureVisitor(CythonTransform):
         collector = YieldNodeCollector()
         collector.visitchildren(node)
 
+        if node.is_async_def:
             if collector.yields:
-            if isinstance(node, Nodes.CFuncDefNode):
-                # Will report error later
+                error(collector.yields[0].pos, "'yield' not allowed in async coroutines (use 'await')")
+            yields = collector.awaits
+        elif collector.yields:
+            if collector.awaits:
+                error(collector.yields[0].pos, "'await' not allowed in generators (use 'yield')")
+            yields = collector.yields
+        else:
             return node
-            for i, yield_expr in enumerate(collector.yields, 1):
+
+        for i, yield_expr in enumerate(yields, 1):
             yield_expr.label_num = i
         for retnode in collector.returns:
             retnode.in_generator = True
 
         gbody = Nodes.GeneratorBodyDefNode(
             pos=node.pos, name=node.name, body=node.body)
-            generator = Nodes.GeneratorDefNode(
+        coroutine = (Nodes.AsyncDefNode if node.is_async_def else Nodes.GeneratorDefNode)(
             pos=node.pos, name=node.name, args=node.args,
             star_arg=node.star_arg, starstar_arg=node.starstar_arg,
             doc=node.doc, decorators=node.decorators,
             gbody=gbody, lambda_name=node.lambda_name)
-            return generator
-        return node
+        return coroutine
 
     def visit_CFuncDefNode(self, node):
-        self.visit_FuncDefNode(node)
+        self.needs_closure = False
+        self.visitchildren(node)
+        node.needs_closure = self.needs_closure
+        self.needs_closure = True
         if node.needs_closure and node.overridable:
             error(node.pos, "closures inside cpdef functions not yet supported")
         return node
@@ -2287,6 +2301,7 @@ class MarkClosureVisitor(CythonTransform):
         self.needs_closure = True
         return node
 
+
 class CreateClosureClasses(CythonTransform):
     # Output closure classes in module scope for all functions
     # that really need it.

Cython/Compiler/Parsing.pxd

@@ -44,6 +44,8 @@ cdef p_typecast(PyrexScanner s)
 cdef p_sizeof(PyrexScanner s)
 cdef p_yield_expression(PyrexScanner s)
 cdef p_yield_statement(PyrexScanner s)
+cdef p_await_expression(PyrexScanner s)
+cdef p_async_statement(PyrexScanner s, ctx)
 cdef p_power(PyrexScanner s)
 cdef p_new_expr(PyrexScanner s)
 cdef p_trailer(PyrexScanner s, node1)
@@ -128,7 +130,7 @@ cdef p_IF_statement(PyrexScanner s, ctx)
 cdef p_statement(PyrexScanner s, ctx, bint first_statement = *)
 cdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *)
 cdef p_suite(PyrexScanner s, ctx = *)
-cdef tuple p_suite_with_docstring(PyrexScanner s, ctx, with_doc_only = *)
+cdef tuple p_suite_with_docstring(PyrexScanner s, ctx, bint with_doc_only=*)
 cdef tuple _extract_docstring(node)
 cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *)
 
@@ -176,7 +178,7 @@ cdef p_c_modifiers(PyrexScanner s)
 cdef p_c_func_or_var_declaration(PyrexScanner s, pos, ctx)
 cdef p_ctypedef_statement(PyrexScanner s, ctx)
 cdef p_decorators(PyrexScanner s)
-cdef p_def_statement(PyrexScanner s, list decorators = *)
+cdef p_def_statement(PyrexScanner s, list decorators=*, bint is_async_def=*)
 cdef p_varargslist(PyrexScanner s, terminator=*, bint annotated = *)
 cdef p_py_arg_decl(PyrexScanner s, bint annotated = *)
 cdef p_class_statement(PyrexScanner s, decorators)

Cython/Compiler/Parsing.py

@@ -55,6 +55,7 @@ class Ctx(object):
         d.update(kwds)
         return ctx
 
+
 def p_ident(s, message="Expected an identifier"):
     if s.sy == 'IDENT':
         name = s.systring
@@ -350,6 +351,7 @@ def p_sizeof(s):
     s.expect(')')
     return node
 
+
 def p_yield_expression(s):
     # s.sy == "yield"
     pos = s.position()
@@ -370,19 +372,52 @@ def p_yield_expression(s):
     else:
         return ExprNodes.YieldExprNode(pos, arg=arg)
 
+
 def p_yield_statement(s):
     # s.sy == "yield"
     yield_expr = p_yield_expression(s)
     return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr)
 
-#power: atom trailer* ('**' factor)*
+
+def p_async_statement(s, ctx, decorators):
+    # s.sy >> 'async' ...
+    if s.sy == 'def':
+        # 'async def' statements aren't allowed in pxd files
+        if 'pxd' in ctx.level:
+            s.error('def statement not allowed here')
+        s.level = ctx.level
+        return p_def_statement(s, decorators, is_async_def=True)
+    elif decorators:
+        s.error("Decorators can only be followed by functions or classes")
+    elif s.sy == 'for':
+        #s.error("'async for' is not currently supported", fatal=False)
+        return p_statement(s, ctx)  # TODO: implement
+    elif s.sy == 'with':
+        #s.error("'async with' is not currently supported", fatal=False)
+        return p_statement(s, ctx)  # TODO: implement
+    else:
+        s.error("expected one of 'def', 'for', 'with' after 'async'")
+
+
+def p_await_expression(s):
+    n1 = p_atom(s)
+
+
+#power: atom_expr ('**' factor)*
+#atom_expr: ['await'] atom trailer*
 
 def p_power(s):
     if s.systring == 'new' and s.peek()[0] == 'IDENT':
         return p_new_expr(s)
+    await_pos = None
+    if s.sy == 'await':
+        await_pos = s.position()
+        s.next()
     n1 = p_atom(s)
     while s.sy in ('(', '[', '.'):
         n1 = p_trailer(s, n1)
+    if await_pos:
+        n1 = ExprNodes.AwaitExprNode(await_pos, arg=n1)
     if s.sy == '**':
         pos = s.position()
         s.next()
@@ -390,6 +425,7 @@ def p_power(s):
         n1 = ExprNodes.binop_node(pos, '**', n1, n2)
     return n1
 
+
 def p_new_expr(s):
     # s.systring == 'new'.
     pos = s.position()
@@ -1929,12 +1965,14 @@ def p_statement(s, ctx, first_statement = 0):
             s.error('decorator not allowed here')
         s.level = ctx.level
         decorators = p_decorators(s)
-        bad_toks =  'def', 'cdef', 'cpdef', 'class'
-        if not ctx.allow_struct_enum_decorator and s.sy not in bad_toks:
+        if not ctx.allow_struct_enum_decorator and s.sy not in ('def', 'cdef', 'cpdef', 'class'):
+            if s.sy == 'IDENT' and s.systring == 'async':
+                pass  # handled below
+            else:
                 s.error("Decorators can only be followed by functions or classes")
     elif s.sy == 'pass' and cdef_flag:
         # empty cdef block
-        return p_pass_statement(s, with_newline = 1)
+        return p_pass_statement(s, with_newline=1)
 
     overridable = 0
     if s.sy == 'cdef':
@@ -1948,11 +1986,11 @@ def p_statement(s, ctx, first_statement = 0):
         if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
             s.error('cdef statement not allowed here')
         s.level = ctx.level
-        node = p_cdef_statement(s, ctx(overridable = overridable))
+        node = p_cdef_statement(s, ctx(overridable=overridable))
         if decorators is not None:
-            tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode
+            tup = (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode)
             if ctx.allow_struct_enum_decorator:
-                tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode
+                tup += (Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode)
             if not isinstance(node, tup):
                 s.error("Decorators can only be followed by functions or classes")
             node.decorators = decorators
@@ -1995,9 +2033,25 @@ def p_statement(s, ctx, first_statement = 0):
                 return p_try_statement(s)
             elif s.sy == 'with':
                 return p_with_statement(s)
+            elif s.sy == 'async':
+                s.next()
+                return p_async_statement(s, ctx, decorators)
             else:
-                return p_simple_statement_list(
-                    s, ctx, first_statement = first_statement)
+                if s.sy == 'IDENT' and s.systring == 'async':
+                    # PEP 492 enables the async/await keywords when it spots "async def ..."
+                    s.next()
+                    if s.sy == 'def':
+                        s.enable_keyword('async')
+                        s.enable_keyword('await')
+                        result = p_async_statement(s, ctx, decorators)
+                        s.enable_keyword('await')
+                        s.disable_keyword('async')
+                        return result
+                    elif decorators:
+                        s.error("Decorators can only be followed by functions or classes")
+                    s.put_back('IDENT', 'async')
+                return p_simple_statement_list(s, ctx, first_statement=first_statement)
+
 
 def p_statement_list(s, ctx, first_statement = 0):
     # Parse a series of statements separated by newlines.
@@ -3002,7 +3056,8 @@ def p_decorators(s):
         s.expect_newline("Expected a newline after decorator")
     return decorators
 
-def p_def_statement(s, decorators=None):
+
+def p_def_statement(s, decorators=None, is_async_def=False):
     # s.sy == 'def'
     pos = s.position()
     s.next()
@@ -3017,10 +3072,11 @@ def p_def_statement(s, decorators=None):
         s.next()
         return_type_annotation = p_test(s)
     doc, body = p_suite_with_docstring(s, Ctx(level='function'))
-    return Nodes.DefNode(pos, name = name, args = args,
-        star_arg = star_arg, starstar_arg = starstar_arg,
-        doc = doc, body = body, decorators = decorators,
-        return_type_annotation = return_type_annotation)
+    return Nodes.DefNode(
+        pos, name=name, args=args, star_arg=star_arg, starstar_arg=starstar_arg,
+        doc=doc, body=body, decorators=decorators, is_async_def=is_async_def,
+        return_type_annotation=return_type_annotation)
+
 
 def p_varargslist(s, terminator=')', annotated=1):
     args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,

Cython/Compiler/Scanning.pxd

@@ -30,6 +30,7 @@ cdef class PyrexScanner(Scanner):
     cdef public bint in_python_file
     cdef public source_encoding
     cdef set keywords
+    cdef public dict keywords_stack
     cdef public list indentation_stack
     cdef public indentation_char
     cdef public int bracket_nesting_level
@@ -57,3 +58,5 @@ cdef class PyrexScanner(Scanner):
     cdef expect_indent(self)
     cdef expect_dedent(self)
     cdef expect_newline(self, message=*, bint ignore_semicolon=*)
+    cdef enable_keyword(self, name)
+    cdef disable_keyword(self, name)

Cython/Compiler/Scanning.py

@@ -319,6 +319,7 @@ class PyrexScanner(Scanner):
             self.in_python_file = False
             self.keywords = set(pyx_reserved_words)
         self.trace = trace_scanner
+        self.keywords_stack = {}
         self.indentation_stack = [0]
         self.indentation_char = None
         self.bracket_nesting_level = 0
@@ -497,3 +498,18 @@ class PyrexScanner(Scanner):
             self.expect('NEWLINE', message)
         if useless_trailing_semicolon is not None:
             warning(useless_trailing_semicolon, "useless trailing semicolon")
+
+    def enable_keyword(self, name):
+        if name in self.keywords_stack:
+            self.keywords_stack[name] += 1
+        else:
+            self.keywords_stack[name] = 1
+            self.keywords.add(name)
+
+    def disable_keyword(self, name):
+        count = self.keywords_stack.get(name, 1)
+        if count == 1:
+            self.keywords.discard(name)
+            del self.keywords_stack[name]
+        else:
+            self.keywords_stack[name] = count - 1

Cython/Parser/Grammar

@@ -13,7 +13,8 @@ eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_PY_NAME [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
-decorated: decorators (classdef | funcdef | cdef_stmt)
+decorated: decorators (classdef | funcdef | async_funcdef | cdef_stmt)
+async_funcdef: 'async' funcdef
 funcdef: 'def' PY_NAME parameters ['->' test] ':' suite
 parameters: '(' [typedargslist] ')'
 typedargslist: (tfpdef ['=' (test | '*')] (',' tfpdef ['=' (test | '*')])* [','
@@ -96,7 +97,8 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power | address | size_of | cast
-power: atom trailer* ['**' factor]
+power: atom_expr ['**' factor]
+atom_expr: ['await'] atom trailer*
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |

Cython/Utility/Coroutine.c

-//////////////////// YieldFrom.proto ////////////////////
+//////////////////// GeneratorYieldFrom.proto ////////////////////
 
 static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source);
 
-//////////////////// YieldFrom ////////////////////
+//////////////////// GeneratorYieldFrom ////////////////////
 //@requires: Generator
 
 static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) {
@@ -21,6 +21,125 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject
 }
 
 
+//////////////////// CoroutineYieldFrom.proto ////////////////////
+
+static CYTHON_INLINE PyObject* __Pyx_Coroutine_Yield_From(__pyx_CoroutineObject *gen, PyObject *source);
+
+//////////////////// CoroutineYieldFrom ////////////////////
+//@requires: Coroutine
+//@requires: GetAwaitIter
+
+static CYTHON_INLINE PyObject* __Pyx_Coroutine_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) {
+    PyObject *retval;
+    if (__Pyx_Coroutine_CheckExact(source)) {
+        retval = __Pyx_Generator_Next(source);
+        if (retval) {
+            Py_INCREF(source);
+            gen->yieldfrom = source;
+            return retval;
+        }
+    } else {
+        PyObject *source_gen = __Pyx__Coroutine_GetAwaitableIter(source);
+        if (unlikely(!source_gen))
+            return NULL;
+        // source_gen is now the iterator, make the first next() call
+        if (__Pyx_Coroutine_CheckExact(source_gen)) {
+            retval = __Pyx_Generator_Next(source_gen);
+        } else {
+            retval = Py_TYPE(source_gen)->tp_iternext(source_gen);
+        }
+        if (retval) {
+            gen->yieldfrom = source_gen;
+            return retval;
+        }
+        Py_DECREF(source_gen);
+    }
+    return NULL;
+}
+
+
+//////////////////// GetAwaitIter.proto ////////////////////
+
+static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAwaitableIter(PyObject *o); /*proto*/
+static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *o); /*proto*/
+
+//////////////////// GetAwaitIter ////////////////////
+//@requires: Coroutine
+//@requires: ObjectHandling.c::PyObjectGetAttrStr
+//@requires: ObjectHandling.c::PyObjectCallNoArg
+//@requires: ObjectHandling.c::PyObjectCallOneArg
+
+static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAwaitableIter(PyObject *o) {
+#ifdef __Pyx_Coroutine_USED
+    if (__Pyx_Coroutine_CheckExact(o)) {
+        Py_INCREF(o);
+        return o;
+    }
+#endif
+    return __Pyx__Coroutine_GetAwaitableIter(o);
+}
+
+// copied and adapted from genobject.c in Py3.5
+static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *o) {
+    PyObject *res;
+#if PY_VERSION_HEX >= 0x030500B1
+    unaryfunc getter = NULL;
+    PyTypeObject *ot;
+
+    ot = Py_TYPE(o);
+    if (likely(ot->tp_as_async)) {
+        getter = (unaryfunc) ot->tp_as_async->am_await;
+    }
+    if (unlikely(getter)) goto slot_error;
+    res = (*getter)(o);
+#else
+    PyObject *method = __Pyx_PyObject_GetAttrStr(o, PYIDENT("__await__"));
+    if (unlikely(!method)) goto slot_error;
+    #if CYTHON_COMPILING_IN_CPYTHON
+    if (likely(PyMethod_Check(method))) {
+        PyObject *self = PyMethod_GET_SELF(method);
+        if (likely(self)) {
+            PyObject *function = PyMethod_GET_FUNCTION(method);
+            res = __Pyx_PyObject_CallOneArg(function, self);
+        } else
+            res = __Pyx_PyObject_CallNoArg(method);
+    } else
+    #endif
+        res = __Pyx_PyObject_CallNoArg(method);
+    Py_DECREF(method);
+#endif
+    if (unlikely(!res)) goto bad;
+    if (!PyIter_Check(res)) {
+        PyErr_Format(PyExc_TypeError,
+                     "__await__() returned non-iterator of type '%.100s'",
+                     Py_TYPE(res)->tp_name);
+        Py_CLEAR(res);
+    } else {
+        int is_coroutine = 0;
+        #ifdef __Pyx_Coroutine_USED
+        is_coroutine |= __Pyx_Coroutine_CheckExact(res);
+        #endif
+        #if PY_VERSION_HEX >= 0x030500B1
+        is_coroutine |= PyGen_CheckCoroutineExact(res);
+        #endif
+        if (unlikely(is_coroutine)) {
+            /* __await__ must return an *iterator*, not
+               a coroutine or another awaitable (see PEP 492) */
+            PyErr_SetString(PyExc_TypeError,
+                            "__await__() returned a coroutine");
+            Py_CLEAR(res);
+        }
+    }
+    return res;
+slot_error:
+    PyErr_Format(PyExc_TypeError,
+                 "object %.100s can't be used in 'await' expression",
+                 Py_TYPE(o)->tp_name);
+bad:
+    return NULL;
+}
+
+
 //////////////////// pep479.proto ////////////////////
 
 static void __Pyx_Generator_Replace_StopIteration(void); /*proto*/
@@ -107,6 +226,7 @@ static int __pyx_Generator_init(void);
 #include <structmember.h>
 #include <frameobject.h>
 
+static PyObject *__Pyx_Generator_Next(PyObject *self);
 static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value);
 static PyObject *__Pyx_Coroutine_Close(PyObject *self);
 static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args);
@@ -403,6 +523,28 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) {
     return err;
 }
 
+static PyObject *__Pyx_Generator_Next(PyObject *self) {
+    __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self;
+    PyObject *yf = gen->yieldfrom;
+    if (unlikely(__Pyx_Coroutine_CheckRunning(gen)))
+        return NULL;
+    if (yf) {
+        PyObject *ret;
+        // FIXME: does this really need an INCREF() ?
+        //Py_INCREF(yf);
+        // YieldFrom code ensures that yf is an iterator
+        gen->is_running = 1;
+        ret = Py_TYPE(yf)->tp_iternext(yf);
+        gen->is_running = 0;
+        //Py_DECREF(yf);
+        if (likely(ret)) {
+            return ret;
+        }
+        return __Pyx_Coroutine_FinishDelegation(gen);
+    }
+    return __Pyx_Coroutine_SendEx(gen, Py_None);
+}
+
 static PyObject *__Pyx_Coroutine_Close(PyObject *self) {
     __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self;
     PyObject *retval, *raised_exception;
@@ -729,6 +871,14 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_New(PyTypeObject* type, __pyx_cor
 //@requires: CoroutineBase
 //@requires: PatchGeneratorABC
 
+#if PY_VERSION_HEX >= 0x030500B1
+static PyAsyncMethods __pyx_Coroutine_as_async {
+    0, /*am_await*/
+    0, /*am_aiter*/
+    0, /*am_anext*/
+}
+#endif
+
 static PyTypeObject __pyx_CoroutineType_type = {
     PyVarObject_HEAD_INIT(0, 0)
     "coroutine",                        /*tp_name*/
@@ -738,10 +888,10 @@ static PyTypeObject __pyx_CoroutineType_type = {
     0,                                  /*tp_print*/
     0,                                  /*tp_getattr*/
     0,                                  /*tp_setattr*/
-#if PY_MAJOR_VERSION < 3
-    0,                                  /*tp_compare*/
+#if PY_VERSION_HEX >= 0x030500B1
+    __pyx_Coroutine_as_async,           /*tp_as_async*/
 #else
-    0,                                  /*reserved*/
+    0,                                  /*tp_reserved resp. tp_compare*/
 #endif
     0,                                  /*tp_repr*/
     0,                                  /*tp_as_number*/
@@ -759,8 +909,9 @@ static PyTypeObject __pyx_CoroutineType_type = {
     0,                                  /*tp_clear*/
     0,                                  /*tp_richcompare*/
     offsetof(__pyx_CoroutineObject, gi_weakreflist), /*tp_weaklistoffset*/
+// no tp_iter() as iterator is only available through __await__()
     0,                                  /*tp_iter*/
-    0,                                  /*tp_iternext*/
+    (iternextfunc) __Pyx_Generator_Next, /*tp_iternext*/
     __pyx_Coroutine_methods,            /*tp_methods*/
     __pyx_Coroutine_memberlist,         /*tp_members*/
     __pyx_Coroutine_getsets,            /*tp_getset*/
@@ -790,7 +941,7 @@ static PyTypeObject __pyx_CoroutineType_type = {
 #endif
 };
 
-static int __pyx_Generator_init(void) {
+static int __pyx_Coroutine_init(void) {
     // on Windows, C-API functions can't be used in slots statically
     __pyx_CoroutineType_type.tp_getattro = PyObject_GenericGetAttr;
 
@@ -801,35 +952,10 @@ static int __pyx_Generator_init(void) {
     return 0;
 }
 
-
 //////////////////// Generator ////////////////////
 //@requires: CoroutineBase
 //@requires: PatchGeneratorABC
 
-static PyObject *__Pyx_Generator_Next(PyObject *self);
-
-static PyObject *__Pyx_Generator_Next(PyObject *self) {
-    __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self;
-    PyObject *yf = gen->yieldfrom;
-    if (unlikely(__Pyx_Coroutine_CheckRunning(gen)))
-        return NULL;
-    if (yf) {
-        PyObject *ret;
-        // FIXME: does this really need an INCREF() ?
-        //Py_INCREF(yf);
-        // YieldFrom code ensures that yf is an iterator
-        gen->is_running = 1;
-        ret = Py_TYPE(yf)->tp_iternext(yf);
-        gen->is_running = 0;
-        //Py_DECREF(yf);
-        if (likely(ret)) {
-            return ret;
-        }
-        return __Pyx_Coroutine_FinishDelegation(gen);
-    }
-    return __Pyx_Coroutine_SendEx(gen, Py_None);
-}
-
 static PyTypeObject __pyx_GeneratorType_type = {
     PyVarObject_HEAD_INIT(0, 0)
     "generator",                        /*tp_name*/