parser support for generator expressions so that we can give a better error...

parser support for generator expressions so that we can give a better error message than "Expected ')'"

Cython/Compiler/ExprNodes.py

@@ -3935,6 +3935,30 @@ class DictComprehensionAppendNode(ComprehensionAppendNode):
              code.error_goto_if(self.result(), self.pos)))
 
 
+class GeneratorExpressionNode(ExprNode):
+    # A generator expression, e.g.  (i for i in range(10))
+    #
+    # Result is a generator.
+    #
+    # loop   ForStatNode   the for-loop, containing a YieldExprNode
+    subexprs = []
+    child_attrs = ["loop"]
+
+    type = py_object_type
+
+    def analyse_declarations(self, env):
+        self.loop.analyse_declarations(env)
+
+    def analyse_types(self, env):
+        self.loop.analyse_expressions(env)
+
+    def may_be_none(self):
+        return False
+
+    def annotate(self, code):
+        self.loop.annotate(code)
+
+
 class SetNode(ExprNode):
     #  Set constructor.
 
@@ -4325,6 +4349,7 @@ class YieldExprNode(ExprNode):
             self.arg.analyse_types(env)
             if not self.arg.type.is_pyobject:
                 self.arg = self.arg.coerce_to_pyobject(env)
+        error(self.pos, "Generators are not supported")
 
     def generate_result_code(self, code):
         self.label_name = code.new_label('resume_from_yield')

Cython/Compiler/ParseTreeTransforms.py

@@ -1030,6 +1030,11 @@ property NAME:
         node.analyse_declarations(self.env_stack[-1])
         return node
 
+    def visit_GeneratorExpressionNode(self, node):
+        self.visitchildren(node)
+        node.analyse_declarations(self.env_stack[-1])
+        return node
+
     # Some nodes are no longer needed after declaration
     # analysis and can be dropped. The analysis was performed
     # on these nodes in a seperate recursive process from the

Cython/Compiler/Parsing.py

@@ -421,7 +421,10 @@ def p_call(s, function):
             break
         s.next()
 
-    if s.sy == '**':
+    if s.sy == 'for':
+        if len(positional_args) == 1 and not star_arg:
+            positional_args = [ p_genexp(s, positional_args[0]) ]
+    elif s.sy == '**':
         s.next()
         starstar_arg = p_simple_expr(s)
         if s.sy == ',':
@@ -547,7 +550,7 @@ def make_slice_node(pos, start, stop = None, step = None):
     return ExprNodes.SliceNode(pos,
         start = start, stop = stop, step = step)
 
-#atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
+#atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
 
 def p_atom(s):
     pos = s.position()
@@ -559,7 +562,7 @@ def p_atom(s):
         elif s.sy == 'yield':
             result = p_yield_expression(s)
         else:
-            result = p_expr(s)
+            result = p_testlist_comp(s)
         s.expect(')')
         return result
     elif sy == '[':
@@ -922,7 +925,28 @@ def p_testlist(s):
         return ExprNodes.TupleNode(pos, args = exprs)
     else:
         return expr
-        
+
+# testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+
+def p_testlist_comp(s):
+    pos = s.position()
+    expr = p_simple_expr(s)
+    if s.sy == ',':
+        exprs = [expr]
+        while s.sy == ',':
+            s.next()
+            exprs.append(p_test(s))
+        return ExprNodes.TupleNode(pos, args = exprs)
+    elif s.sy == 'for':
+        return p_genexp(s, expr)
+    else:
+        return expr
+
+def p_genexp(s, expr):
+    # s.sy == 'for'
+    loop = p_comp_for(s, ExprNodes.YieldExprNode(expr.pos, arg=expr))
+    return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop)
+
 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')
 
 #-------------------------------------------------------