cleanup in parser to make it a better match with the CPython grammar, fix some minor grammar issues

Cython/Compiler/Parsing.pxd

@@ -6,8 +6,8 @@ from Cython.Compiler.Scanning cimport PyrexScanner
 cpdef p_ident(PyrexScanner s, message =*)
 cpdef p_ident_list(PyrexScanner s)
 
+cpdef p_binop_operator(PyrexScanner s)
 cpdef p_binop_expr(PyrexScanner s, ops, p_sub_expr)
-cpdef p_simple_expr(PyrexScanner s)
 cpdef p_lambdef(PyrexScanner s, bint allow_conditional=*)
 cpdef p_lambdef_nocond(PyrexScanner s)
 cpdef p_test(PyrexScanner s)
@@ -17,9 +17,10 @@ cpdef p_rassoc_binop_expr(PyrexScanner s, ops, p_subexpr)
 cpdef p_and_test(PyrexScanner s)
 cpdef p_not_test(PyrexScanner s)
 cpdef p_comparison(PyrexScanner s)
+cpdef p_test_or_starred_expr(PyrexScanner s)
+cpdef p_starred_expr(PyrexScanner s)
 cpdef p_cascaded_cmp(PyrexScanner s)
 cpdef p_cmp_op(PyrexScanner s)
-cpdef p_starred_expr(PyrexScanner s)
 cpdef p_bit_expr(PyrexScanner s)
 cpdef p_xor_expr(PyrexScanner s)
 cpdef p_and_expr(PyrexScanner s)
@@ -30,6 +31,7 @@ cpdef p_factor(PyrexScanner s)
 cpdef p_typecast(PyrexScanner s)
 cpdef p_sizeof(PyrexScanner s)
 cpdef p_yield_expression(PyrexScanner s)
+cpdef p_yield_statement(PyrexScanner s)
 cpdef p_power(PyrexScanner s)
 cpdef p_new_expr(PyrexScanner s)
 cpdef p_trailer(PyrexScanner s, node1)
@@ -52,9 +54,12 @@ cpdef p_comp_for(PyrexScanner s, body)
 cpdef p_comp_if(PyrexScanner s, body)
 cpdef p_dict_or_set_maker(PyrexScanner s)
 cpdef p_backquote_expr(PyrexScanner s)
-cpdef p_simple_expr_list(PyrexScanner s)
-cpdef p_expr(PyrexScanner s)
+cpdef p_simple_expr_list(PyrexScanner s, expr=*)
+cpdef p_test_or_starred_expr_list(s, expr=*)
 cpdef p_testlist(PyrexScanner s)
+cpdef p_testlist_star_expr(PyrexScanner s)
+cpdef p_testlist_comp(PyrexScanner s)
+cpdef p_genexp(PyrexScanner s, expr)
 
 #-------------------------------------------------------
 #
@@ -83,12 +88,12 @@ cpdef p_if_clause(PyrexScanner s)
 cpdef p_else_clause(PyrexScanner s)
 cpdef p_while_statement(PyrexScanner s)
 cpdef p_for_statement(PyrexScanner s)
-cpdef p_for_bounds(PyrexScanner s)
+cpdef p_for_bounds(PyrexScanner s, bint allow_testlist = *)
 cpdef p_for_from_relation(PyrexScanner s)
 cpdef p_for_from_step(PyrexScanner s)
 cpdef p_target(PyrexScanner s, terminator)
 cpdef p_for_target(PyrexScanner s)
-cpdef p_for_iterator(PyrexScanner s)
+cpdef p_for_iterator(PyrexScanner s, bint allow_testlist = *)
 cpdef p_try_statement(PyrexScanner s)
 cpdef p_except_clause(PyrexScanner s)
 cpdef p_include_statement(PyrexScanner s, ctx)

Cython/Compiler/Parsing.py

@@ -184,6 +184,12 @@ def p_comparison(s):
             n1.cascade = p_cascaded_cmp(s)
     return n1
 
+def p_test_or_starred_expr(s):
+    if s.sy == '*':
+        return p_starred_expr(s)
+    else:
+        return p_test(s)
+
 def p_starred_expr(s):
     pos = s.position()
     if s.sy == '*':
@@ -326,7 +332,7 @@ def p_yield_expression(s):
     pos = s.position()
     s.next()
     if s.sy != ')' and s.sy not in statement_terminators:
-        arg = p_expr(s)
+        arg = p_testlist(s)
     else:
         arg = None
     return ExprNodes.YieldExprNode(pos, arg=arg)
@@ -776,10 +782,11 @@ def p_list_maker(s):
         return ExprNodes.ComprehensionNode(
             pos, loop=loop, append=append, target=target)
     else:
-        exprs = [expr]
         if s.sy == ',':
             s.next()
-            exprs += p_simple_expr_list(s)
+            exprs = p_simple_expr_list(s, expr)
+        else:
+            exprs = [expr]
         s.expect(']')
         return ExprNodes.ListNode(pos, args = exprs)
         
@@ -796,7 +803,7 @@ def p_comp_for(s, body):
     # s.sy == 'for'
     pos = s.position()
     s.next()
-    kw = p_for_bounds(s)
+    kw = p_for_bounds(s, allow_testlist=False)
     kw['else_clause'] = None
     kw['body'] = p_comp_iter(s, body)
     return Nodes.ForStatNode(pos, **kw)
@@ -874,44 +881,62 @@ def p_dict_or_set_maker(s):
         s.expect('}')
     return ExprNodes.DictNode(pos, key_value_pairs = [])
 
+# NOTE: no longer in Py3 :)
 def p_backquote_expr(s):
     # s.sy == '`'
     pos = s.position()
     s.next()
-    arg = p_expr(s)
+    args = [p_test(s)]
+    while s.sy == ',':
+        s.next()
+        args.append(p_test(s))
     s.expect('`')
+    if len(args) == 1:
+        arg = args[0]
+    else:
+        arg = ExprNodes.TupleNode(pos, args = args)
     return ExprNodes.BackquoteNode(pos, arg = arg)
 
-def p_simple_expr_list(s):
-    exprs = []
+def p_simple_expr_list(s, expr=None):
+    exprs = expr is not None and [expr] or []
     while s.sy not in expr_terminators:
-        expr = p_test(s)
-        exprs.append(expr)
+        exprs.append( p_test(s) )
         if s.sy != ',':
             break
         s.next()
     return exprs
 
-def p_expr(s):
+def p_test_or_starred_expr_list(s, expr=None):
+    exprs = expr is not None and [expr] or []
+    while s.sy not in expr_terminators:
+        exprs.append( p_test_or_starred_expr(s) )
+        if s.sy != ',':
+            break
+        s.next()
+    return exprs
+    
+
+#testlist: test (',' test)* [',']
+
+def p_testlist(s):
     pos = s.position()
     expr = p_test(s)
     if s.sy == ',':
         s.next()
-        exprs = [expr] + p_simple_expr_list(s)
+        exprs = p_simple_expr_list(s, expr)
         return ExprNodes.TupleNode(pos, args = exprs)
     else:
         return expr
 
+# testlist_star_expr: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 
-#testlist: test (',' test)* [',']
-# differs from p_expr only in the fact that it cannot contain conditional expressions
-
-def p_testlist(s):
+def p_testlist_star_expr(s):
     pos = s.position()
-    expr = p_test(s)
+    expr = p_test_or_starred_expr(s)
     if s.sy == ',':
         s.next()
-        return p_testlist_tuple(s, pos, expr)
+        exprs = p_test_or_starred_expr_list(s, expr)
+        return ExprNodes.TupleNode(pos, args = exprs)
     else:
         return expr
 
@@ -919,24 +944,16 @@ def p_testlist(s):
 
 def p_testlist_comp(s):
     pos = s.position()
-    expr = p_test(s)
+    expr = p_test_or_starred_expr(s)
     if s.sy == ',':
         s.next()
-        return p_testlist_tuple(s, pos, expr)
+        exprs = p_test_or_starred_expr_list(s, expr)
+        return ExprNodes.TupleNode(pos, args = exprs)
     elif s.sy == 'for':
         return p_genexp(s, expr)
     else:
         return expr
 
-def p_testlist_tuple(s, pos, expr):
-    exprs = [expr]
-    while s.sy not in expr_terminators:
-        exprs.append(p_test(s))
-        if s.sy != ',':
-            break
-        s.next()
-    return ExprNodes.TupleNode(pos, args = exprs)
-
 def p_genexp(s, expr):
     # s.sy == 'for'
     loop = p_comp_for(s, ExprNodes.YieldExprNode(expr.pos, arg=expr))
@@ -958,10 +975,14 @@ def p_global_statement(s):
     return Nodes.GlobalNode(pos, names = names)
 
 def p_expression_or_assignment(s):
-    expr_list = [p_expr(s)]
+    expr_list = [p_testlist_star_expr(s)]
     while s.sy == '=':
         s.next()
-        expr_list.append(p_expr(s))
+        if s.sy == 'yield':
+            expr = p_yield_expression(s)
+        else:
+            expr = p_testlist_star_expr(s)
+        expr_list.append(expr)
     if len(expr_list) == 1:
         if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):
             lhs = expr_list[0]
@@ -969,7 +990,10 @@ def p_expression_or_assignment(s):
                 error(lhs.pos, "Illegal operand for inplace operation.")
             operator = s.sy[:-1]
             s.next()
-            rhs = p_expr(s)
+            if s.sy == 'yield':
+                rhs = p_yield_expression(s)
+            else:
+                rhs = p_testlist(s)
             return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
         expr = expr_list[0]
         if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):
@@ -1059,7 +1083,7 @@ def p_return_statement(s):
     pos = s.position()
     s.next()
     if s.sy not in statement_terminators:
-        value = p_expr(s)
+        value = p_testlist(s)
     else:
         value = None
     return Nodes.ReturnStatNode(pos, value = value)
@@ -1273,16 +1297,16 @@ def p_for_statement(s):
     # s.sy == 'for'
     pos = s.position()
     s.next()
-    kw = p_for_bounds(s)
+    kw = p_for_bounds(s, allow_testlist=True)
     kw['body'] = p_suite(s)
     kw['else_clause'] = p_else_clause(s)
     return Nodes.ForStatNode(pos, **kw)
             
-def p_for_bounds(s):
+def p_for_bounds(s, allow_testlist=True):
     target = p_for_target(s)
     if s.sy == 'in':
         s.next()
-        iterator = p_for_iterator(s)
+        iterator = p_for_iterator(s, allow_testlist)
         return { 'target': target, 'iterator': iterator }
     else:
         if s.sy == 'from':
@@ -1353,9 +1377,12 @@ def p_target(s, terminator):
 def p_for_target(s):
     return p_target(s, 'in')
 
-def p_for_iterator(s):
+def p_for_iterator(s, allow_testlist=True):
     pos = s.position()
-    expr = p_testlist(s)
+    if allow_testlist:
+        expr = p_testlist(s)
+    else:
+        expr = p_or_test(s)
     return ExprNodes.IteratorNode(pos, sequence = expr)
 
 def p_try_statement(s):
@@ -1462,7 +1489,7 @@ def p_with_statement(s):
         else:
             error(pos, "Syntax error in template function declaration")
     else:
-        manager = p_expr(s)
+        manager = p_test(s)
         target = None
         if s.sy == 'IDENT' and s.systring == 'as':
             s.next()
@@ -1525,7 +1552,7 @@ def p_simple_statement_list(s, ctx, first_statement = 0):
 def p_compile_time_expr(s):
     old = s.compile_time_expr
     s.compile_time_expr = 1
-    expr = p_expr(s)
+    expr = p_testlist(s)
     s.compile_time_expr = old
     return expr
 
@@ -1996,7 +2023,7 @@ def p_c_array_declarator(s, base):
     pos = s.position()
     s.next() # '['
     if s.sy != ']':
-        dim = p_expr(s)
+        dim = p_testlist(s)
     else:
         dim = None
     s.expect(']')