refactored 'starred' status into a separate node to support syntax error...

refactored 'starred' status into a separate node to support syntax error handling outside of assignments

Cython/Compiler/ExprNodes.py

@@ -2866,6 +2866,49 @@ class AttributeNode(ExprNode):
 #
 #-------------------------------------------------------------------
 
+class StarredTargetNode(ExprNode):
+    #  A starred expression like "*a"
+    #
+    #  This is only allowed in sequence assignment targets such as
+    #
+    #      a, *b = (1,2,3,4)    =>     a = 1 ; b = [2,3,4]
+    #
+    #  and will be removed during type analysis (or generate an error
+    #  if it's found at unexpected places).
+    #
+    #  target          ExprNode
+
+    subexprs = ['target']
+    is_starred = 1
+    type = py_object_type
+
+    def __init__(self, pos, target):
+        self.pos = pos
+        self.target = target
+
+    def analyse_declarations(self, env):
+        error(self.pos, "can use starred expression only as assignment target")
+        self.target.analyse_declarations(env)
+
+    def analyse_types(self, env):
+        error(self.pos, "can use starred expression only as assignment target")
+        self.target.analyse_types(env)
+        self.type = self.target.type
+
+    def analyse_target_declaration(self, env):
+        self.target.analyse_target_declaration(env)
+
+    def analyse_target_types(self, env):
+        self.target.analyse_target_types(env)
+        self.type = self.target.type
+
+    def calculate_result_code(self):
+        return ""
+
+    def generate_result_code(self, code):
+        pass
+
+
 class SequenceNode(ExprNode):
     #  Base class for list and tuple constructor nodes.
     #  Contains common code for performing sequence unpacking.
@@ -2883,7 +2926,22 @@ class SequenceNode(ExprNode):
     def compile_time_value_list(self, denv):
         return [arg.compile_time_value(denv) for arg in self.args]
 
+    def replace_starred_target_node(self):
+        # replace a starred node in the targets by the contained expression
+        self.starred_assignment = False
+        args = []
+        for arg in self.args:
+            if arg.is_starred:
+                if self.starred_assignment:
+                    error(arg.pos, "more than 1 starred expression in assignment")
+                self.starred_assignment = True
+                arg = arg.target
+                arg.is_starred = True
+            args.append(arg)
+        self.args = args
+
     def analyse_target_declaration(self, env):
+        self.replace_starred_target_node()
         for arg in self.args:
             arg.analyse_target_declaration(env)
 
@@ -2899,7 +2957,6 @@ class SequenceNode(ExprNode):
         self.iterator = PyTempNode(self.pos, env)
         self.unpacked_items = []
         self.coerced_unpacked_items = []
-        self.starred_assignment = False
         for arg in self.args:
             arg.analyse_target_types(env)
             if arg.is_starred:
@@ -2908,14 +2965,13 @@ class SequenceNode(ExprNode):
                           "starred target must have Python object (list) type")
                 if arg.type is py_object_type:
                     arg.type = Builtin.list_type
-                self.starred_assignment = True
             unpacked_item = PyTempNode(self.pos, env)
             coerced_unpacked_item = unpacked_item.coerce_to(arg.type, env)
             self.unpacked_items.append(unpacked_item)
             self.coerced_unpacked_items.append(coerced_unpacked_item)
         self.type = py_object_type
         env.use_utility_code(unpacking_utility_code)
-    
+
     def generate_result_code(self, code):
         self.generate_operation_code(code)
     
@@ -2923,13 +2979,13 @@ class SequenceNode(ExprNode):
         if self.starred_assignment:
             self.generate_starred_assignment_code(rhs, code)
         else:
-            self.generate_normal_assignment_code(rhs, code)
+            self.generate_parallel_assignment_code(rhs, code)
 
         for item in self.unpacked_items:
             item.release(code)
         rhs.free_temps(code)
 
-    def generate_normal_assignment_code(self, rhs, code):
+    def generate_parallel_assignment_code(self, rhs, code):
         # Need to work around the fact that generate_evaluation_code
         # allocates the temps in a rather hacky way -- the assignment
         # is evaluated twice, within each if-block.

Cython/Compiler/Parsing.pxd

@@ -59,7 +59,6 @@ cpdef p_testlist(PyrexScanner s)
 #-------------------------------------------------------
 
 cpdef flatten_parallel_assignments(input, output)
-cpdef find_parallel_assignment_size(input)
 
 cpdef p_global_statement(PyrexScanner s)
 cpdef p_expression_or_assignment(PyrexScanner s)

Cython/Compiler/Parsing.py

@@ -142,13 +142,15 @@ def p_comparison(s):
     return n1
 
 def p_starred_expr(s):
+    pos = s.position()
     if s.sy == '*':
         starred = True
         s.next()
     else:
         starred = False
     expr = p_bit_expr(s)
-    expr.is_starred = starred
+    if starred:
+        expr = ExprNodes.StarredTargetNode(pos, expr)
     return expr
 
 def p_cascaded_cmp(s):
@@ -935,18 +937,26 @@ def flatten_parallel_assignments(input, output):
         if starred_targets:
             if starred_targets > 1:
                 error(lhs.pos, "more than 1 starred expression in assignment")
+                output.append([lhs,rhs])
+                continue
             elif lhs_size - starred_targets > rhs_size:
                 error(lhs.pos, "need more than %d value%s to unpack"
                       % (rhs_size, (rhs_size != 1) and 's' or ''))
+                output.append([lhs,rhs])
+                continue
             map_starred_assignment(lhs_targets, starred_assignments,
                                    lhs.args, rhs.args)
         else:
             if lhs_size > rhs_size:
                 error(lhs.pos, "need more than %d value%s to unpack"
                       % (rhs_size, (rhs_size != 1) and 's' or ''))
+                output.append([lhs,rhs])
+                continue
             elif lhs_size < rhs_size:
                 error(lhs.pos, "too many values to unpack (expected %d, got %d)"
                       % (lhs_size, rhs_size))
+                output.append([lhs,rhs])
+                continue
             else:
                 for targets, expr in zip(lhs_targets, lhs.args):
                     targets.append(expr)
@@ -988,8 +998,7 @@ def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args)
         targets.append(expr)
 
     # the starred target itself, must be assigned a (potentially empty) list
-    target = lhs_args[starred]
-    target.is_starred = False
+    target = lhs_args[starred].target # unpack starred node
     starred_rhs = rhs_args[starred:]
     if lhs_remaining:
         starred_rhs = starred_rhs[:-lhs_remaining]

tests/bugs/extended_unpacking_T235.pyx

 __doc__ = u"""
+>>> class FakeSeq(object):
+...     def __init__(self, length):
+...         self._values = range(1,length+1)
+...     def __getitem__(self, i):
+...         return self._values[i]
+
 >>> unpack([1,2])
 (1, 2)
 >>> unpack_list([1,2])
@@ -6,6 +12,8 @@ __doc__ = u"""
 >>> unpack_tuple((1,2))
 (1, 2)
 
+>>> unpack( FakeSeq(2) )
+(1, 2)
 >>> unpack('12')
 ('1', '2')
 
@@ -30,6 +38,8 @@ __doc__ = u"""
 
 >>> unpack_recursive((1,2,3,4))
 (1, [2, 3], 4)
+>>> unpack_recursive( FakeSeq(4) )
+(1, [2, 3], 4)
 >>> unpack_typed((1,2))
 ([1], 2)
 

tests/errors/extended_unpacking.pyx

+
+# invalid syntax (not handled by the parser)
+
+def syntax1():
+    a = b = c = d = e = f = g = h = i = 1 # prevent undefined names
+    list_of_sequences = [[1,2], [3,4]]
+
+    *a
+
+    *1
+
+    *"abc"
+
+    *a*b
+
+    [*a, *b]
+
+    (a, b, *c, d, e, f, *g, h, i)
+
+    for *a,*b in list_of_sequences:
+        pass
+
+
+_ERRORS = u"""
+ 8: 4: can use starred expression only as assignment target
+10: 4: can use starred expression only as assignment target
+12: 4: can use starred expression only as assignment target
+14: 4: can use starred expression only as assignment target
+16: 5: can use starred expression only as assignment target
+16: 9: can use starred expression only as assignment target
+18:11: can use starred expression only as assignment target
+18:24: can use starred expression only as assignment target
+20:11: more than 1 starred expression in assignment
+"""

tests/errors/extended_unpacking_parser.pyx

+
+# invalid syntax (as handled by the parser)
+
+def syntax():
+    *a, *b = 1,2,3,4,5
+
+# wrong size RHS (as handled by the parser)
+
+def length1():
+    a, b = [1,2,3]
+
+def length2():
+    a, b = [1]
+
+def length3():
+    a, b = []
+
+def length4():
+    a, *b = []
+
+def length5():
+    a, *b, c = []
+    a, *b, c = [1]
+
+def length_recursive():
+    *(a, b), c  = (1,2)
+
+
+_ERRORS = u"""
+ 5:4: more than 1 starred expression in assignment
+10:4: too many values to unpack (expected 2, got 3)
+13:4: need more than 1 value to unpack
+16:4: need more than 0 values to unpack
+19:4: need more than 0 values to unpack
+22:4: need more than 0 values to unpack
+23:4: need more than 1 value to unpack
+26:6: need more than 1 value to unpack
+"""