implement PEP 448 also for list/tuple literals

CHANGES.rst

@@ -15,8 +15,7 @@ Features added
 
 * Tracing is supported in ``nogil`` functions/sections and module init code.
 
-* PEP 448 (Additional Unpacking Generalizations) was partially implemented
-  for function calls, set and dict literals.
+* PEP 448 (Additional Unpacking Generalizations) was implemented.
 
 * When generators are used in a Cython module and the module imports the
   modules "inspect" and/or "asyncio", Cython enables interoperability by

Cython/Compiler/Optimize.py

@@ -3833,6 +3833,93 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations):
                 sequence_node.mult_factor = factor
         return sequence_node
 
+    def visit_MergedDictNode(self, node):
+        """Unpack **args in place if we can."""
+        self.visitchildren(node)
+        args = []
+        items = []
+
+        def add(arg):
+            if arg.is_dict_literal:
+                if items:
+                    items[0].key_value_pairs.extend(arg.key_value_pairs)
+                else:
+                    items.append(arg)
+            elif isinstance(arg, ExprNodes.MergedDictNode):
+                for child_arg in arg.keyword_args:
+                    add(child_arg)
+            else:
+                if items:
+                    args.append(items[0])
+                    del items[:]
+                args.append(arg)
+
+        for arg in node.keyword_args:
+            add(arg)
+        if items:
+            args.append(items[0])
+
+        if len(args) == 1:
+            arg = args[0]
+            if arg.is_dict_literal or isinstance(arg, ExprNodes.MergedDictNode):
+                return arg
+        node.keyword_args[:] = args
+        self._calculate_const(node)
+        return node
+
+    def visit_MergedSequenceNode(self, node):
+        """Unpack *args in place if we can."""
+        self.visitchildren(node)
+
+        is_set = node.type is Builtin.set_type
+        args = []
+        values = []
+
+        def add(arg):
+            if (is_set and arg.is_set_literal) or (arg.is_sequence_constructor and not arg.mult_factor):
+                if values:
+                    values[0].args.extend(arg.args)
+                else:
+                    values.append(arg)
+            elif isinstance(arg, ExprNodes.MergedSequenceNode):
+                for child_arg in arg.args:
+                    add(child_arg)
+            else:
+                if values:
+                    args.append(values[0])
+                    del values[:]
+                args.append(arg)
+
+        for arg in node.args:
+            add(arg)
+        if values:
+            args.append(values[0])
+
+        if len(args) == 1:
+            arg = args[0]
+            if ((is_set and arg.is_set_literal) or
+                    (arg.is_sequence_constructor and arg.type is node.type) or
+                    isinstance(arg, ExprNodes.MergedSequenceNode)):
+                return arg
+        node.args[:] = args
+        self._calculate_const(node)
+        return node
+
+    def visit_SequenceNode(self, node):
+        """Unpack *args in place if we can."""
+        self.visitchildren(node)
+        args = []
+        for arg in node.args:
+            if not arg.is_starred:
+                args.append(arg)
+            elif arg.target.is_sequence_constructor and not arg.target.mult_factor:
+                args.extend(arg.target.args)
+            else:
+                args.append(arg)
+        node.args[:] = args
+        self._calculate_const(node)
+        return node
+
     def visit_PrimaryCmpNode(self, node):
         # calculate constant partial results in the comparison cascade
         self.visitchildren(node, ['operand1'])

Cython/Compiler/Parsing.py

@@ -210,7 +210,7 @@ def p_starred_expr(s):
         starred = False
     expr = p_bit_expr(s)
     if starred:
-        expr = ExprNodes.StarredTargetNode(pos, expr)
+        expr = ExprNodes.StarredUnpackingNode(pos, expr)
     return expr
 
 def p_cascaded_cmp(s):
@@ -915,11 +915,13 @@ def p_string_literal(s, kind_override=None):
     s.next()
     return (kind, bytes_value, unicode_value)
 
-# list_display      ::=      "[" [listmaker] "]"
-# listmaker     ::=     expression ( comp_for | ( "," expression )* [","] )
+
+# since PEP 448:
+# list_display  ::=     "[" [listmaker] "]"
+# listmaker     ::=     (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 # comp_iter     ::=     comp_for | comp_if
-# comp_for     ::=     "for" expression_list "in" testlist [comp_iter]
-# comp_if     ::=     "if" test [comp_iter]
+# comp_for      ::=     "for" expression_list "in" testlist [comp_iter]
+# comp_if       ::=     "if" test [comp_iter]
 
 def p_list_maker(s):
     # s.sy == '['
@@ -927,24 +929,29 @@ def p_list_maker(s):
     s.next()
     if s.sy == ']':
         s.expect(']')
-        return ExprNodes.ListNode(pos, args = [])
-    expr = p_test(s)
+        return ExprNodes.ListNode(pos, args=[])
+
+    expr = p_test_or_starred_expr(s)
     if s.sy == 'for':
+        if expr.is_starred:
+            s.error("iterable unpacking cannot be used in comprehension")
         append = ExprNodes.ComprehensionAppendNode(pos, expr=expr)
         loop = p_comp_for(s, append)
         s.expect(']')
         return ExprNodes.ComprehensionNode(
-            pos, loop=loop, append=append, type = Builtin.list_type,
+            pos, loop=loop, append=append, type=Builtin.list_type,
             # list comprehensions leak their loop variable in Py2
-            has_local_scope = s.context.language_level >= 3)
+            has_local_scope=s.context.language_level >= 3)
+
+    # (merged) list literal
+    if s.sy == ',':
+        s.next()
+        exprs = p_test_or_starred_expr_list(s, expr)
     else:
-        if s.sy == ',':
-            s.next()
-            exprs = p_simple_expr_list(s, expr)
-        else:
-            exprs = [expr]
-        s.expect(']')
-        return ExprNodes.ListNode(pos, args = exprs)
+        exprs = [expr]
+    s.expect(']')
+    return ExprNodes.ListNode(pos, args=exprs)
+
 
 def p_comp_iter(s, body):
     if s.sy == 'for':
@@ -1000,7 +1007,9 @@ def p_dict_or_set_maker(s):
                 s.error("unexpected %sitem found in %s literal" % (
                     s.sy, 'set' if target_type == 1 else 'dict'))
             s.next()
-            item = p_test(s)
+            if s.sy == '*':
+                s.error("expected expression, found '*'")
+            item = p_starred_expr(s)
             parts.append(item)
             last_was_simple_item = False
         else:
@@ -1058,9 +1067,6 @@ def p_dict_or_set_maker(s):
         for part in parts:
             if isinstance(part, list):
                 set_items.extend(part)
-            elif part.is_set_literal or part.is_sequence_constructor:
-                # unpack *{1,2,3} and *[1,2,3] in place
-                set_items.extend(part.args)
             else:
                 if set_items:
                     items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
@@ -1070,7 +1076,7 @@ def p_dict_or_set_maker(s):
             items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
         if len(items) == 1 and items[0].is_set_literal:
             return items[0]
-        return ExprNodes.MergedSetNode(pos, args=items)
+        return ExprNodes.MergedSequenceNode(pos, args=items, type=Builtin.set_type)
     else:
         # (merged) dict literal
         items = []
@@ -1078,9 +1084,6 @@ def p_dict_or_set_maker(s):
         for part in parts:
             if isinstance(part, list):
                 dict_items.extend(part)
-            elif part.is_dict_literal:
-                # unpack **{...} in place
-                dict_items.extend(part.key_value_pairs)
             else:
                 if dict_items:
                     items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
@@ -1118,10 +1121,11 @@ def p_simple_expr_list(s, expr=None):
         s.next()
     return exprs
 
+
 def p_test_or_starred_expr_list(s, expr=None):
     exprs = expr is not None and [expr] or []
     while s.sy not in expr_terminators:
-        exprs.append( p_test_or_starred_expr(s) )
+        exprs.append(p_test_or_starred_expr(s))
         if s.sy != ',':
             break
         s.next()

tests/errors/extended_unpacking.pyx

@@ -16,6 +16,8 @@ def syntax1():
     [*a, *b]
 
     (a, b, *c, d, e, f, *g, h, i)
+    [a, b, *c, d, e, f, *g, h, i]
+    {a, b, *c, d, e, f, *g, h, i}
 
 
 def syntax2():
@@ -33,19 +35,15 @@ def types(l):
 
 _ERRORS = u"""
 # syntax1()
- 8: 4: can use starred expression only as assignment target
-10: 4: can use starred expression only as assignment target
-12: 4: can use starred expression only as assignment target
-14: 4: can use starred expression only as assignment target
-16: 5: can use starred expression only as assignment target
-16: 9: can use starred expression only as assignment target
-18:11: can use starred expression only as assignment target
-18:24: can use starred expression only as assignment target
+ 8: 4: starred expression is not allowed here
+10: 4: starred expression is not allowed here
+12: 4: starred expression is not allowed here
+14: 4: starred expression is not allowed here
 
 # syntax2()
-24:11: more than 1 starred expression in assignment
+26:11: more than 1 starred expression in assignment
 
 # types()
-30:15: Cannot coerce list to type 'int'
-31:10: starred target must have Python object (list) type
+32:15: Cannot coerce list to type 'int'
+33:10: starred target must have Python object (list) type
 """

tests/errors/nogil.pyx

@@ -42,6 +42,7 @@ cdef object m():
         (x, y)
         [x, y]
         {x: y}
+        {x, y}
         obj and x
         t(obj)
 #        f(42) # Cython handles this internally
@@ -129,35 +130,37 @@ _ERRORS = u"""
 42:9: Discarding owned Python object not allowed without gil
 43:8: Constructing Python list not allowed without gil
 43:8: Discarding owned Python object not allowed without gil
-44:8: Constructing Python dict not allowed without gil
-44:8: Discarding owned Python object not allowed without gil
-45:12: Discarding owned Python object not allowed without gil
-45:12: Truth-testing Python object not allowed without gil
-46:13: Python type test not allowed without gil
-48:10: Discarding owned Python object not allowed without gil
-48:10: Operation not allowed without gil
-49:8: Discarding owned Python object not allowed without gil
-49:8: Operation not allowed without gil
-50:10: Assignment of Python object not allowed without gil
-50:14: Assignment of Python object not allowed without gil
-51:9: Assignment of Python object not allowed without gil
-51:13: Assignment of Python object not allowed without gil
-51:16: Creating temporary Python reference not allowed without gil
-51:19: Creating temporary Python reference not allowed without gil
-52:11: Assignment of Python object not allowed without gil
-52:11: Indexing Python object not allowed without gil
-53:11: Accessing Python attribute not allowed without gil
+44:10: Constructing Python dict not allowed without gil
+44:10: Discarding owned Python object not allowed without gil
+45:10: Constructing Python set not allowed without gil
+45:10: Discarding owned Python object not allowed without gil
+46:12: Discarding owned Python object not allowed without gil
+46:12: Truth-testing Python object not allowed without gil
+47:13: Python type test not allowed without gil
+49:10: Discarding owned Python object not allowed without gil
+49:10: Operation not allowed without gil
+50:8: Discarding owned Python object not allowed without gil
+50:8: Operation not allowed without gil
+51:10: Assignment of Python object not allowed without gil
+51:14: Assignment of Python object not allowed without gil
+52:9: Assignment of Python object not allowed without gil
+52:13: Assignment of Python object not allowed without gil
+52:16: Creating temporary Python reference not allowed without gil
+52:19: Creating temporary Python reference not allowed without gil
 53:11: Assignment of Python object not allowed without gil
-54:8: Constructing Python tuple not allowed without gil
-54:8: Python print statement not allowed without gil
-55:8: Deleting Python object not allowed without gil
-56:8: Returning Python object not allowed without gil
-57:8: Raising exception not allowed without gil
-58:14: Truth-testing Python object not allowed without gil
-60:17: Truth-testing Python object not allowed without gil
-62:8: For-loop using object bounds or target not allowed without gil
-62:14: Coercion from Python not allowed without the GIL
-62:25: Coercion from Python not allowed without the GIL
-64:8: Try-except statement not allowed without gil
-85:8: For-loop using object bounds or target not allowed without gil
+53:11: Indexing Python object not allowed without gil
+54:11: Accessing Python attribute not allowed without gil
+54:11: Assignment of Python object not allowed without gil
+55:8: Constructing Python tuple not allowed without gil
+55:8: Python print statement not allowed without gil
+56:8: Deleting Python object not allowed without gil
+57:8: Returning Python object not allowed without gil
+58:8: Raising exception not allowed without gil
+59:14: Truth-testing Python object not allowed without gil
+61:17: Truth-testing Python object not allowed without gil
+63:8: For-loop using object bounds or target not allowed without gil
+63:14: Coercion from Python not allowed without the GIL
+63:25: Coercion from Python not allowed without the GIL
+65:8: Try-except statement not allowed without gil
+86:8: For-loop using object bounds or target not allowed without gil
 """

tests/errors/pep448_syntax_1.pyx

+# mode: error
+# tag: pep448
+
+def unpack_mix():
+    [*1, **1]
+
+
+_ERRORS = """
+5:9: Expected an identifier or literal
+"""

tests/errors/pep448_syntax_2.pyx

+# mode: error
+# tag: pep448
+
+def unpack_wrong_stars():
+    [**1]
+
+
+_ERRORS = """
+5:5: Expected an identifier or literal
+"""

tests/errors/pep448_syntax_3.pyx

+# mode: error
+# tag: pep448
+
+def unpack_mix_in_set():
+    {*1, **2}
+
+
+_ERRORS = """
+5:9: unexpected **item found in set literal
+"""

tests/run/pep448_extended_unpacking.pyx

@@ -23,10 +23,200 @@ class Map(object):
         return self.mapping[key]
 
 
+#### tuples
+
+
+@cython.test_fail_if_path_exists(
+    "//TupleNode//TupleNode",
+    "//MergedSequenceNode",
+)
+def unpack_tuple_literal():
+    """
+    >>> unpack_tuple_literal()
+    (1, 2, 4, 5)
+    """
+    return (*(1, 2, *(4, 5)),)
+
+
+def unpack_tuple_literal_mult():
+    """
+    >>> unpack_tuple_literal_mult()
+    (1, 2, 4, 5, 4, 5, 1, 2, 4, 5, 4, 5, 1, 2, 4, 5, 4, 5)
+    """
+    return (*((1, 2, *((4, 5) * 2)) * 3),)
+
+
+@cython.test_fail_if_path_exists(
+    "//TupleNode//TupleNode",
+    "//MergedSequenceNode",
+)
+def unpack_tuple_literal_empty():
+    """
+    >>> unpack_tuple_literal_empty()
+    ()
+    """
+    return (*(*(), *()), *(), *(*(*(),),))
+
+
+def unpack_tuple_simple(it):
+    """
+    >>> unpack_tuple_simple([])
+    ()
+    >>> unpack_tuple_simple(set())
+    ()
+    >>> unpack_tuple_simple(Iter())
+    ()
+
+    >>> unpack_tuple_simple([1])
+    (1,)
+
+    >>> unpack_tuple_simple([2, 1])
+    (2, 1)
+    >>> unpack_tuple_simple((2, 1))
+    (2, 1)
+    >>> sorted(unpack_tuple_simple(set([2, 1])))
+    [1, 2]
+    >>> unpack_tuple_simple(Iter([2, 1]))
+    (2, 1)
+    """
+    return (*it,)
+
+
+def unpack_tuple_from_iterable(it):
+    """
+    >>> unpack_tuple_from_iterable([1, 2, 3])
+    (1, 2, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 1, 1, 2, 3)
+    >>> unpack_tuple_from_iterable((1, 2, 3))
+    (1, 2, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 1, 1, 2, 3)
+    >>> sorted(unpack_tuple_from_iterable(set([1, 2, 3])))
+    [1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3]
+
+    >>> unpack_tuple_from_iterable([1, 2])
+    (1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 2, 2, 1, 1, 2)
+    >>> sorted(unpack_tuple_from_iterable(set([1, 2])))
+    [1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2]
+    >>> unpack_tuple_from_iterable(Iter([1, 2]))
+    (1, 2, 1, 2, 1, 2, 1)
+
+    >>> unpack_tuple_from_iterable([3])
+    (1, 2, 3, 1, 3, 3, 3, 2, 1, 3)
+    >>> unpack_tuple_from_iterable(set([3]))
+    (1, 2, 3, 1, 3, 3, 3, 2, 1, 3)
+    >>> unpack_tuple_from_iterable(Iter([3]))
+    (1, 2, 3, 1, 2, 1)
+
+    >>> unpack_tuple_from_iterable([])
+    (1, 2, 1, 2, 1)
+    >>> unpack_tuple_from_iterable(set([]))
+    (1, 2, 1, 2, 1)
+    >>> unpack_tuple_from_iterable([])
+    (1, 2, 1, 2, 1)
+    >>> unpack_tuple_from_iterable(Iter([1, 2, 3]))
+    (1, 2, 1, 2, 3, 1, 2, 1)
+    """
+    return (1, 2, *it, 1, *(*it, *it), *it, 2, 1, *it)
+
+
+#### lists
+
+
+@cython.test_fail_if_path_exists(
+    "//ListNode//ListNode",
+    "//MergedSequenceNode",
+)
+def unpack_list_literal():
+    """
+    >>> unpack_list_literal()
+    [1, 2, 4, 5]
+    """
+    return [*[1, 2, *[4, 5]]]
+
+
+def unpack_list_literal_mult():
+    """
+    >>> unpack_list_literal_mult()
+    [1, 2, 4, 5, 4, 5, 1, 2, 4, 5, 4, 5, 1, 2, 4, 5, 4, 5]
+    """
+    return [*([1, 2, *([4, 5] * 2)] * 3)]
+
+
+@cython.test_fail_if_path_exists(
+    "//ListNode//ListNode",
+    "//MergedSequenceNode",
+)
+def unpack_list_literal_empty():
+    """
+    >>> unpack_list_literal_empty()
+    []
+    """
+    return [*[*[], *[]], *[], *[*[*[]]]]
+
+
+def unpack_list_simple(it):
+    """
+    >>> unpack_list_simple([])
+    []
+    >>> unpack_list_simple(set())
+    []
+    >>> unpack_list_simple(Iter())
+    []
+
+    >>> unpack_list_simple([1])
+    [1]
+
+    >>> unpack_list_simple([2, 1])
+    [2, 1]
+    >>> unpack_list_simple((2, 1))
+    [2, 1]
+    >>> sorted(unpack_list_simple(set([2, 1])))
+    [1, 2]
+    >>> unpack_list_simple(Iter([2, 1]))
+    [2, 1]
+    """
+    return [*it]
+
+
+def unpack_list_from_iterable(it):
+    """
+    >>> unpack_list_from_iterable([1, 2, 3])
+    [1, 2, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 1, 1, 2, 3]
+    >>> unpack_list_from_iterable((1, 2, 3))
+    [1, 2, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 1, 1, 2, 3]
+    >>> sorted(unpack_list_from_iterable(set([1, 2, 3])))
+    [1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3]
+
+    >>> unpack_list_from_iterable([1, 2])
+    [1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 2, 2, 1, 1, 2]
+    >>> sorted(unpack_list_from_iterable(set([1, 2])))
+    [1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2]
+    >>> unpack_list_from_iterable(Iter([1, 2]))
+    [1, 2, 1, 2, 1, 2, 1]
+
+    >>> unpack_list_from_iterable([3])
+    [1, 2, 3, 1, 3, 3, 3, 2, 1, 3]
+    >>> unpack_list_from_iterable(set([3]))
+    [1, 2, 3, 1, 3, 3, 3, 2, 1, 3]
+    >>> unpack_list_from_iterable(Iter([3]))
+    [1, 2, 3, 1, 2, 1]
+
+    >>> unpack_list_from_iterable([])
+    [1, 2, 1, 2, 1]
+    >>> unpack_list_from_iterable(set([]))
+    [1, 2, 1, 2, 1]
+    >>> unpack_list_from_iterable([])
+    [1, 2, 1, 2, 1]
+    >>> unpack_list_from_iterable(Iter([1, 2, 3]))
+    [1, 2, 1, 2, 3, 1, 2, 1]
+    """
+    return [1, 2, *it, 1, *[*it, *it], *it, 2, 1, *it]
+
+
+###### sets
+
+
 @cython.test_fail_if_path_exists(
     "//SetNode//SetNode",
-    "//MergedSetNode//SetNode",
-    "//MergedSetNode//MergedSetNode",
+    "//MergedSequenceNode",
 )
 def unpack_set_literal():
     """
@@ -131,10 +321,12 @@ def unpack_set_from_iterable(it):
     return {1, 2, *it, 1, *{*it, *it}, *it, 2, 1, *it, *it}
 
 
+#### dicts
+
+
 @cython.test_fail_if_path_exists(
     "//DictNode//DictNode",
-    "//MergedDictNode//DictNode",
-    "//MergedDictNode//MergedDictNode",
+    "//MergedDictNode",
 )
 def unpack_dict_literal():
     """
@@ -145,6 +337,18 @@ def unpack_dict_literal():
     return {**{'a': 1, 'b': 2, **{'c': 4, 'd': 5}}}
 
 
+@cython.test_fail_if_path_exists(
+    "//DictNode//DictNode",
+    "//MergedDictNode",
+)
+def unpack_dict_literal_empty():
+    """
+    >>> unpack_dict_literal_empty()
+    {}
+    """
+    return {**{**{}, **{}}, **{}, **{**{**{}}}}
+
+
 def unpack_dict_simple(it):
     """
     >>> d = unpack_dict_simple({})