Commit a8683244 authored by Stefan Behnel's avatar Stefan Behnel

implement PEP 448 also for list/tuple literals

parent 0e6a7b7d
......@@ -15,8 +15,7 @@ Features added
* Tracing is supported in ``nogil`` functions/sections and module init code.
* PEP 448 (Additional Unpacking Generalizations) was partially implemented
for function calls, set and dict literals.
* PEP 448 (Additional Unpacking Generalizations) was implemented.
* When generators are used in a Cython module and the module imports the
modules "inspect" and/or "asyncio", Cython enables interoperability by
......
This diff is collapsed.
......@@ -3833,6 +3833,93 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations):
sequence_node.mult_factor = factor
return sequence_node
def visit_MergedDictNode(self, node):
"""Unpack **args in place if we can."""
self.visitchildren(node)
args = []
items = []
def add(arg):
if arg.is_dict_literal:
if items:
items[0].key_value_pairs.extend(arg.key_value_pairs)
else:
items.append(arg)
elif isinstance(arg, ExprNodes.MergedDictNode):
for child_arg in arg.keyword_args:
add(child_arg)
else:
if items:
args.append(items[0])
del items[:]
args.append(arg)
for arg in node.keyword_args:
add(arg)
if items:
args.append(items[0])
if len(args) == 1:
arg = args[0]
if arg.is_dict_literal or isinstance(arg, ExprNodes.MergedDictNode):
return arg
node.keyword_args[:] = args
self._calculate_const(node)
return node
def visit_MergedSequenceNode(self, node):
"""Unpack *args in place if we can."""
self.visitchildren(node)
is_set = node.type is Builtin.set_type
args = []
values = []
def add(arg):
if (is_set and arg.is_set_literal) or (arg.is_sequence_constructor and not arg.mult_factor):
if values:
values[0].args.extend(arg.args)
else:
values.append(arg)
elif isinstance(arg, ExprNodes.MergedSequenceNode):
for child_arg in arg.args:
add(child_arg)
else:
if values:
args.append(values[0])
del values[:]
args.append(arg)
for arg in node.args:
add(arg)
if values:
args.append(values[0])
if len(args) == 1:
arg = args[0]
if ((is_set and arg.is_set_literal) or
(arg.is_sequence_constructor and arg.type is node.type) or
isinstance(arg, ExprNodes.MergedSequenceNode)):
return arg
node.args[:] = args
self._calculate_const(node)
return node
def visit_SequenceNode(self, node):
"""Unpack *args in place if we can."""
self.visitchildren(node)
args = []
for arg in node.args:
if not arg.is_starred:
args.append(arg)
elif arg.target.is_sequence_constructor and not arg.target.mult_factor:
args.extend(arg.target.args)
else:
args.append(arg)
node.args[:] = args
self._calculate_const(node)
return node
def visit_PrimaryCmpNode(self, node):
# calculate constant partial results in the comparison cascade
self.visitchildren(node, ['operand1'])
......
......@@ -210,7 +210,7 @@ def p_starred_expr(s):
starred = False
expr = p_bit_expr(s)
if starred:
expr = ExprNodes.StarredTargetNode(pos, expr)
expr = ExprNodes.StarredUnpackingNode(pos, expr)
return expr
def p_cascaded_cmp(s):
......@@ -915,11 +915,13 @@ def p_string_literal(s, kind_override=None):
s.next()
return (kind, bytes_value, unicode_value)
# list_display ::= "[" [listmaker] "]"
# listmaker ::= expression ( comp_for | ( "," expression )* [","] )
# since PEP 448:
# list_display ::= "[" [listmaker] "]"
# listmaker ::= (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
# comp_iter ::= comp_for | comp_if
# comp_for ::= "for" expression_list "in" testlist [comp_iter]
# comp_if ::= "if" test [comp_iter]
# comp_for ::= "for" expression_list "in" testlist [comp_iter]
# comp_if ::= "if" test [comp_iter]
def p_list_maker(s):
# s.sy == '['
......@@ -927,24 +929,29 @@ def p_list_maker(s):
s.next()
if s.sy == ']':
s.expect(']')
return ExprNodes.ListNode(pos, args = [])
expr = p_test(s)
return ExprNodes.ListNode(pos, args=[])
expr = p_test_or_starred_expr(s)
if s.sy == 'for':
if expr.is_starred:
s.error("iterable unpacking cannot be used in comprehension")
append = ExprNodes.ComprehensionAppendNode(pos, expr=expr)
loop = p_comp_for(s, append)
s.expect(']')
return ExprNodes.ComprehensionNode(
pos, loop=loop, append=append, type = Builtin.list_type,
pos, loop=loop, append=append, type=Builtin.list_type,
# list comprehensions leak their loop variable in Py2
has_local_scope = s.context.language_level >= 3)
has_local_scope=s.context.language_level >= 3)
# (merged) list literal
if s.sy == ',':
s.next()
exprs = p_test_or_starred_expr_list(s, expr)
else:
if s.sy == ',':
s.next()
exprs = p_simple_expr_list(s, expr)
else:
exprs = [expr]
s.expect(']')
return ExprNodes.ListNode(pos, args = exprs)
exprs = [expr]
s.expect(']')
return ExprNodes.ListNode(pos, args=exprs)
def p_comp_iter(s, body):
if s.sy == 'for':
......@@ -1000,7 +1007,9 @@ def p_dict_or_set_maker(s):
s.error("unexpected %sitem found in %s literal" % (
s.sy, 'set' if target_type == 1 else 'dict'))
s.next()
item = p_test(s)
if s.sy == '*':
s.error("expected expression, found '*'")
item = p_starred_expr(s)
parts.append(item)
last_was_simple_item = False
else:
......@@ -1058,9 +1067,6 @@ def p_dict_or_set_maker(s):
for part in parts:
if isinstance(part, list):
set_items.extend(part)
elif part.is_set_literal or part.is_sequence_constructor:
# unpack *{1,2,3} and *[1,2,3] in place
set_items.extend(part.args)
else:
if set_items:
items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
......@@ -1070,7 +1076,7 @@ def p_dict_or_set_maker(s):
items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
if len(items) == 1 and items[0].is_set_literal:
return items[0]
return ExprNodes.MergedSetNode(pos, args=items)
return ExprNodes.MergedSequenceNode(pos, args=items, type=Builtin.set_type)
else:
# (merged) dict literal
items = []
......@@ -1078,9 +1084,6 @@ def p_dict_or_set_maker(s):
for part in parts:
if isinstance(part, list):
dict_items.extend(part)
elif part.is_dict_literal:
# unpack **{...} in place
dict_items.extend(part.key_value_pairs)
else:
if dict_items:
items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
......@@ -1118,10 +1121,11 @@ def p_simple_expr_list(s, expr=None):
s.next()
return exprs
def p_test_or_starred_expr_list(s, expr=None):
exprs = expr is not None and [expr] or []
while s.sy not in expr_terminators:
exprs.append( p_test_or_starred_expr(s) )
exprs.append(p_test_or_starred_expr(s))
if s.sy != ',':
break
s.next()
......
......@@ -16,6 +16,8 @@ def syntax1():
[*a, *b]
(a, b, *c, d, e, f, *g, h, i)
[a, b, *c, d, e, f, *g, h, i]
{a, b, *c, d, e, f, *g, h, i}
def syntax2():
......@@ -33,19 +35,15 @@ def types(l):
_ERRORS = u"""
# syntax1()
8: 4: can use starred expression only as assignment target
10: 4: can use starred expression only as assignment target
12: 4: can use starred expression only as assignment target
14: 4: can use starred expression only as assignment target
16: 5: can use starred expression only as assignment target
16: 9: can use starred expression only as assignment target
18:11: can use starred expression only as assignment target
18:24: can use starred expression only as assignment target
8: 4: starred expression is not allowed here
10: 4: starred expression is not allowed here
12: 4: starred expression is not allowed here
14: 4: starred expression is not allowed here
# syntax2()
24:11: more than 1 starred expression in assignment
26:11: more than 1 starred expression in assignment
# types()
30:15: Cannot coerce list to type 'int'
31:10: starred target must have Python object (list) type
32:15: Cannot coerce list to type 'int'
33:10: starred target must have Python object (list) type
"""
......@@ -42,6 +42,7 @@ cdef object m():
(x, y)
[x, y]
{x: y}
{x, y}
obj and x
t(obj)
# f(42) # Cython handles this internally
......@@ -129,35 +130,37 @@ _ERRORS = u"""
42:9: Discarding owned Python object not allowed without gil
43:8: Constructing Python list not allowed without gil
43:8: Discarding owned Python object not allowed without gil
44:8: Constructing Python dict not allowed without gil
44:8: Discarding owned Python object not allowed without gil
45:12: Discarding owned Python object not allowed without gil
45:12: Truth-testing Python object not allowed without gil
46:13: Python type test not allowed without gil
48:10: Discarding owned Python object not allowed without gil
48:10: Operation not allowed without gil
49:8: Discarding owned Python object not allowed without gil
49:8: Operation not allowed without gil
50:10: Assignment of Python object not allowed without gil
50:14: Assignment of Python object not allowed without gil
51:9: Assignment of Python object not allowed without gil
51:13: Assignment of Python object not allowed without gil
51:16: Creating temporary Python reference not allowed without gil
51:19: Creating temporary Python reference not allowed without gil
52:11: Assignment of Python object not allowed without gil
52:11: Indexing Python object not allowed without gil
53:11: Accessing Python attribute not allowed without gil
44:10: Constructing Python dict not allowed without gil
44:10: Discarding owned Python object not allowed without gil
45:10: Constructing Python set not allowed without gil
45:10: Discarding owned Python object not allowed without gil
46:12: Discarding owned Python object not allowed without gil
46:12: Truth-testing Python object not allowed without gil
47:13: Python type test not allowed without gil
49:10: Discarding owned Python object not allowed without gil
49:10: Operation not allowed without gil
50:8: Discarding owned Python object not allowed without gil
50:8: Operation not allowed without gil
51:10: Assignment of Python object not allowed without gil
51:14: Assignment of Python object not allowed without gil
52:9: Assignment of Python object not allowed without gil
52:13: Assignment of Python object not allowed without gil
52:16: Creating temporary Python reference not allowed without gil
52:19: Creating temporary Python reference not allowed without gil
53:11: Assignment of Python object not allowed without gil
54:8: Constructing Python tuple not allowed without gil
54:8: Python print statement not allowed without gil
55:8: Deleting Python object not allowed without gil
56:8: Returning Python object not allowed without gil
57:8: Raising exception not allowed without gil
58:14: Truth-testing Python object not allowed without gil
60:17: Truth-testing Python object not allowed without gil
62:8: For-loop using object bounds or target not allowed without gil
62:14: Coercion from Python not allowed without the GIL
62:25: Coercion from Python not allowed without the GIL
64:8: Try-except statement not allowed without gil
85:8: For-loop using object bounds or target not allowed without gil
53:11: Indexing Python object not allowed without gil
54:11: Accessing Python attribute not allowed without gil
54:11: Assignment of Python object not allowed without gil
55:8: Constructing Python tuple not allowed without gil
55:8: Python print statement not allowed without gil
56:8: Deleting Python object not allowed without gil
57:8: Returning Python object not allowed without gil
58:8: Raising exception not allowed without gil
59:14: Truth-testing Python object not allowed without gil
61:17: Truth-testing Python object not allowed without gil
63:8: For-loop using object bounds or target not allowed without gil
63:14: Coercion from Python not allowed without the GIL
63:25: Coercion from Python not allowed without the GIL
65:8: Try-except statement not allowed without gil
86:8: For-loop using object bounds or target not allowed without gil
"""
# mode: error
# tag: pep448
def unpack_mix():
[*1, **1]
_ERRORS = """
5:9: Expected an identifier or literal
"""
# mode: error
# tag: pep448
def unpack_wrong_stars():
[**1]
_ERRORS = """
5:5: Expected an identifier or literal
"""
# mode: error
# tag: pep448
def unpack_mix_in_set():
{*1, **2}
_ERRORS = """
5:9: unexpected **item found in set literal
"""
......@@ -23,10 +23,200 @@ class Map(object):
return self.mapping[key]
#### tuples
@cython.test_fail_if_path_exists(
"//TupleNode//TupleNode",
"//MergedSequenceNode",
)
def unpack_tuple_literal():
"""
>>> unpack_tuple_literal()
(1, 2, 4, 5)
"""
return (*(1, 2, *(4, 5)),)
def unpack_tuple_literal_mult():
"""
>>> unpack_tuple_literal_mult()
(1, 2, 4, 5, 4, 5, 1, 2, 4, 5, 4, 5, 1, 2, 4, 5, 4, 5)
"""
return (*((1, 2, *((4, 5) * 2)) * 3),)
@cython.test_fail_if_path_exists(
"//TupleNode//TupleNode",
"//MergedSequenceNode",
)
def unpack_tuple_literal_empty():
"""
>>> unpack_tuple_literal_empty()
()
"""
return (*(*(), *()), *(), *(*(*(),),))
def unpack_tuple_simple(it):
"""
>>> unpack_tuple_simple([])
()
>>> unpack_tuple_simple(set())
()
>>> unpack_tuple_simple(Iter())
()
>>> unpack_tuple_simple([1])
(1,)
>>> unpack_tuple_simple([2, 1])
(2, 1)
>>> unpack_tuple_simple((2, 1))
(2, 1)
>>> sorted(unpack_tuple_simple(set([2, 1])))
[1, 2]
>>> unpack_tuple_simple(Iter([2, 1]))
(2, 1)
"""
return (*it,)
def unpack_tuple_from_iterable(it):
"""
>>> unpack_tuple_from_iterable([1, 2, 3])
(1, 2, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 1, 1, 2, 3)
>>> unpack_tuple_from_iterable((1, 2, 3))
(1, 2, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 1, 1, 2, 3)
>>> sorted(unpack_tuple_from_iterable(set([1, 2, 3])))
[1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3]
>>> unpack_tuple_from_iterable([1, 2])
(1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 2, 2, 1, 1, 2)
>>> sorted(unpack_tuple_from_iterable(set([1, 2])))
[1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2]
>>> unpack_tuple_from_iterable(Iter([1, 2]))
(1, 2, 1, 2, 1, 2, 1)
>>> unpack_tuple_from_iterable([3])
(1, 2, 3, 1, 3, 3, 3, 2, 1, 3)
>>> unpack_tuple_from_iterable(set([3]))
(1, 2, 3, 1, 3, 3, 3, 2, 1, 3)
>>> unpack_tuple_from_iterable(Iter([3]))
(1, 2, 3, 1, 2, 1)
>>> unpack_tuple_from_iterable([])
(1, 2, 1, 2, 1)
>>> unpack_tuple_from_iterable(set([]))
(1, 2, 1, 2, 1)
>>> unpack_tuple_from_iterable([])
(1, 2, 1, 2, 1)
>>> unpack_tuple_from_iterable(Iter([1, 2, 3]))
(1, 2, 1, 2, 3, 1, 2, 1)
"""
return (1, 2, *it, 1, *(*it, *it), *it, 2, 1, *it)
#### lists
@cython.test_fail_if_path_exists(
"//ListNode//ListNode",
"//MergedSequenceNode",
)
def unpack_list_literal():
"""
>>> unpack_list_literal()
[1, 2, 4, 5]
"""
return [*[1, 2, *[4, 5]]]
def unpack_list_literal_mult():
"""
>>> unpack_list_literal_mult()
[1, 2, 4, 5, 4, 5, 1, 2, 4, 5, 4, 5, 1, 2, 4, 5, 4, 5]
"""
return [*([1, 2, *([4, 5] * 2)] * 3)]
@cython.test_fail_if_path_exists(
"//ListNode//ListNode",
"//MergedSequenceNode",
)
def unpack_list_literal_empty():
"""
>>> unpack_list_literal_empty()
[]
"""
return [*[*[], *[]], *[], *[*[*[]]]]
def unpack_list_simple(it):
"""
>>> unpack_list_simple([])
[]
>>> unpack_list_simple(set())
[]
>>> unpack_list_simple(Iter())
[]
>>> unpack_list_simple([1])
[1]
>>> unpack_list_simple([2, 1])
[2, 1]
>>> unpack_list_simple((2, 1))
[2, 1]
>>> sorted(unpack_list_simple(set([2, 1])))
[1, 2]
>>> unpack_list_simple(Iter([2, 1]))
[2, 1]
"""
return [*it]
def unpack_list_from_iterable(it):
"""
>>> unpack_list_from_iterable([1, 2, 3])
[1, 2, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 1, 1, 2, 3]
>>> unpack_list_from_iterable((1, 2, 3))
[1, 2, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 1, 1, 2, 3]
>>> sorted(unpack_list_from_iterable(set([1, 2, 3])))
[1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3]
>>> unpack_list_from_iterable([1, 2])
[1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 2, 2, 1, 1, 2]
>>> sorted(unpack_list_from_iterable(set([1, 2])))
[1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2]
>>> unpack_list_from_iterable(Iter([1, 2]))
[1, 2, 1, 2, 1, 2, 1]
>>> unpack_list_from_iterable([3])
[1, 2, 3, 1, 3, 3, 3, 2, 1, 3]
>>> unpack_list_from_iterable(set([3]))
[1, 2, 3, 1, 3, 3, 3, 2, 1, 3]
>>> unpack_list_from_iterable(Iter([3]))
[1, 2, 3, 1, 2, 1]
>>> unpack_list_from_iterable([])
[1, 2, 1, 2, 1]
>>> unpack_list_from_iterable(set([]))
[1, 2, 1, 2, 1]
>>> unpack_list_from_iterable([])
[1, 2, 1, 2, 1]
>>> unpack_list_from_iterable(Iter([1, 2, 3]))
[1, 2, 1, 2, 3, 1, 2, 1]
"""
return [1, 2, *it, 1, *[*it, *it], *it, 2, 1, *it]
###### sets
@cython.test_fail_if_path_exists(
"//SetNode//SetNode",
"//MergedSetNode//SetNode",
"//MergedSetNode//MergedSetNode",
"//MergedSequenceNode",
)
def unpack_set_literal():
"""
......@@ -131,10 +321,12 @@ def unpack_set_from_iterable(it):
return {1, 2, *it, 1, *{*it, *it}, *it, 2, 1, *it, *it}
#### dicts
@cython.test_fail_if_path_exists(
"//DictNode//DictNode",
"//MergedDictNode//DictNode",
"//MergedDictNode//MergedDictNode",
"//MergedDictNode",
)
def unpack_dict_literal():
"""
......@@ -145,6 +337,18 @@ def unpack_dict_literal():
return {**{'a': 1, 'b': 2, **{'c': 4, 'd': 5}}}
@cython.test_fail_if_path_exists(
"//DictNode//DictNode",
"//MergedDictNode",
)
def unpack_dict_literal_empty():
"""
>>> unpack_dict_literal_empty()
{}
"""
return {**{**{}, **{}}, **{}, **{**{**{}}}}
def unpack_dict_simple(it):
"""
>>> d = unpack_dict_simple({})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment