Issue #25969: Update the lib2to3 grammar to handle the unpacking

generalizations added in 3.5.

Lib/lib2to3/Grammar.txt

@@ -136,15 +136,26 @@ subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
-dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
-                (test (comp_for | (',' test)* [','])) )
+dictsetmaker: ( ((test ':' test | '**' expr)
+                 (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                ((test | star_expr)
+		 (comp_for | (',' (test | star_expr))* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
-arglist: (argument ',')* (argument [',']
-                         |'*' test (',' argument)* [',' '**' test] 
-                         |'**' test)
-argument: test [comp_for] | test '=' test  # Really [keyword '='] test
+arglist: argument (',' argument)* [',']
+
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguements are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test '=' test |
+	    '**' expr |
+	    star_expr )
 
 comp_iter: comp_for | comp_if
 comp_for: 'for' exprlist 'in' testlist_safe [comp_iter]

Lib/lib2to3/fixes/fix_apply.py

@@ -34,6 +34,17 @@ class FixApply(fixer_base.BaseFix):
         func = results["func"]
         args = results["args"]
         kwds = results.get("kwds")
+        # I feel like we should be able to express this logic in the
+        # PATTERN above but I don't know how to do it so...
+        if args:
+            if args.type == self.syms.star_expr:
+                return  # Make no change.
+            if (args.type == self.syms.argument and
+                args.children[0].value == '**'):
+                return  # Make no change.
+        if kwds and (kwds.type == self.syms.argument and
+                     kwds.children[0].value == '**'):
+            return  # Make no change.
         prefix = node.prefix
         func = func.clone()
         if (func.type not in (token.NAME, syms.atom) and

Lib/lib2to3/fixes/fix_intern.py

@@ -26,6 +26,16 @@ class FixIntern(fixer_base.BaseFix):
     """
 
     def transform(self, node, results):
+        if results:
+            # I feel like we should be able to express this logic in the
+            # PATTERN above but I don't know how to do it so...
+            obj = results['obj']
+            if obj:
+                if obj.type == self.syms.star_expr:
+                    return  # Make no change.
+                if (obj.type == self.syms.argument and
+                    obj.children[0].value == '**'):
+                    return  # Make no change.
         syms = self.syms
         obj = results["obj"].clone()
         if obj.type == syms.arglist:

Lib/lib2to3/tests/test_fixers.py

@@ -260,6 +260,10 @@ class Test_apply(FixerTestCase):
         s = """apply(f, *args)"""
         self.unchanged(s)
 
+    def test_unchanged_6b(self):
+        s = """apply(f, **kwds)"""
+        self.unchanged(s)
+
     def test_unchanged_7(self):
         s = """apply(func=f, args=args, kwds=kwds)"""
         self.unchanged(s)
@@ -2861,98 +2865,6 @@ class Test_unicode(FixerTestCase):
         a = f + """r'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
         self.check(b, a)
 
-class Test_callable(FixerTestCase):
-    fixer = "callable"
-
-    def test_prefix_preservation(self):
-        b = """callable(    x)"""
-        a = """import collections\nisinstance(    x, collections.Callable)"""
-        self.check(b, a)
-
-        b = """if     callable(x): pass"""
-        a = """import collections
-if     isinstance(x, collections.Callable): pass"""
-        self.check(b, a)
-
-    def test_callable_call(self):
-        b = """callable(x)"""
-        a = """import collections\nisinstance(x, collections.Callable)"""
-        self.check(b, a)
-
-    def test_global_import(self):
-        b = """
-def spam(foo):
-    callable(foo)"""[1:]
-        a = """
-import collections
-def spam(foo):
-    isinstance(foo, collections.Callable)"""[1:]
-        self.check(b, a)
-
-        b = """
-import collections
-def spam(foo):
-    callable(foo)"""[1:]
-        # same output if it was already imported
-        self.check(b, a)
-
-        b = """
-from collections import *
-def spam(foo):
-    callable(foo)"""[1:]
-        a = """
-from collections import *
-import collections
-def spam(foo):
-    isinstance(foo, collections.Callable)"""[1:]
-        self.check(b, a)
-
-        b = """
-do_stuff()
-do_some_other_stuff()
-assert callable(do_stuff)"""[1:]
-        a = """
-import collections
-do_stuff()
-do_some_other_stuff()
-assert isinstance(do_stuff, collections.Callable)"""[1:]
-        self.check(b, a)
-
-        b = """
-if isinstance(do_stuff, Callable):
-    assert callable(do_stuff)
-    do_stuff(do_stuff)
-    if not callable(do_stuff):
-        exit(1)
-    else:
-        assert callable(do_stuff)
-else:
-    assert not callable(do_stuff)"""[1:]
-        a = """
-import collections
-if isinstance(do_stuff, Callable):
-    assert isinstance(do_stuff, collections.Callable)
-    do_stuff(do_stuff)
-    if not isinstance(do_stuff, collections.Callable):
-        exit(1)
-    else:
-        assert isinstance(do_stuff, collections.Callable)
-else:
-    assert not isinstance(do_stuff, collections.Callable)"""[1:]
-        self.check(b, a)
-
-    def test_callable_should_not_change(self):
-        a = """callable(*x)"""
-        self.unchanged(a)
-
-        a = """callable(x, y)"""
-        self.unchanged(a)
-
-        a = """callable(x, kw=y)"""
-        self.unchanged(a)
-
-        a = """callable()"""
-        self.unchanged(a)
 
 class Test_filter(FixerTestCase):
     fixer = "filter"

Lib/lib2to3/tests/test_parser.py

@@ -154,6 +154,41 @@ class TestRaiseChanges(GrammarTest):
         self.invalid_syntax("raise E from")
 
 
+# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292
+# and Lib/test/text_parser.py test_list_displays, test_set_displays,
+# test_dict_displays, test_argument_unpacking, ... changes.
+class TestUnpackingGeneralizations(GrammarTest):
+    def test_mid_positional_star(self):
+        self.validate("""func(1, *(2, 3), 4)""")
+
+    def test_double_star_dict_literal(self):
+        self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""")
+
+    def test_double_star_dict_literal_after_keywords(self):
+        self.validate("""func(spam='fried', **{'eggs':'scrambled'})""")
+
+    def test_list_display(self):
+        self.validate("""[*{2}, 3, *[4]]""")
+
+    def test_set_display(self):
+        self.validate("""{*{2}, 3, *[4]}""")
+
+    def test_dict_display_1(self):
+        self.validate("""{**{}}""")
+
+    def test_dict_display_2(self):
+        self.validate("""{**{}, 3:4, **{5:6, 7:8}}""")
+
+    def test_argument_unpacking_1(self):
+        self.validate("""f(a, *b, *c, d)""")
+
+    def test_argument_unpacking_2(self):
+        self.validate("""f(**a, **b)""")
+
+    def test_argument_unpacking_3(self):
+        self.validate("""f(2, *a, *b, **b, **c, **d)""")
+
+
 # Adaptated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
 class TestFunctionAnnotations(GrammarTest):
     def test_1(self):

Misc/NEWS

@@ -42,6 +42,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #25969: Update the lib2to3 grammar to handle the unpacking
+  generalizations added in 3.5.
+
 - Issue #24594: Validates persist parameter when opening MSI database
 
 - Issue #27570: Avoid zero-length memcpy() etc calls with null source