Adds positional only args support (PEP 570)

Cython/Compiler/Nodes.py

@@ -861,8 +861,9 @@ class CArgDeclNode(Node):
     # annotation     ExprNode or None   Py3 function arg annotation
     # is_self_arg    boolean            Is the "self" arg of an extension type method
     # is_type_arg    boolean            Is the "class" arg of an extension type classmethod
-    # is_kw_only     boolean            Is a keyword-only argument
+    # kw_only        boolean            Is a keyword-only argument
     # is_dynamic     boolean            Non-literal arg stored inside CyFunction
+    # pos_only       boolean            Is a positional-only argument
 
     child_attrs = ["base_type", "declarator", "default", "annotation"]
     outer_attrs = ["default", "annotation"]
@@ -871,6 +872,7 @@ class CArgDeclNode(Node):
     is_type_arg = 0
     is_generic = 1
     kw_only = 0
+    pos_only = 0
     not_none = 0
     or_none = 0
     type = None
@@ -3655,6 +3657,7 @@ class DefNodeWrapper(FuncDefNode):
         positional_args = []
         required_kw_only_args = []
         optional_kw_only_args = []
+        num_pos_only_args = 0
         for arg in args:
             if arg.is_generic:
                 if arg.default:
@@ -3668,6 +3671,9 @@ class DefNodeWrapper(FuncDefNode):
                 elif not arg.is_self_arg and not arg.is_type_arg:
                     positional_args.append(arg)
 
+                if arg.pos_only:
+                    num_pos_only_args += 1
+
         # sort required kw-only args before optional ones to avoid special
         # cases in the unpacking code
         kw_only_args = required_kw_only_args + optional_kw_only_args
@@ -3685,10 +3691,10 @@ class DefNodeWrapper(FuncDefNode):
 
         code.putln('{')
         all_args = tuple(positional_args) + tuple(kw_only_args)
-        code.putln("static PyObject **%s[] = {%s,0};" % (
+        code.putln("static PyObject **%s[] = {%s};" % (
             Naming.pykwdlist_cname,
             ','.join(['&%s' % code.intern_identifier(arg.name)
-                      for arg in all_args])))
+                      for arg in all_args if not arg.pos_only] + ['0'])))
 
         # Before being converted and assigned to the target variables,
         # borrowed references to all unpacked argument values are
@@ -3706,8 +3712,8 @@ class DefNodeWrapper(FuncDefNode):
             Naming.kwds_cname))
         self.generate_keyword_unpacking_code(
             min_positional_args, max_positional_args,
-            has_fixed_positional_count, has_kw_only_args,
-            all_args, argtuple_error_label, code)
+            num_pos_only_args, has_fixed_positional_count,
+            has_kw_only_args, all_args, argtuple_error_label, code)
 
         # --- optimised code when we do not receive any keyword arguments
         if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args:
@@ -3870,8 +3876,8 @@ class DefNodeWrapper(FuncDefNode):
                 code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value)))
 
     def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args,
-                                        has_fixed_positional_count, has_kw_only_args,
-                                        all_args, argtuple_error_label, code):
+                                        num_pos_only_args, has_fixed_positional_count,
+                                        has_kw_only_args, all_args, argtuple_error_label, code):
         code.putln('Py_ssize_t kw_args;')
         code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname)
         # copy the values from the args tuple and check that it's not too long
@@ -3901,9 +3907,12 @@ class DefNodeWrapper(FuncDefNode):
         code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname)
         if self.num_required_args or max_positional_args > 0:
             last_required_arg = -1
+            last_required_posonly_arg = -1
             for i, arg in enumerate(all_args):
                 if not arg.default:
                     last_required_arg = i
+                if arg.pos_only and not arg.default:
+                    last_required_posonly_arg = i
             if last_required_arg < max_positional_args:
                 last_required_arg = max_positional_args-1
             if max_positional_args > 0:
@@ -3917,6 +3926,12 @@ class DefNodeWrapper(FuncDefNode):
                     else:
                         code.putln('case %2d:' % i)
                 pystring_cname = code.intern_identifier(arg.name)
+                if arg.pos_only:
+                    if i == last_required_posonly_arg:
+                        code.put_goto(argtuple_error_label)
+                    if i == last_required_arg:
+                        code.putln('break;')
+                    continue
                 if arg.default:
                     if arg.kw_only:
                         # optional kw-only args are handled separately below
@@ -3971,14 +3986,34 @@ class DefNodeWrapper(FuncDefNode):
         # arguments, this will always do the right thing for unpacking
         # keyword arguments, so that we can concentrate on optimising
         # common cases above.
+        #
+        # ParseOptionalKeywords() needs to know how many of the arguments
+        # that could be passed as keywords have in fact been passed as
+        # positional args.
+        if num_pos_only_args > 0:
+            # There are positional-only arguments which we don't want to count,
+            # since they cannot be keyword arguments.  Subtract the number of
+            # pos-only arguments from the number of positional arguments we got.
+            # If we get a negative number then none of the keyword arguments were
+            # passed as positional args.
+            code.putln('const Py_ssize_t kwd_pos_args = (pos_args < %d) ? 0 : (pos_args - %d);' % (
+                num_pos_only_args, num_pos_only_args))
+        elif max_positional_args > 0:
+            code.putln('const Py_ssize_t kwd_pos_args = pos_args;')
+
         if max_positional_args == 0:
             pos_arg_count = "0"
         elif self.star_arg:
-            code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % (
-                max_positional_args, max_positional_args))
+            # If there is a *arg, the number of used positional args could be larger than
+            # the number of possible keyword arguments.  But ParseOptionalKeywords() uses the
+            # number of positional args as an index into the keyword argument name array,
+            # if this is larger than the number of kwd args we get a segfault.  So round
+            # this down to max_positional_args - num_pos_only_args (= num possible kwd args).
+            code.putln("const Py_ssize_t used_pos_args = (kwd_pos_args < %d) ? kwd_pos_args : %d;" % (
+                max_positional_args - num_pos_only_args, max_positional_args - num_pos_only_args))
             pos_arg_count = "used_pos_args"
         else:
-            pos_arg_count = "pos_args"
+            pos_arg_count = "kwd_pos_args"
         code.globalstate.use_utility_code(
             UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
         code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % (

Cython/Compiler/Parsing.py

@@ -2965,7 +2965,7 @@ def p_exception_value_clause(s):
             exc_val = p_test(s)
     return exc_val, exc_check
 
-c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':']))
+c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':', '/']))
 
 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
                  nonempty_declarators = 0, kw_only = 0, annotated = 1):
@@ -3424,6 +3424,20 @@ def p_varargslist(s, terminator=')', annotated=1):
                         annotated = annotated)
     star_arg = None
     starstar_arg = None
+    if s.sy == '/':
+        if len(args) == 0:
+            s.error("Got zero positional-only arguments despite presence of "
+                    "positional-only specifier '/'")
+        s.next()
+        # Mark all args to the left as pos only
+        for arg in args:
+            arg.pos_only = 1
+        if s.sy == ',':
+            s.next()
+            args.extend(p_c_arg_list(s, in_pyfunc = 1,
+                nonempty_declarators = 1, annotated = annotated))
+        elif s.sy != terminator:
+            s.error("Syntax error in Python function argument list")
     if s.sy == '*':
         s.next()
         if s.sy == 'IDENT':

tests/compile/posonly.pyx

+# mode: compile
+# tag: posonly
+
+# TODO: remove posonly tag before merge (and maybe remove this test,
+#       since it seems covered by the runs/ test)
+
+def test(x, y, z=42, /, w=43):
+    pass
+
+def test2(x, y, /):
+    pass
+
+def test3(x, /, z):
+    pass
+
+def test4(x, /, z, *, w):
+    pass

tests/run/posonly.pyx

+# mode: run
+# tag: posonly
+
+# TODO: remove posonly tag before merge
+
+import cython
+
+# TODO: add the test below to an 'error' test
+#def test_invalid_syntax_errors():
+#    def f(a, b = 5, /, c): pass
+#    def f(a = 5, b, /, c): pass
+#    def f(a = 5, b, /): pass
+#    def f(*args, /): pass
+#    def f(*args, a, /): pass
+#    def f(**kwargs, /): pass
+#    def f(/, a = 1): pass
+#    def f(/, a): pass
+#    def f(/): pass
+#    def f(*, a, /): pass
+#    def f(*, /, a): pass
+#    def f(a, /, a): pass
+#    def f(a, /, *, a): pass
+#    def f(a, b/2, c): pass
+
+def test_optional_posonly_args1(a, b=10, /, c=100):
+    """
+    >>> test_optional_posonly_args1(1, 2, 3)
+    6
+    >>> test_optional_posonly_args1(1, 2, c=3)
+    6
+    >>> test_optional_posonly_args1(1, b=2, c=3)
+    Traceback (most recent call last):
+    TypeError: test_optional_posonly_args1() got an unexpected keyword argument 'b'
+    >>> test_optional_posonly_args1(1, 2)
+    103
+    >>> test_optional_posonly_args1(1, b=2)
+    Traceback (most recent call last):
+    TypeError: test_optional_posonly_args1() got an unexpected keyword argument 'b'
+    """
+    return a + b + c
+
+def test_optional_posonly_args2(a=1, b=10, /, c=100):
+    """
+    >>> test_optional_posonly_args2(1, 2, 3)
+    6
+    >>> test_optional_posonly_args2(1, 2, c=3)
+    6
+    >>> test_optional_posonly_args2(1, b=2, c=3)
+    Traceback (most recent call last):
+    TypeError: test_optional_posonly_args2() got an unexpected keyword argument 'b'
+    >>> test_optional_posonly_args2(1, 2)
+    103
+    >>> test_optional_posonly_args2(1, b=2)
+    Traceback (most recent call last):
+    TypeError: test_optional_posonly_args2() got an unexpected keyword argument 'b'
+    >>> test_optional_posonly_args2(1, c=2)
+    13
+    """
+    return a + b + c
+
+# TODO: remove the test below?  would need to hard-code the function with > 255 posonly args
+#def test_syntax_for_many_positional_only():
+#    # more than 255 positional only arguments, should compile ok
+#    fundef = "def f(%s, /):\n  pass\n" % ', '.join('i%d' % i for i in range(300))
+#    compile(fundef, "<test>", "single")
+
+# TODO: remove the test below?  doesn't seem relevant to Cython implementation
+#def test_pos_only_definition(self):
+#    def f(a, b, c, /, d, e=1, *, f, g=2):
+#        pass
+#
+#    self.assertEqual(2, f.__code__.co_argcount)  # 2 "standard args"
+#    self.assertEqual(3, f.__code__.co_posonlyargcount)
+#    self.assertEqual((1,), f.__defaults__)
+#
+#    def f(a, b, c=1, /, d=2, e=3, *, f, g=4):
+#        pass
+#
+#    self.assertEqual(2, f.__code__.co_argcount)  # 2 "standard args"
+#    self.assertEqual(3, f.__code__.co_posonlyargcount)
+#    self.assertEqual((1, 2, 3), f.__defaults__)
+
+def test_pos_only_call_via_unpacking(a, b, /):
+    """
+    >>> test_pos_only_call_via_unpacking(*[1,2])
+    3
+    """
+    return a + b
+
+def test_use_positional_as_keyword1(a, /):
+    """
+    >>> test_use_positional_as_keyword1(a=1)
+    Traceback (most recent call last):
+    TypeError: test_use_positional_as_keyword1() takes no keyword arguments
+    """
+    pass
+
+def test_use_positional_as_keyword2(a, /, b):
+    """
+    >>> test_use_positional_as_keyword2(a=1, b=2)
+    Traceback (most recent call last):
+    TypeError: test_use_positional_as_keyword2() takes exactly 2 positional arguments (0 given)
+    """
+    pass
+
+def test_use_positional_as_keyword3(a, b, /):
+    """
+    >>> test_use_positional_as_keyword3(a=1, b=2)
+    Traceback (most recent call last):
+    TypeError: test_use_positional_as_keyword3() takes exactly 2 positional arguments (0 given)
+    """
+    pass
+
+def test_positional_only_and_arg_invalid_calls(a, b, /, c):
+    """
+    >>> test_positional_only_and_arg_invalid_calls(1, 2)
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_arg_invalid_calls() takes exactly 3 positional arguments (2 given)
+    >>> test_positional_only_and_arg_invalid_calls(1)
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_arg_invalid_calls() takes exactly 3 positional arguments (1 given)
+    >>> test_positional_only_and_arg_invalid_calls(1,2,3,4)
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_arg_invalid_calls() takes exactly 3 positional arguments (4 given)
+    """
+    pass
+
+def test_positional_only_and_optional_arg_invalid_calls(a, b, /, c=3):
+    """
+    >>> test_positional_only_and_optional_arg_invalid_calls(1, 2)
+    >>> test_positional_only_and_optional_arg_invalid_calls(1)
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_optional_arg_invalid_calls() takes at least 2 positional arguments (1 given)
+    >>> test_positional_only_and_optional_arg_invalid_calls()
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_optional_arg_invalid_calls() takes at least 2 positional arguments (0 given)
+    >>> test_positional_only_and_optional_arg_invalid_calls(1, 2, 3, 4)
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_optional_arg_invalid_calls() takes at most 3 positional arguments (4 given)
+    """
+    pass
+
+def test_positional_only_invalid_calls(a, b, /):
+    """
+    >>> test_positional_only_invalid_calls(1, 2)
+    >>> test_positional_only_invalid_calls(1)
+    Traceback (most recent call last):
+    TypeError: test_positional_only_invalid_calls() takes exactly 2 positional arguments (1 given)
+    >>> test_positional_only_invalid_calls()
+    Traceback (most recent call last):
+    TypeError: test_positional_only_invalid_calls() takes exactly 2 positional arguments (0 given)
+    >>> test_positional_only_invalid_calls(1, 2, 3)
+    Traceback (most recent call last):
+    TypeError: test_positional_only_invalid_calls() takes exactly 2 positional arguments (3 given)
+    """
+    pass
+
+def test_positional_only_with_optional_invalid_calls(a, b=2, /):
+    """
+    >>> test_positional_only_with_optional_invalid_calls(1)
+    >>> test_positional_only_with_optional_invalid_calls()
+    Traceback (most recent call last):
+    TypeError: test_positional_only_with_optional_invalid_calls() takes at least 1 positional argument (0 given)
+    >>> test_positional_only_with_optional_invalid_calls(1, 2, 3)
+    Traceback (most recent call last):
+    TypeError: test_positional_only_with_optional_invalid_calls() takes at most 2 positional arguments (3 given)
+    """
+    pass
+
+def test_no_standard_args_usage(a, b, /, *, c):
+    """
+    >>> test_no_standard_args_usage(1, 2, c=3)
+    >>> test_no_standard_args_usage(1, b=2, c=3)
+    Traceback (most recent call last):
+    TypeError: test_no_standard_args_usage() takes exactly 2 positional arguments (1 given)
+    """
+    pass
+
+#def test_change_default_pos_only():
+# TODO: probably remove this, since we have no __defaults__ in Cython?
+#    """
+#    >>> test_change_default_pos_only()
+#    True
+#    True
+#    """
+#    def f(a, b=2, /, c=3):
+#        return a + b + c
+#
+#    print((2,3) == f.__defaults__)
+#    f.__defaults__ = (1, 2, 3)
+#    print(f(1, 2, 3) == 6)
+
+def test_lambdas():
+    """
+    >>> test_lambdas()
+    3
+    3
+    3
+    3
+    3
+    """
+    x = lambda a, /, b: a + b
+    print(x(1,2))
+    print(x(1,b=2))
+
+    x = lambda a, /, b=2: a + b
+    print(x(1))
+
+    x = lambda a, b, /: a + b
+    print(x(1, 2))
+
+    x = lambda a, b, /, : a + b
+    print(x(1, 2))
+
+
+#TODO: need to implement this in the 'error' test
+#def test_invalid_syntax_lambda(self):
+#    lambda a, b = 5, /, c: None
+#    lambda a = 5, b, /, c: None
+#    lambda a = 5, b, /: None
+#    lambda a, /, a: None
+#    lambda a, /, *, a: None
+#    lambda *args, /: None
+#    lambda *args, a, /: None
+#    lambda **kwargs, /: None
+#    lambda /, a = 1: None
+#    lambda /, a: None
+#    lambda /: None
+#    lambda *, a, /: None
+#    lambda *, /, a: None
+
+class Example:
+    def f(self, a, b, /):
+        return a, b
+
+def test_posonly_methods():
+    """
+    >>> Example().f(1,2)
+    (1, 2)
+    >>> Example.f(Example(), 1, 2)
+    (1, 2)
+    >>> try:
+    ...     Example.f(1,2)
+    ... except TypeError:
+    ...    print("Got type error")
+    Got type error
+    >>> Example().f(1, b=2)
+    Traceback (most recent call last):
+    TypeError: f() takes exactly 3 positional arguments (2 given)
+    """
+    pass
+
+class X:
+    def f(self, *, __a=42):
+        return __a
+def test_mangling():
+    """
+    >>> X().f()
+    42
+    """
+    pass
+
+def global_pos_only_f(a, b, /):
+    pass
+
+def test_module_function():
+    """
+    >>> global_pos_only_f()
+    Traceback (most recent call last):
+    TypeError: global_pos_only_f() takes exactly 2 positional arguments (0 given)
+    """
+    pass
+
+def test_closures1(x,y):
+    """
+    >>> test_closures1(1,2)(3,4)
+    10
+    >>> test_closures1(1,2)(3)
+    Traceback (most recent call last):
+    TypeError: g() takes exactly 2 positional arguments (1 given)
+    >>> test_closures1(1,2)(3,4,5)
+    Traceback (most recent call last):
+    TypeError: g() takes exactly 2 positional arguments (3 given)
+    """
+    def g(x2,/,y2):
+        return x + y + x2 + y2
+    return g
+
+def test_closures2(x,/,y):
+    """
+    >>> test_closures2(1,2)(3,4)
+    10
+    """
+    def g(x2,y2):
+        return x + y + x2 + y2
+    return g
+
+def test_closures3(x,/,y):
+    """
+    >>> test_closures3(1,2)(3,4)
+    10
+    >>> test_closures3(1,2)(3)
+    Traceback (most recent call last):
+    TypeError: g() takes exactly 2 positional arguments (1 given)
+    >>> test_closures3(1,2)(3,4,5)
+    Traceback (most recent call last):
+    TypeError: g() takes exactly 2 positional arguments (3 given)
+    """
+    def g(x2,/,y2):
+        return x + y + x2 + y2
+    return g
+
+def test_same_keyword_as_positional_with_kwargs(something, /, **kwargs):
+    """
+    >>> test_same_keyword_as_positional_with_kwargs(42, something=42)
+    (42, {'something': 42})
+    >>> test_same_keyword_as_positional_with_kwargs(something=42)
+    Traceback (most recent call last):
+    TypeError: test_same_keyword_as_positional_with_kwargs() takes exactly 1 positional argument (0 given)
+    >>> test_same_keyword_as_positional_with_kwargs(42)
+    (42, {})
+    """
+    return (something, kwargs)