Use METH_FASTCALL on CPython >= 3.7 (GH-3021)

Cython/Compiler/ExprNodes.py

@@ -9625,8 +9625,8 @@ class LambdaNode(InnerFunctionNode):
         self.lambda_name = self.def_node.lambda_name = env.next_id('lambda')
         self.def_node.no_assignment_synthesis = True
         self.def_node.pymethdef_required = True
-        self.def_node.analyse_declarations(env)
         self.def_node.is_cyfunction = True
+        self.def_node.analyse_declarations(env)
         self.pymethdef_cname = self.def_node.entry.pymethdef_cname
         env.add_lambda_def(self.def_node)
 

Cython/Compiler/FusedNode.py

@@ -827,6 +827,10 @@ class FusedCFuncDefNode(StatListNode):
         else:
             nodes = self.nodes
 
+        # For the moment, fused functions do not support METH_FASTCALL
+        for node in nodes:
+            node.entry.signature.use_fastcall = False
+
         signatures = [StringEncoding.EncodedString(node.specialized_signature_string)
                       for node in nodes]
         keys = [ExprNodes.StringNode(node.pos, value=sig)

Cython/Compiler/Naming.py

@@ -77,6 +77,7 @@ interned_prefixes = {
 ctuple_type_prefix = pyrex_prefix + "ctuple_"
 args_cname       = pyrex_prefix + "args"
 nargs_cname      = pyrex_prefix + "nargs"
+kwvalues_cname   = pyrex_prefix + "kwvalues"
 generator_cname  = pyrex_prefix + "generator"
 sent_value_cname = pyrex_prefix + "sent_value"
 pykwdlist_cname  = pyrex_prefix + "pyargnames"

Cython/Compiler/ParseTreeTransforms.py

@@ -1800,6 +1800,8 @@ if VALUE is not None:
         node.stats.insert(0, node.py_func)
         node.py_func = self.visit(node.py_func)
         node.update_fused_defnode_entry(env)
+        # For the moment, fused functions do not support METH_FASTCALL
+        node.py_func.entry.signature.use_fastcall = False
         pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func, binding=True)
         pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env))
         node.resulting_fused_function = pycfunc
@@ -1937,6 +1939,9 @@ if VALUE is not None:
             rhs.binding = True
 
         node.is_cyfunction = rhs.binding
+        if rhs.binding:
+            # For the moment, CyFunctions do not support METH_FASTCALL
+            node.entry.signature.use_fastcall = False
         return self._create_assignment(node, rhs, env)
 
     def _create_assignment(self, def_node, rhs, env):

Cython/Compiler/TypeSlots.py

@@ -9,6 +9,8 @@ from . import Naming
 from . import PyrexTypes
 from .Errors import error
 
+import copy
+
 invisible = ['__cinit__', '__dealloc__', '__richcmp__',
              '__nonzero__', '__bool__']
 
@@ -23,6 +25,7 @@ class Signature(object):
     #  fixed_arg_format   string
     #  ret_format         string
     #  error_value        string
+    #  use_fastcall       boolean
     #
     #  The formats are strings made up of the following
     #  characters:
@@ -86,6 +89,9 @@ class Signature(object):
         'z': "-1",
     }
 
+    # Use METH_FASTCALL instead of METH_VARARGS
+    use_fastcall = False
+
     def __init__(self, arg_format, ret_format, nogil=False):
         self.has_dummy_arg = False
         self.has_generic_args = False
@@ -159,15 +165,20 @@ class Signature(object):
             if self.has_dummy_arg:
                 full_args = "O" + full_args
             if full_args in ["O", "T"]:
-                if self.has_generic_args:
-                    return [method_varargs, method_keywords]
-                else:
+                if not self.has_generic_args:
                     return [method_noargs]
+                elif self.use_fastcall:
+                    return [method_fastcall, method_keywords]
+                else:
+                    return [method_varargs, method_keywords]
             elif full_args in ["OO", "TO"] and not self.has_generic_args:
                 return [method_onearg]
 
             if self.is_staticmethod:
-                return [method_varargs, method_keywords]
+                if self.use_fastcall:
+                    return [method_fastcall, method_keywords]
+                else:
+                    return [method_varargs, method_keywords]
         return None
 
     def method_function_type(self):
@@ -179,8 +190,25 @@ class Signature(object):
                 return "PyCFunction"
             if m == method_varargs:
                 return "PyCFunction" + kw
+            if m == method_fastcall:
+                return "__Pyx_PyCFunction_FastCall" + kw
         return None
 
+    def with_fastcall(self):
+        # Return a copy of this Signature with use_fastcall=True
+        sig = copy.copy(self)
+        sig.use_fastcall = True
+        return sig
+
+    @property
+    def fastvar(self):
+        # Used to select variants of functions, one dealing with METH_VARARGS
+        # and one dealing with __Pyx_METH_FASTCALL
+        if self.use_fastcall:
+            return "FASTCALL"
+        else:
+            return "VARARGS"
+
 
 class SlotDescriptor(object):
     #  Abstract base class for type slot descriptors.
@@ -958,5 +986,6 @@ MethodSlot(descrdelfunc, "", "__delete__")
 method_noargs   = "METH_NOARGS"
 method_onearg   = "METH_O"
 method_varargs  = "METH_VARARGS"
+method_fastcall = "__Pyx_METH_FASTCALL"  # Actually VARARGS on versions < 3.7
 method_keywords = "METH_KEYWORDS"
 method_coexist  = "METH_COEXIST"

Cython/Utility/FunctionArguments.c

@@ -117,16 +117,19 @@ static void __Pyx_RaiseMappingExpectedError(PyObject* arg) {
 
 //////////////////// KeywordStringCheck.proto ////////////////////
 
-static int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /*proto*/
+static int __Pyx_CheckKeywordStrings(PyObject *kw, const char* function_name, int kw_allowed); /*proto*/
 
 //////////////////// KeywordStringCheck ////////////////////
 
-//  __Pyx_CheckKeywordStrings raises an error if non-string keywords
-//  were passed to a function, or if any keywords were passed to a
-//  function that does not accept them.
+// __Pyx_CheckKeywordStrings raises an error if non-string keywords
+// were passed to a function, or if any keywords were passed to a
+// function that does not accept them.
+//
+// The "kw" argument is either a dict (for METH_VARARGS) or a tuple
+// (for METH_FASTCALL).
 
 static int __Pyx_CheckKeywordStrings(
-    PyObject *kwdict,
+    PyObject *kw,
     const char* function_name,
     int kw_allowed)
 {
@@ -134,18 +137,35 @@ static int __Pyx_CheckKeywordStrings(
     Py_ssize_t pos = 0;
 #if CYTHON_COMPILING_IN_PYPY
     /* PyPy appears to check keywords at call time, not at unpacking time => not much to do here */
-    if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0))
+    if (!kw_allowed && PyDict_Next(kw, &pos, &key, 0))
         goto invalid_keyword;
     return 1;
 #else
-    while (PyDict_Next(kwdict, &pos, &key, 0)) {
+    if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kw))) {
+        if (unlikely(PyTuple_GET_SIZE(kw) == 0))
+            return 1;
+        if (!kw_allowed)
+            goto invalid_keyword;
+#if PY_VERSION_HEX < 0x03090000
+        // On CPython >= 3.9, the FASTCALL protocol guarantees that keyword
+        // names are strings (see https://bugs.python.org/issue37540)
+        for (pos = 0; pos < PyTuple_GET_SIZE(kw); pos++) {
+            key = PyTuple_GET_ITEM(kw, pos);
+            if (unlikely(!PyUnicode_Check(key)))
+                goto invalid_keyword_type;
+        }
+#endif
+        return 1;
+    }
+
+    while (PyDict_Next(kw, &pos, &key, 0)) {
         #if PY_MAJOR_VERSION < 3
         if (unlikely(!PyString_Check(key)))
         #endif
             if (unlikely(!PyUnicode_Check(key)))
                 goto invalid_keyword_type;
     }
-    if ((!kw_allowed) && unlikely(key))
+    if (!kw_allowed && unlikely(key))
         goto invalid_keyword;
     return 1;
 invalid_keyword_type:
@@ -154,11 +174,12 @@ invalid_keyword_type:
     return 0;
 #endif
 invalid_keyword:
-    PyErr_Format(PyExc_TypeError,
     #if PY_MAJOR_VERSION < 3
+    PyErr_Format(PyExc_TypeError,
         "%.200s() got an unexpected keyword argument '%.200s'",
         function_name, PyString_AsString(key));
     #else
+    PyErr_Format(PyExc_TypeError,
         "%s() got an unexpected keyword argument '%U'",
         function_name, key);
     #endif
@@ -168,17 +189,22 @@ invalid_keyword:
 
 //////////////////// ParseKeywords.proto ////////////////////
 
-static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \
-    PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \
+static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject *const *kwvalues,
+    PyObject **argnames[],
+    PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,
     const char* function_name); /*proto*/
 
 //////////////////// ParseKeywords ////////////////////
 //@requires: RaiseDoubleKeywords
 
 //  __Pyx_ParseOptionalKeywords copies the optional/unknown keyword
-//  arguments from the kwds dict into kwds2.  If kwds2 is NULL, unknown
+//  arguments from kwds into the dict kwds2.  If kwds2 is NULL, unknown
 //  keywords will raise an invalid keyword error.
 //
+//  When not using METH_FASTCALL, kwds is a dict and kwvalues is NULL.
+//  Otherwise, kwds is a tuple with keyword names and kwvalues is a C
+//  array with the corresponding values.
+//
 //  Three kinds of errors are checked: 1) non-string keywords, 2)
 //  unexpected keywords and 3) overlap with positional arguments.
 //
@@ -190,6 +216,7 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \
 
 static int __Pyx_ParseOptionalKeywords(
     PyObject *kwds,
+    PyObject *const *kwvalues,
     PyObject **argnames[],
     PyObject *kwds2,
     PyObject *values[],
@@ -200,8 +227,20 @@ static int __Pyx_ParseOptionalKeywords(
     Py_ssize_t pos = 0;
     PyObject*** name;
     PyObject*** first_kw_arg = argnames + num_pos_args;
+    int kwds_is_tuple = CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds));
+
+    while (1) {
+        if (kwds_is_tuple) {
+            if (pos >= PyTuple_GET_SIZE(kwds)) break;
+            key = PyTuple_GET_ITEM(kwds, pos);
+            value = kwvalues[pos];
+            pos++;
+        }
+        else
+        {
+            if (!PyDict_Next(kwds, &pos, &key, &value)) break;
+        }
 
-    while (PyDict_Next(kwds, &pos, &key, &value)) {
         name = first_kw_arg;
         while (*name && (**name != key)) name++;
         if (*name) {
@@ -284,11 +323,12 @@ invalid_keyword_type:
         "%.200s() keywords must be strings", function_name);
     goto bad;
 invalid_keyword:
-    PyErr_Format(PyExc_TypeError,
     #if PY_MAJOR_VERSION < 3
+    PyErr_Format(PyExc_TypeError,
         "%.200s() got an unexpected keyword argument '%.200s'",
         function_name, PyString_AsString(key));
     #else
+    PyErr_Format(PyExc_TypeError,
         "%s() got an unexpected keyword argument '%U'",
         function_name, key);
     #endif
@@ -350,3 +390,74 @@ bad:
     Py_XDECREF(iter);
     return -1;
 }
+
+
+/////////////// fastcall.proto ///////////////
+
+// We define various functions and macros with two variants:
+//..._FASTCALL and ..._VARARGS
+
+// The first is used when METH_FASTCALL is enabled and the second is used
+// otherwise. If the Python implementation does not support METH_FASTCALL
+// (because it's an old version of CPython or it's not CPython at all),
+// then the ..._FASTCALL macros simply alias ..._VARARGS
+
+#define __Pyx_Arg_VARARGS(args, i) PyTuple_GET_ITEM(args, i)
+#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds)
+#define __Pyx_KwValues_VARARGS(args, nargs) NULL
+#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s)
+#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw)
+#if CYTHON_METH_FASTCALL
+    #define __Pyx_Arg_FASTCALL(args, i) args[i]
+    #define __Pyx_NumKwargs_FASTCALL(kwds) PyTuple_GET_SIZE(kwds)
+    #define __Pyx_KwValues_FASTCALL(args, nargs) (&args[nargs])
+    static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s);
+    #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw)
+#else
+    #define __Pyx_Arg_FASTCALL __Pyx_Arg_VARARGS
+    #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS
+    #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS
+    #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS
+    #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS
+#endif
+
+#if CYTHON_COMPILING_IN_CPYTHON
+#define __Pyx_ArgsSlice_VARARGS(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_VARARGS(args, start), stop - start)
+#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_FASTCALL(args, start), stop - start)
+#else
+/* Not CPython, so certainly no METH_FASTCALL support */
+#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop)
+#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop)
+#endif
+
+
+/////////////// fastcall ///////////////
+//@requires: ObjectHandling.c::TupleAndListFromArray
+//@requires: StringTools.c::UnicodeEquals
+
+#if CYTHON_METH_FASTCALL
+// kwnames: tuple with names of keyword arguments
+// kwvalues: C array with values of keyword arguments
+// s: str with the keyword name to look for
+static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s)
+{
+    // Search the kwnames array for s and return the corresponding value.
+    // We do two loops: a first one to compare pointers (which will find a
+    // match if the name in kwnames is interned, given that s is interned
+    // by Cython). A second loop compares the actual strings.
+    Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames);
+    for (i = 0; i < n; i++)
+    {
+        if (s == PyTuple_GET_ITEM(kwnames, i)) return kwvalues[i];
+    }
+    for (i = 0; i < n; i++)
+    {
+        int eq = __Pyx_PyUnicode_Equals(s, PyTuple_GET_ITEM(kwnames, i), Py_EQ);
+        if (unlikely(eq != 0)) {
+            if (unlikely(eq < 0)) return NULL;  // error
+            return kwvalues[i];
+        }
+    }
+    return NULL;  // not found (no exception set)
+}
+#endif

Cython/Utility/ModuleSetupCode.c

@@ -73,6 +73,8 @@
   #define CYTHON_FAST_THREAD_STATE 0
   #undef CYTHON_FAST_GIL
   #define CYTHON_FAST_GIL 0
+  #undef CYTHON_METH_FASTCALL
+  #define CYTHON_METH_FASTCALL 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
   #undef CYTHON_PEP489_MULTI_PHASE_INIT
@@ -118,6 +120,8 @@
   #define CYTHON_FAST_THREAD_STATE 0
   #undef CYTHON_FAST_GIL
   #define CYTHON_FAST_GIL 0
+  #undef CYTHON_METH_FASTCALL
+  #define CYTHON_METH_FASTCALL 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
   #undef CYTHON_PEP489_MULTI_PHASE_INIT
@@ -177,6 +181,11 @@
     // Py3<3.5.2 does not support _PyThreadState_UncheckedGet().
     #define CYTHON_FAST_GIL (PY_MAJOR_VERSION < 3 || PY_VERSION_HEX >= 0x03060000)
   #endif
+  #ifndef CYTHON_METH_FASTCALL
+    /* CPython 3.6 introduced METH_FASTCALL but with slightly different
+     * semantics. It became stable starting from CPython 3.7 */
+    #define CYTHON_METH_FASTCALL (PY_VERSION_HEX >= 0x030700A1)
+  #endif
   #ifndef CYTHON_FAST_PYCALL
     #define CYTHON_FAST_PYCALL 1
   #endif
@@ -450,6 +459,16 @@ class __Pyx_FakeReference {
   #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords
 #endif
 
+#if CYTHON_METH_FASTCALL
+  #define __Pyx_METH_FASTCALL METH_FASTCALL
+  #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast
+  #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords
+#else
+  #define __Pyx_METH_FASTCALL METH_VARARGS
+  #define __Pyx_PyCFunction_FastCall PyCFunction
+  #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords
+#endif
+
 #if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc)
   #define PyObject_Malloc(s)   PyMem_Malloc(s)
   #define PyObject_Free(p)     PyMem_Free(p)
@@ -531,10 +550,26 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) {
   #define __Pyx_PyNumber_InPlaceDivide(x,y)  PyNumber_InPlaceDivide(x,y)
 #endif
 
-#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 && CYTHON_USE_UNICODE_INTERNALS
-#define __Pyx_PyDict_GetItemStr(dict, name)  _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash)
+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX > 0x030600B4 && CYTHON_USE_UNICODE_INTERNALS
+// _PyDict_GetItem_KnownHash() exists since CPython 3.5, but it was
+// dropping exceptions. Since 3.6, exceptions are kept.
+#define __Pyx_PyDict_GetItemStrWithError(dict, name)  _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash)
+static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) {
+    PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name);
+    if (res == NULL) PyErr_Clear();
+    return res;
+}
+#elif PY_MAJOR_VERSION >= 3
+#define __Pyx_PyDict_GetItemStrWithError  PyDict_GetItemWithError
+#define __Pyx_PyDict_GetItemStr           PyDict_GetItem
 #else
-#define __Pyx_PyDict_GetItemStr(dict, name)  PyDict_GetItem(dict, name)
+static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) {
+    PyObject *res = PyObject_GetItem(dict, name);
+    if (res == NULL && PyErr_ExceptionMatches(PyExc_KeyError))
+        PyErr_Clear();
+    return res;
+}
+#define __Pyx_PyDict_GetItemStr           PyDict_GetItem
 #endif
 
 /* new Py3.3 unicode type (PEP 393) */

Cython/Utility/ObjectHandling.c

@@ -738,8 +738,8 @@ bad:
 /////////////// TupleAndListFromArray.proto ///////////////
 
 #if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n);
 static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n);
+static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n);
 #endif
 
 /////////////// TupleAndListFromArray ///////////////

tests/run/fastcall.pyx

 # mode: run
 # tag: METH_FASTCALL
 
+cimport cython
+
 import sys
 import struct
 from collections import deque
@@ -63,3 +65,45 @@ cdef class SelfCast:
     """
     def index_of_self(self, list orbit not None):
         return orbit.index(self)
+
+
+cdef extern from *:
+    int PyCFunction_Check(op)
+    int PyCFunction_GET_FLAGS(op)
+
+
+def has_fastcall(meth):
+    """
+    Given a builtin_function_or_method ``meth``, return whether it uses
+    ``METH_FASTCALL``.
+    """
+    if not PyCFunction_Check(meth):
+        raise TypeError("not a builtin_function_or_method")
+    # Hardcode METH_FASTCALL constant equal to 0x80 for simplicity
+    return bool(PyCFunction_GET_FLAGS(meth) & 0x80)
+
+
+def assert_fastcall(meth):
+    """
+    Assert that ``meth`` uses ``METH_FASTCALL`` if the Python
+    implementation supports it.
+    """
+    # getattr uses METH_FASTCALL on CPython >= 3.7
+    if has_fastcall(getattr) and not has_fastcall(meth):
+        raise AssertionError(f"{meth} does not use METH_FASTCALL")
+
+
+@cython.binding(False)
+def fastcall_function(**kw):
+    """
+    >>> assert_fastcall(fastcall_function)
+    """
+    return kw
+
+cdef class Dummy:
+    @cython.binding(False)
+    def fastcall_method(self, x, *args, **kw):
+        """
+        >>> assert_fastcall(Dummy().fastcall_method)
+        """
+        return tuple(args) + tuple(kw)

tests/run/str_subclass_kwargs.pyx

+def test_str_subclass_kwargs(k=None):
+    """
+    Test passing keywords with names that are not of type ``str``
+    but a subclass:
+
+    >>> class StrSubclass(str):
+    ...     pass
+    >>> class StrNoCompare(str):
+    ...     def __eq__(self, other):
+    ...         raise RuntimeError("do not compare me")
+    ...     def __hash__(self):
+    ...         return hash(str(self))
+    >>> kwargs = {StrSubclass('k'): 'value'}
+    >>> test_str_subclass_kwargs(**kwargs)
+    'value'
+    >>> kwargs = {StrNoCompare('k'): 'value'}
+    >>> test_str_subclass_kwargs(**kwargs)
+    Traceback (most recent call last):
+    RuntimeError: do not compare me
+    """
+    return k