Implement PEP-487: simpler customisation of class creation (GH-3533)

Currently excludes PyPy2.
Closes GH-2781.

CHANGES.rst

@@ -11,6 +11,9 @@ Features added
 * Cython functions now use the PEP-590 vectorcall protocol in Py3.7+.
   Patch by Jeroen Demeyer.  (Github issue #2263)
 
+* The simplified Py3.6 customisation of class creation is implemented (PEP-487).
+  (Github issue #2781)
+
 * Unicode identifiers are supported in Cython code (PEP 3131).
   Patch by David Woods.  (Github issue #2601)
 

Cython/Utility/ModuleSetupCode.c

@@ -91,6 +91,9 @@
   #define CYTHON_METH_FASTCALL 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
+  #ifndef CYTHON_PEP487_INIT_SUBCLASS
+    #define CYTHON_PEP487_INIT_SUBCLASS (PY_MAJOR_VERSION >= 3)
+  #endif
   #undef CYTHON_PEP489_MULTI_PHASE_INIT
   #define CYTHON_PEP489_MULTI_PHASE_INIT 0
   #undef CYTHON_USE_TP_FINALIZE
@@ -139,6 +142,8 @@
   #define CYTHON_METH_FASTCALL 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
+  #undef CYTHON_PEP487_INIT_SUBCLASS
+  #define CYTHON_PEP487_INIT_SUBCLASS 0
   #undef CYTHON_PEP489_MULTI_PHASE_INIT
   #define CYTHON_PEP489_MULTI_PHASE_INIT 0
   #undef CYTHON_USE_TP_FINALIZE
@@ -183,6 +188,9 @@
   #define CYTHON_METH_FASTCALL 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
+  #ifndef CYTHON_PEP487_INIT_SUBCLASS
+    #define CYTHON_PEP487_INIT_SUBCLASS 1
+  #endif
   #undef CYTHON_PEP489_MULTI_PHASE_INIT
   #define CYTHON_PEP489_MULTI_PHASE_INIT 0
   #ifndef CYTHON_USE_TP_FINALIZE
@@ -250,6 +258,9 @@
   #ifndef CYTHON_FAST_PYCALL
     #define CYTHON_FAST_PYCALL 1
   #endif
+  #ifndef CYTHON_PEP487_INIT_SUBCLASS
+    #define CYTHON_PEP487_INIT_SUBCLASS 1
+  #endif
   #ifndef CYTHON_PEP489_MULTI_PHASE_INIT
     #define CYTHON_PEP489_MULTI_PHASE_INIT (PY_VERSION_HEX >= 0x03050000)
   #endif

Cython/Utility/ObjectHandling.c

@@ -1020,8 +1020,14 @@ static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObj
                                       PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); /*proto*/
 
 /////////////// Py3ClassCreate ///////////////
+//@substitute: naming
 //@requires: PyObjectGetAttrStrNoError
 //@requires: CalculateMetaclass
+//@requires: PyObjectCall
+//@requires: PyObjectCall2Args
+//@requires: PyObjectLookupSpecial
+// only in fallback code:
+//@requires: GetBuiltinName
 
 static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name,
                                            PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) {
@@ -1063,6 +1069,153 @@ bad:
     return NULL;
 }
 
+#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS
+// https://www.python.org/dev/peps/pep-0487/
+static int __Pyx_SetNamesPEP487(PyObject *type_obj) {
+    PyTypeObject *type = (PyTypeObject*) type_obj;
+    PyObject *names_to_set, *key, *value, *set_name, *tmp;
+    Py_ssize_t i = 0;
+
+#if CYTHON_USE_TYPE_SLOTS
+    names_to_set = PyDict_Copy(type->tp_dict);
+#else
+    {
+        PyObject *d = PyObject_GetAttr(type_obj, PYIDENT("__dict__"));
+        names_to_set = NULL;
+        if (likely(d)) {
+            // d may not be a dict, e.g. PyDictProxy in PyPy2.
+            PyObject *names_to_set = PyDict_New();
+            int ret = likely(names_to_set) ? PyDict_Update(names_to_set, d) : -1;
+            Py_DECREF(d);
+            if (unlikely(ret < 0))
+                Py_CLEAR(names_to_set);
+        }
+    }
+#endif
+    if (unlikely(names_to_set == NULL))
+        goto bad;
+
+    while (PyDict_Next(names_to_set, &i, &key, &value)) {
+        set_name = __Pyx_PyObject_LookupSpecialNoError(value, PYIDENT("__set_name__"));
+        if (unlikely(set_name != NULL)) {
+            tmp = __Pyx_PyObject_Call2Args(set_name, type_obj, key);
+            Py_DECREF(set_name);
+            if (unlikely(tmp == NULL)) {
+                PyErr_Format(PyExc_RuntimeError,
+#if PY_MAJOR_VERSION >= 3
+                    "Error calling __set_name__ on '%.100s' instance %R "
+                    "in '%.100s'",
+                    Py_TYPE(value)->tp_name, key, type->tp_name);
+#else
+                    "Error calling __set_name__ on '%.100s' instance %.100s "
+                    "in '%.100s'",
+                    Py_TYPE(value)->tp_name, PyString_Check(key) ? PyString_AS_STRING(key) : "?", type->tp_name);
+#endif
+                goto bad;
+            } else {
+                Py_DECREF(tmp);
+            }
+        }
+        else if (unlikely(PyErr_Occurred())) {
+            goto bad;
+        }
+    }
+
+    Py_DECREF(names_to_set);
+    return 0;
+bad:
+    Py_XDECREF(names_to_set);
+    return -1;
+}
+
+static PyObject *__Pyx_InitSubclassPEP487(PyObject *type_obj, PyObject *mkw) {
+#if CYTHON_USE_TYPE_SLOTS && !CYTHON_AVOID_BORROWED_REFS
+// Stripped-down version of "super(type_obj, type_obj).__init_subclass__(**mkw)" in CPython 3.8.
+    PyTypeObject *type = (PyTypeObject*) type_obj;
+    PyObject *mro = type->tp_mro;
+    Py_ssize_t i, nbases;
+    if (unlikely(!mro)) goto done;
+
+    // avoid "unused" warning
+    (void) __Pyx_GetBuiltinName;
+
+    Py_INCREF(mro);
+    nbases = PyTuple_GET_SIZE(mro);
+
+    // Skip over the type itself and 'object'.
+    assert(PyTuple_GET_ITEM(mro, 0) == type_obj);
+    for (i = 1; i < nbases-1; i++) {
+        PyObject *base, *dict, *meth;
+        base = PyTuple_GET_ITEM(mro, i);
+        dict = ((PyTypeObject *)base)->tp_dict;
+        meth = __Pyx_PyDict_GetItemStrWithError(dict, PYIDENT("__init_subclass__"));
+        if (unlikely(meth)) {
+            descrgetfunc f = Py_TYPE(meth)->tp_descr_get;
+            PyObject *res;
+            Py_INCREF(meth);
+            if (likely(f)) {
+                res = f(meth, NULL, type_obj);
+                Py_DECREF(meth);
+                if (unlikely(!res)) goto bad;
+                meth = res;
+            }
+            res = __Pyx_PyObject_Call(meth, $empty_tuple, mkw);
+            Py_DECREF(meth);
+            if (unlikely(!res)) goto bad;
+            Py_DECREF(res);
+            goto done;
+        } else if (unlikely(PyErr_Occurred())) {
+            goto bad;
+        }
+    }
+
+done:
+    Py_XDECREF(mro);
+    return type_obj;
+
+bad:
+    Py_XDECREF(mro);
+    Py_DECREF(type_obj);
+    return NULL;
+
+// CYTHON_USE_TYPE_SLOTS && !CYTHON_AVOID_BORROWED_REFS
+#else
+// Generic fallback: "super(type_obj, type_obj).__init_subclass__(**mkw)", as used in CPython 3.8.
+    PyObject *super_type, *super, *func, *res;
+
+#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type)
+    super_type = __Pyx_GetBuiltinName(PYIDENT("super"));
+#else
+    super_type = (PyObject*) &PySuper_Type;
+#endif
+    super = likely(super_type) ? __Pyx_PyObject_Call2Args(super_type, type_obj, type_obj) : NULL;
+#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type)
+    Py_XDECREF(super_type);
+#endif
+    if (unlikely(!super)) {
+        Py_CLEAR(type_obj);
+        goto done;
+    }
+    func = __Pyx_PyObject_GetAttrStrNoError(super, PYIDENT("__init_subclass__"));
+    Py_DECREF(super);
+    if (likely(!func)) {
+        if (unlikely(PyErr_Occurred()))
+            Py_CLEAR(type_obj);
+        goto done;
+    }
+    res = __Pyx_PyObject_Call(func, $empty_tuple, mkw);
+    Py_DECREF(func);
+    if (unlikely(!res))
+        Py_CLEAR(type_obj);
+    Py_XDECREF(res);
+done:
+    return type_obj;
+#endif
+}
+
+// PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS
+#endif
+
 static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases,
                                       PyObject *dict, PyObject *mkw,
                                       int calculate_metaclass, int allow_py2_metaclass) {
@@ -1086,14 +1239,26 @@ static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObj
             return NULL;
         owned_metaclass = metaclass;
     }
+    result = NULL;
     margs = PyTuple_Pack(3, name, bases, dict);
-    if (unlikely(!margs)) {
-        result = NULL;
-    } else {
-        result = PyObject_Call(metaclass, margs, mkw);
+    if (likely(margs)) {
+        // Before PEP-487, type(a,b,c) did not accept any keyword arguments, so guard at least against that case.
+        PyObject *mc_kwargs = (PY_VERSION_HEX >= 0x030600A4) ? mkw : (
+            (metaclass == (PyObject*)&PyType_Type) ? NULL : mkw);
+
+        result = __Pyx_PyObject_Call(metaclass, margs, mc_kwargs);
         Py_DECREF(margs);
     }
     Py_XDECREF(owned_metaclass);
+#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS
+    if (likely(result) && likely(PyType_Check(result))) {
+        if (unlikely(__Pyx_SetNamesPEP487(result) < 0)) {
+            Py_CLEAR(result);
+        } else {
+            result = __Pyx_InitSubclassPEP487(result, mkw);
+        }
+    }
+#endif
     return result;
 }
 
@@ -1355,16 +1520,31 @@ static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) {
     return PyObject_GetAttr(o, n);
 }
 
+
 /////////////// PyObjectLookupSpecial.proto ///////////////
+
+#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
+#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name)  __Pyx__PyObject_LookupSpecial(obj, attr_name, 0)
+#define __Pyx_PyObject_LookupSpecial(obj, attr_name)  __Pyx__PyObject_LookupSpecial(obj, attr_name, 1)
+
+static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); /*proto*/
+
+#else
+#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n)
+#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n)
+#endif
+
+/////////////// PyObjectLookupSpecial ///////////////
 //@requires: PyObjectGetAttrStr
+//@requires: PyObjectGetAttrStrNoError
 
 #if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
-static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) {
+static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) {
     PyObject *res;
     PyTypeObject *tp = Py_TYPE(obj);
 #if PY_MAJOR_VERSION < 3
     if (unlikely(PyInstance_Check(obj)))
-        return __Pyx_PyObject_GetAttrStr(obj, attr_name);
+        return with_error ? __Pyx_PyObject_GetAttrStr(obj, attr_name) : __Pyx_PyObject_GetAttrStrNoError(obj, attr_name);
 #endif
     // adapted from CPython's special_lookup() in ceval.c
     res = _PyType_Lookup(tp, attr_name);
@@ -1375,13 +1555,11 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObj
         } else {
             res = f(res, obj, (PyObject *)tp);
         }
-    } else {
+    } else if (with_error) {
         PyErr_SetObject(PyExc_AttributeError, attr_name);
     }
     return res;
 }
-#else
-#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n)
 #endif
 
 

tests/pypy2_bugs.txt

@@ -23,3 +23,7 @@ run.with_gil
 
 # looks like a "when does the GC run?" issue - slightly surprised it's OK on pypy3
 memoryview.numpy_memoryview
+
+# type features that are disabled in PyPy2:
+#run.test_genericclass
+run.test_subclassinit

tests/run/test_subclassinit.py

+# mode: run
+# tag: pure3.6
+# cython: language_level=3str
+
+import sys
+HAS_NATIVE_SUPPORT = sys.version_info >= (3, 6)
+IS_PY2 = sys.version_info[0] == 2
+
+import re
+import types
+import unittest
+
+ZERO = 0
+
+skip_if_not_native = unittest.skipIf(not HAS_NATIVE_SUPPORT, "currently requires Python 3.6+")
+
+
+class Test(unittest.TestCase):
+    if not hasattr(unittest.TestCase, 'assertRegex'):
+        def assertRegex(self, value, regex):
+            self.assertTrue(re.search(regex, str(value)),
+                            "'%s' did not match '%s'" % (value, regex))
+
+    if not hasattr(unittest.TestCase, 'assertCountEqual'):
+        def assertCountEqual(self, first, second):
+            self.assertEqual(set(first), set(second))
+            self.assertEqual(len(first), len(second))
+
+    def test_init_subclass(self):
+        class A:
+            initialized = False
+
+            def __init_subclass__(cls):
+                if HAS_NATIVE_SUPPORT:
+                    super().__init_subclass__()
+                cls.initialized = True
+
+        class B(A):
+            pass
+
+        self.assertFalse(A.initialized)
+        self.assertTrue(B.initialized)
+
+    def test_init_subclass_dict(self):
+        class A(dict):
+            initialized = False
+
+            def __init_subclass__(cls):
+                if HAS_NATIVE_SUPPORT:
+                    super().__init_subclass__()
+                cls.initialized = True
+
+        class B(A):
+            pass
+
+        self.assertFalse(A.initialized)
+        self.assertTrue(B.initialized)
+
+    def test_init_subclass_kwargs(self):
+        class A:
+            def __init_subclass__(cls, **kwargs):
+                cls.kwargs = kwargs
+
+        class B(A, x=3):
+            pass
+
+        self.assertEqual(B.kwargs, dict(x=3))
+
+    def test_init_subclass_error(self):
+        class A:
+            def __init_subclass__(cls):
+                raise RuntimeError
+
+        with self.assertRaises(RuntimeError):
+            class B(A):
+                pass
+
+    def test_init_subclass_wrong(self):
+        class A:
+            def __init_subclass__(cls, whatever):
+                pass
+
+        with self.assertRaises(TypeError):
+            class B(A):
+                pass
+
+    def test_init_subclass_skipped(self):
+        class BaseWithInit:
+            def __init_subclass__(cls, **kwargs):
+                if HAS_NATIVE_SUPPORT:
+                    super().__init_subclass__(**kwargs)
+                cls.initialized = cls
+
+        class BaseWithoutInit(BaseWithInit):
+            pass
+
+        class A(BaseWithoutInit):
+            pass
+
+        self.assertIs(A.initialized, A)
+        self.assertIs(BaseWithoutInit.initialized, BaseWithoutInit)
+
+    def test_init_subclass_diamond(self):
+        class Base:
+            def __init_subclass__(cls, **kwargs):
+                if HAS_NATIVE_SUPPORT:
+                    super().__init_subclass__(**kwargs)
+                cls.calls = []
+
+        class Left(Base):
+            pass
+
+        class Middle:
+            def __init_subclass__(cls, middle, **kwargs):
+                super().__init_subclass__(**kwargs)
+                cls.calls += [middle]
+
+        class Right(Base):
+            def __init_subclass__(cls, right="right", **kwargs):
+                super().__init_subclass__(**kwargs)
+                cls.calls += [right]
+
+        class A(Left, Middle, Right, middle="middle"):
+            pass
+
+        self.assertEqual(A.calls, ["right", "middle"])
+        self.assertEqual(Left.calls, [])
+        self.assertEqual(Right.calls, [])
+
+    def test_set_name(self):
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                self.owner = owner
+                self.name = name
+
+        class A:
+            d = Descriptor()
+
+        self.assertEqual(A.d.name, "d")
+        self.assertIs(A.d.owner, A)
+
+    @skip_if_not_native
+    def test_set_name_metaclass(self):
+        class Meta(type):
+            def __new__(cls, name, bases, ns):
+                ret = super().__new__(cls, name, bases, ns)
+                self.assertEqual(ret.d.name, "d")
+                self.assertIs(ret.d.owner, ret)
+                return 0
+
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                self.owner = owner
+                self.name = name
+
+        class A(metaclass=Meta):
+            d = Descriptor()
+        self.assertEqual(A, 0)
+
+    def test_set_name_error(self):
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                1 / ZERO
+
+        with self.assertRaises(RuntimeError) as cm:
+            class NotGoingToWork:
+                attr = Descriptor()
+
+        exc = cm.exception
+        self.assertRegex(str(exc), r'\bNotGoingToWork\b')
+        self.assertRegex(str(exc), r'\battr\b')
+        self.assertRegex(str(exc), r'\bDescriptor\b')
+        if HAS_NATIVE_SUPPORT:
+            self.assertIsInstance(exc.__cause__, ZeroDivisionError)
+
+    def test_set_name_wrong(self):
+        class Descriptor:
+            def __set_name__(self):
+                pass
+
+        with self.assertRaises(RuntimeError) as cm:
+            class NotGoingToWork:
+                attr = Descriptor()
+
+        exc = cm.exception
+        self.assertRegex(str(exc), r'\bNotGoingToWork\b')
+        self.assertRegex(str(exc), r'\battr\b')
+        self.assertRegex(str(exc), r'\bDescriptor\b')
+        if HAS_NATIVE_SUPPORT:
+            self.assertIsInstance(exc.__cause__, TypeError)
+
+    def test_set_name_lookup(self):
+        resolved = []
+        class NonDescriptor:
+            def __getattr__(self, name):
+                resolved.append(name)
+
+        class A:
+            d = NonDescriptor()
+
+        self.assertNotIn('__set_name__', resolved,
+                         '__set_name__ is looked up in instance dict')
+
+    @skip_if_not_native
+    def test_set_name_init_subclass(self):
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                self.owner = owner
+                self.name = name
+
+        class Meta(type):
+            def __new__(cls, name, bases, ns):
+                self = super().__new__(cls, name, bases, ns)
+                self.meta_owner = self.owner
+                self.meta_name = self.name
+                return self
+
+        class A:
+            def __init_subclass__(cls):
+                cls.owner = cls.d.owner
+                cls.name = cls.d.name
+
+        class B(A, metaclass=Meta):
+            d = Descriptor()
+
+        self.assertIs(B.owner, B)
+        self.assertEqual(B.name, 'd')
+        self.assertIs(B.meta_owner, B)
+        self.assertEqual(B.name, 'd')
+
+    def test_set_name_modifying_dict(self):
+        notified = []
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                setattr(owner, name + 'x', None)
+                notified.append(name)
+
+        class A:
+            a = Descriptor()
+            b = Descriptor()
+            c = Descriptor()
+            d = Descriptor()
+            e = Descriptor()
+
+        self.assertCountEqual(notified, ['a', 'b', 'c', 'd', 'e'])
+
+    def test_errors(self):
+        class MyMeta(type):
+            pass
+
+        with self.assertRaises(TypeError):
+            class MyClass(metaclass=MyMeta, otherarg=1):
+                pass
+
+        if not IS_PY2:
+            with self.assertRaises(TypeError):
+                types.new_class("MyClass", (object,),
+                                dict(metaclass=MyMeta, otherarg=1))
+            types.prepare_class("MyClass", (object,),
+                                dict(metaclass=MyMeta, otherarg=1))
+
+        class MyMeta(type):
+            def __init__(self, name, bases, namespace, otherarg):
+                super().__init__(name, bases, namespace)
+
+        with self.assertRaises(TypeError):
+            class MyClass(metaclass=MyMeta, otherarg=1):
+                pass
+
+        class MyMeta(type):
+            def __new__(cls, name, bases, namespace, otherarg):
+                return super().__new__(cls, name, bases, namespace)
+
+            def __init__(self, name, bases, namespace, otherarg):
+                super().__init__(name, bases, namespace)
+                self.otherarg = otherarg
+
+        class MyClass(metaclass=MyMeta, otherarg=1):
+            pass
+
+        self.assertEqual(MyClass.otherarg, 1)
+
+    @skip_if_not_native
+    def test_errors_changed_pep487(self):
+        # These tests failed before Python 3.6, PEP 487
+        class MyMeta(type):
+            def __new__(cls, name, bases, namespace):
+                return super().__new__(cls, name=name, bases=bases,
+                                       dict=namespace)
+
+        with self.assertRaises(TypeError):
+            class MyClass(metaclass=MyMeta):
+                pass
+
+        class MyMeta(type):
+            def __new__(cls, name, bases, namespace, otherarg):
+                self = super().__new__(cls, name, bases, namespace)
+                self.otherarg = otherarg
+                return self
+
+        class MyClass(metaclass=MyMeta, otherarg=1):
+            pass
+
+        self.assertEqual(MyClass.otherarg, 1)
+
+    def test_type(self):
+        t = type('NewClass', (object,), {})
+        self.assertIsInstance(t, type)
+        self.assertEqual(t.__name__, 'NewClass')
+
+        with self.assertRaises(TypeError):
+            type(name='NewClass', bases=(object,), dict={})
+
+
+if __name__ == "__main__":
+    unittest.main()