Support "__del__()" to implement "tp_finalize" according to PEP-442 (GH-3804)

Closes https://github.com/cython/cython/issues/3612

Cython/Compiler/ModuleNode.py

@@ -1400,6 +1400,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 if scope:  # could be None if there was an error
                     self.generate_exttype_vtable(scope, code)
                     self.generate_new_function(scope, code, entry)
+                    self.generate_del_function(scope, code)
                     self.generate_dealloc_function(scope, code)
 
                     if scope.needs_gc():
@@ -1628,6 +1629,29 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln(
             "}")
 
+    def generate_del_function(self, scope, code):
+        tp_slot = TypeSlots.get_slot_by_name("tp_finalize", scope.directives)
+        slot_func_cname = scope.mangle_internal("tp_finalize")
+        if tp_slot.slot_code(scope) != slot_func_cname:
+            return  # never used
+
+        entry = scope.lookup_here("__del__")
+        if entry is None or not entry.is_special:
+            return  # nothing to wrap
+        slot_func_cname = scope.mangle_internal("tp_finalize")
+        code.putln("")
+
+        if tp_slot.used_ifdef:
+            code.putln("#if %s" % tp_slot.used_ifdef)
+        code.putln("static void %s(PyObject *o) {" % slot_func_cname)
+        code.putln("PyObject *etype, *eval, *etb;")
+        code.putln("PyErr_Fetch(&etype, &eval, &etb);")
+        code.putln("%s(o);" % entry.func_cname)
+        code.putln("PyErr_Restore(etype, eval, etb);")
+        code.putln("}")
+        if tp_slot.used_ifdef:
+            code.putln("#endif")
+
     def generate_dealloc_function(self, scope, code):
         tp_slot = TypeSlots.ConstructorSlot("tp_dealloc", '__dealloc__')
         slot_func = scope.mangle_internal("tp_dealloc")
@@ -1671,9 +1695,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 "if (unlikely("
                 "(PY_VERSION_HEX >= 0x03080000 || __Pyx_PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE))"
                 " && Py_TYPE(o)->tp_finalize) && %s) {" % finalised_check)
+
+            code.putln("if (Py_TYPE(o)->tp_dealloc == %s) {" % slot_func_cname)
             # if instance was resurrected by finaliser, return
             code.putln("if (PyObject_CallFinalizerFromDealloc(o)) return;")
             code.putln("}")
+            code.putln("}")
             code.putln("#endif")
 
         if needs_gc:

Cython/Compiler/TypeSlots.py

@@ -219,13 +219,17 @@ class SlotDescriptor(object):
     #  py3                           Indicates presence of slot in Python 3
     #  py2                           Indicates presence of slot in Python 2
     #  ifdef                         Full #ifdef string that slot is wrapped in. Using this causes py3, py2 and flags to be ignored.)
+    #  used_ifdef                    Full #ifdef string that the slot value is wrapped in (otherwise it is assigned NULL)
+    #                                Unlike "ifdef" the slot is defined and this just controls if it receives a value
 
     def __init__(self, slot_name, dynamic=False, inherited=False,
-                 py3=True, py2=True, ifdef=None, is_binop=False):
+                 py3=True, py2=True, ifdef=None, is_binop=False,
+                 used_ifdef=None):
         self.slot_name = slot_name
         self.is_initialised_dynamically = dynamic
         self.is_inherited = inherited
         self.ifdef = ifdef
+        self.used_ifdef = used_ifdef
         self.py3 = py3
         self.py2 = py2
         self.is_binop = is_binop
@@ -293,7 +297,13 @@ class SlotDescriptor(object):
                     code.putln("#else")
                     end_pypy_guard = True
 
+        if self.used_ifdef:
+            code.putln("#if %s" % self.used_ifdef)
         code.putln("%s, /*%s*/" % (value, self.slot_name))
+        if self.used_ifdef:
+            code.putln("#else")
+            code.putln("NULL, /*%s*/" % self.slot_name)
+            code.putln("#endif")
 
         if end_pypy_guard:
             code.putln("#endif")
@@ -546,6 +556,9 @@ class TypeFlagsSlot(SlotDescriptor):
             value += "|Py_TPFLAGS_BASETYPE"
         if scope.needs_gc():
             value += "|Py_TPFLAGS_HAVE_GC"
+        entry = scope.lookup("__del__")
+        if entry and entry.is_special:
+            value += "|Py_TPFLAGS_HAVE_FINALIZE"
         return value
 
     def generate_spec(self, scope, code):
@@ -1061,7 +1074,8 @@ class SlotTable(object):
             EmptySlot("tp_weaklist"),
             EmptySlot("tp_del"),
             EmptySlot("tp_version_tag"),
-            EmptySlot("tp_finalize", ifdef="PY_VERSION_HEX >= 0x030400a1"),
+            SyntheticSlot("tp_finalize", ["__del__"], "0", ifdef="PY_VERSION_HEX >= 0x030400a1",
+                          used_ifdef="CYTHON_USE_TP_FINALIZE"),
             EmptySlot("tp_vectorcall", ifdef="PY_VERSION_HEX >= 0x030800b1"),
             EmptySlot("tp_print", ifdef="PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000"),
             # PyPy specific extension - only here to avoid C compiler warnings.
@@ -1078,6 +1092,7 @@ class SlotTable(object):
 
         MethodSlot(initproc, "", "__cinit__", method_name_to_slot)
         MethodSlot(destructor, "", "__dealloc__", method_name_to_slot)
+        MethodSlot(destructor, "", "__del__", method_name_to_slot)
         MethodSlot(objobjargproc, "", "__setitem__", method_name_to_slot)
         MethodSlot(objargproc, "", "__delitem__", method_name_to_slot)
         MethodSlot(ssizessizeobjargproc, "", "__setslice__", method_name_to_slot)

docs/src/userguide/special_methods.rst

@@ -97,8 +97,8 @@ complaining about the signature mismatch.
 
 .. _finalization_method:
 
-Finalization method: :meth:`__dealloc__`
-----------------------------------------
+Finalization methods: :meth:`__dealloc__` and :meth:`__del__`
+-------------------------------------------------------------
 
 The counterpart to the :meth:`__cinit__` method is the :meth:`__dealloc__`
 method, which should perform the inverse of the :meth:`__cinit__` method. Any
@@ -122,7 +122,13 @@ of the superclass will always be called, even if it is overridden.  This is in
 contrast to typical Python behavior where superclass methods will not be
 executed unless they are explicitly called by the subclass.
 
-.. Note:: There is no :meth:`__del__` method for extension types.
+Python 3.4 made it possible for extension types to safely define
+finalizers for objects. When running a Cython module on Python 3.4 and
+higher you can add a :meth:`__del__` method to extension types in
+order to perform Python cleanup operations. When the :meth:`__del__`
+is called the object is still in a valid state (unlike in the case of
+:meth:`__dealloc__`), permitting the use of Python operations
+on its class members. On Python <3.4 :meth:`__del__` will not be called.
 
 .. _arithmetic_methods:
 

runtests.py

@@ -476,6 +476,7 @@ VER_DEP_MODULES = {
                                           ]),
     (3,4): (operator.lt, lambda x: x in ['run.py34_signature',
                                          'run.test_unicode',  # taken from Py3.7, difficult to backport
+                                         'run.pep442_tp_finalize',
                                          ]),
     (3,4,999): (operator.gt, lambda x: x in ['run.initial_file_path',
                                              ]),

tests/pypy_crash_bugs.txt

@@ -8,3 +8,5 @@ memslice
 # gc issue?
 memoryview_in_subclasses
 
+# """Fatal RPython error: NotImplementedError"""
+pep442_tp_finalize

tests/run/pep442_tp_finalize.pyx

+# mode: run
+
+import gc
+
+cdef class nontrivial_del:
+    def __init__(self):
+        print("init")
+
+    def __del__(self):
+        print("del")
+
+def test_del():
+    """
+    >>> test_del()
+    start
+    init
+    del
+    finish
+    """
+    print("start")
+    d = nontrivial_del()
+    d = None
+    gc.collect()
+    print("finish")
+
+
+cdef class del_and_dealloc:
+    def __init__(self):
+        print("init")
+
+    def __del__(self):
+        print("del")
+
+    def __dealloc__(self):
+        print("dealloc")
+
+def test_del_and_dealloc():
+    """
+    >>> test_del_and_dealloc()
+    start
+    init
+    del
+    dealloc
+    finish
+    """
+    print("start")
+    d = del_and_dealloc()
+    d = None
+    gc.collect()
+    print("finish")
+
+
+cdef class del_with_exception:
+    def __init__(self):
+        print("init")
+
+    def __del__(self):
+        print("del")
+        raise Exception("Error")
+
+def test_del_with_exception():
+    """
+    >>> test_del_with_exception()
+    start
+    init
+    del
+    finish
+    """
+    print("start")
+    d = nontrivial_del()
+    d = None
+    gc.collect()
+    print("finish")
+
+
+def test_nontrivial_del_with_exception():
+    """
+    >>> test_nontrivial_del_with_exception()
+    start
+    init
+    del
+    end
+    """
+    print("start")
+    def inner():
+        c = nontrivial_del()
+        raise RuntimeError()
+
+    try:
+        inner()
+    except RuntimeError:
+        pass
+
+    print("end")
+
+
+cdef class parent:
+    def __del__(self):
+        print("del parent")
+
+class child(parent):
+    def __del__(self):
+        print("del child")
+
+def test_del_inheritance():
+    """
+    >>> test_del_inheritance()
+    start
+    del child
+    finish
+    """
+    print("start")
+    c = child()
+    c = None
+    gc.collect()
+    print("finish")
+
+
+cdef class cy_parent:
+    def __del__(self):
+        print("del cy_parent")
+
+    def __dealloc__(self):
+        print("dealloc cy_parent")
+
+class py_parent:
+    def __del__(self):
+        print("del py_parent")
+
+class multi_child(cy_parent, py_parent):
+    def __del__(self):
+        print("del child")
+
+def test_multiple_inheritance():
+    """
+    >>> test_multiple_inheritance()
+    start
+    del child
+    dealloc cy_parent
+    finish
+    """
+    print("start")
+    c = multi_child()
+    c = None
+    gc.collect()
+    print("finish")
+
+
+cdef class zombie_object:
+    def __del__(self):
+        global global_zombie_object
+        print("del")
+        global_zombie_object = self
+
+    def __dealloc__(self):
+        print("dealloc")
+
+def test_zombie_object():
+    """
+    >>> test_zombie_object()
+    start
+    del
+    del global
+    del
+    finish
+    """
+    global global_zombie_object
+    print("start")
+    i = zombie_object()
+    i = None
+    print("del global")
+    del global_zombie_object
+    gc.collect()
+    print("finish")
+
+
+# Same as above, but the member
+# makes the class GC, so it
+# is deallocated
+cdef class gc_zombie_object:
+    cdef object x
+
+    def __del__(self):
+        global global_gc_zombie_object
+        print("del")
+        global_gc_zombie_object = self
+
+    def __dealloc__(self):
+        print("dealloc")
+
+def test_gc_zombie_object():
+    """
+    >>> test_gc_zombie_object()
+    start
+    del
+    del global
+    dealloc
+    finish
+    """
+    global global_gc_zombie_object
+    print("start")
+    i = gc_zombie_object()
+    i = None
+    print("del global")
+    del global_gc_zombie_object
+    gc.collect()
+    print("finish")
+
+
+cdef class cdef_parent:
+    pass
+
+cdef class cdef_child(cdef_parent):
+    def __del__(self):
+        print("del")
+    def __dealloc__(self):
+        print("dealloc")
+
+def test_cdef_parent_object():
+    """
+    >>> test_cdef_parent_object()
+    start
+    del
+    dealloc
+    finish
+    """
+    print("start")
+    i = cdef_child()
+    i = None
+    gc.collect()
+    print("finish")
+
+
+cdef class cdef_nontrivial_parent:
+    def __del__(self):
+        print("del parent")
+    def __dealloc__(self):
+        print("dealloc parent")
+
+cdef class cdef_nontrivial_child(cdef_nontrivial_parent):
+    def __del__(self):
+        print("del child")
+    def __dealloc__(self):
+        print("dealloc child")
+
+def test_cdef_nontrivial_parent_object():
+    """
+    >>> test_cdef_nontrivial_parent_object()
+    start
+    del child
+    dealloc child
+    dealloc parent
+    finish
+    """
+    print("start")
+    i = cdef_nontrivial_child()
+    i = None
+    gc.collect()
+    print("finish")
+
+
+class python_child(cdef_nontrivial_parent):
+    def __del__(self):
+        print("del python child")
+        super().__del__()
+
+def test_python_child_object():
+    """
+    >>> test_python_child_object()
+    Traceback (most recent call last):
+    ...
+    RuntimeError: End function
+    """
+
+    def func(tp):
+        inst = tp()
+        raise RuntimeError("End function")
+
+    func(python_child)
+
+def test_python_child_fancy_inherit():
+    """
+    >>> test_python_child_fancy_inherit()
+    Traceback (most recent call last):
+    ...
+    RuntimeError: End function
+    """
+
+    # inherit using "true python" rather than Cython
+    globs = { 'cdef_nontrivial_parent': cdef_nontrivial_parent }
+
+    exec("""
+class derived_python_child(cdef_nontrivial_parent):
+    pass
+""", globs)
+
+    derived_python_child = globs['derived_python_child']
+
+    def func(tp):
+        inst = tp()
+        raise RuntimeError("End function")
+
+    func(derived_python_child)