to help in breaking potential reference cycles, clear references in generator...

to help in breaking potential reference cycles, clear references in generator objects immediately on termination instead of waiting for them to be cleaned up

Cython/Compiler/Nodes.py

@@ -4182,6 +4182,8 @@ class GeneratorBodyDefNode(DefNode):
         code.put_label(code.return_label)
         code.put_xdecref(Naming.retval_cname, py_object_type)
         code.putln('%s->resume_label = -1;' % Naming.generator_cname)
+        # clean up as early as possible to help breaking any reference cycles
+        code.putln('__Pyx_Generator_clear((PyObject*)%s);' % Naming.generator_cname)
         code.put_finish_refcount_context()
         code.putln('return NULL;')
         code.putln("}")

Cython/Utility/Generator.c

@@ -43,6 +43,7 @@ typedef struct {
 static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
                                                   PyObject *closure);
 static int __pyx_Generator_init(void);
+static int __Pyx_Generator_clear(PyObject* self);
 
 #if 1 || PY_VERSION_HEX < 0x030300B0
 static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue);
@@ -407,6 +408,18 @@ static int __Pyx_Generator_traverse(PyObject *self, visitproc visit, void *arg)
     return 0;
 }
 
+static int __Pyx_Generator_clear(PyObject *self) {
+    __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
+
+    Py_CLEAR(gen->closure);
+    Py_CLEAR(gen->classobj);
+    Py_CLEAR(gen->yieldfrom);
+    Py_CLEAR(gen->exc_type);
+    Py_CLEAR(gen->exc_value);
+    Py_CLEAR(gen->exc_traceback);
+    return 0;
+}
+
 static void __Pyx_Generator_dealloc(PyObject *self) {
     __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
 
@@ -423,12 +436,7 @@ static void __Pyx_Generator_dealloc(PyObject *self) {
     }
 
     PyObject_GC_UnTrack(self);
-    Py_CLEAR(gen->closure);
-    Py_CLEAR(gen->classobj);
-    Py_CLEAR(gen->yieldfrom);
-    Py_CLEAR(gen->exc_type);
-    Py_CLEAR(gen->exc_value);
-    Py_CLEAR(gen->exc_traceback);
+    __Pyx_Generator_clear(self);
     PyObject_GC_Del(gen);
 }
 
@@ -537,7 +545,7 @@ static PyTypeObject __pyx_GeneratorType = {
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags*/
     0,                                  /*tp_doc*/
     (traverseproc) __Pyx_Generator_traverse,   /*tp_traverse*/
-    0,                                  /*tp_clear*/
+    (inquiry) __Pyx_Generator_clear,           /*tp_clear*/
     0,                                  /*tp_richcompare*/
     offsetof(__pyx_GeneratorObject, gi_weakreflist), /* tp_weaklistoffse */
     PyObject_SelfIter,                  /*tp_iter*/

tests/run/generators_in_refcycles.pyx

+
+def test_reference_cycle_cleanup():
+    """
+    >>> import gc
+    >>> delegator, gen, next, deleted = test_reference_cycle_cleanup()
+
+    >>> next(delegator(gen()))
+    123
+    >>> _ = gc.collect(); print(sorted(deleted))
+    ['bar', 'foo']
+    """
+    deleted = []
+    class Destructed(object):
+        def __init__(self, name):
+            self.name = name
+
+        def __del__(self):
+            deleted.append(self.name)
+
+    def delegator(c):
+        d = Destructed('foo')
+        yield from c
+
+    def gen():
+        d = Destructed('bar')
+        while True:
+            yield 123
+
+    return delegator, gen, next, deleted