Merge pull request #73 from vitek/_generators_cleanup

 generators cleanup

Cython/Compiler/ExprNodes.py

@@ -5830,7 +5830,7 @@ class ClassCellNode(ExprNode):
                 Naming.self_cname))
         else:
             code.putln('%s =  %s->classobj;' % (
-                self.result(), Naming.cur_scope_cname))
+                self.result(), Naming.generator_cname))
         code.putln(
             'if (!%s) { PyErr_SetString(PyExc_SystemError, '
             '"super(): empty __class__ cell"); %s }' % (
@@ -6252,7 +6252,8 @@ class YieldExprNode(ExprNode):
         code.put_xgiveref(Naming.retval_cname)
         code.put_finish_refcount_context()
         code.putln("/* return from generator, yielding value */")
-        code.putln("%s->%s.resume_label = %d;" % (Naming.cur_scope_cname, Naming.obj_base_cname, self.label_num))
+        code.putln("%s->resume_label = %d;" % (
+            Naming.generator_cname, self.label_num))
         code.putln("return %s;" % Naming.retval_cname);
         code.put_label(self.label_name)
         for cname, save_cname, type in saved:
@@ -9814,122 +9815,8 @@ cyfunction_class_cell_utility_code = UtilityCode.load(
     "CythonFunction.c",
     requires=[binding_cfunc_utility_code])
 
-generator_utility_code = UtilityCode(
-proto="""
-static PyObject *__Pyx_Generator_Next(PyObject *self);
-static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value);
-static PyObject *__Pyx_Generator_Close(PyObject *self);
-static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args, CYTHON_UNUSED PyObject *kwds);
-
-typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
-""",
-impl="""
-static CYTHON_INLINE void __Pyx_Generator_ExceptionClear(struct __pyx_Generator_object *self)
-{
-    Py_XDECREF(self->exc_type);
-    Py_XDECREF(self->exc_value);
-    Py_XDECREF(self->exc_traceback);
-
-    self->exc_type = NULL;
-    self->exc_value = NULL;
-    self->exc_traceback = NULL;
-}
-
-static CYTHON_INLINE PyObject *__Pyx_Generator_SendEx(struct __pyx_Generator_object *self, PyObject *value)
-{
-    PyObject *retval;
-
-    if (self->is_running) {
-        PyErr_SetString(PyExc_ValueError,
-                        "generator already executing");
-        return NULL;
-    }
-
-    if (self->resume_label == 0) {
-        if (value && value != Py_None) {
-            PyErr_SetString(PyExc_TypeError,
-                            "can't send non-None value to a "
-                            "just-started generator");
-            return NULL;
-        }
-    }
-
-    if (self->resume_label == -1) {
-        PyErr_SetNone(PyExc_StopIteration);
-        return NULL;
-    }
-
-
-    if (value)
-        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
-    else
-        __Pyx_Generator_ExceptionClear(self);
-
-    self->is_running = 1;
-    retval = self->body((PyObject *) self, value);
-    self->is_running = 0;
-
-    if (retval)
-        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
-    else
-        __Pyx_Generator_ExceptionClear(self);
-
-    return retval;
-}
-
-static PyObject *__Pyx_Generator_Next(PyObject *self)
-{
-    return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, Py_None);
-}
-
-static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value)
-{
-    return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, value);
-}
-
-static PyObject *__Pyx_Generator_Close(PyObject *self)
-{
-    struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
-    PyObject *retval;
-#if PY_VERSION_HEX < 0x02050000
-    PyErr_SetNone(PyExc_StopIteration);
-#else
-    PyErr_SetNone(PyExc_GeneratorExit);
-#endif
-    retval = __Pyx_Generator_SendEx(generator, NULL);
-    if (retval) {
-        Py_DECREF(retval);
-        PyErr_SetString(PyExc_RuntimeError,
-                        "generator ignored GeneratorExit");
-        return NULL;
-    }
-#if PY_VERSION_HEX < 0x02050000
-    if (PyErr_ExceptionMatches(PyExc_StopIteration))
-#else
-    if (PyErr_ExceptionMatches(PyExc_StopIteration)
-        || PyErr_ExceptionMatches(PyExc_GeneratorExit))
-#endif
-    {
-        PyErr_Clear();          /* ignore these errors */
-        Py_INCREF(Py_None);
-        return Py_None;
-    }
-    return NULL;
-}
-
-static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args, CYTHON_UNUSED PyObject *kwds)
-{
-    struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
-    PyObject *typ;
-    PyObject *tb = NULL;
-    PyObject *val = NULL;
-
-    if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
-        return NULL;
-    __Pyx_Raise(typ, val, tb, NULL);
-    return __Pyx_Generator_SendEx(generator, NULL);
-}
-""",
-proto_block='utility_code_proto_before_types',
-requires=[Nodes.raise_utility_code, Nodes.swap_exception_utility_code],
+generator_utility_code = UtilityCode.load(
+    "Generator",
+    "Generator.c",
+    requires=[Nodes.raise_utility_code, Nodes.swap_exception_utility_code],
 )

Cython/Compiler/ModuleNode.py

@@ -1767,6 +1767,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("if (__pyx_FusedFunction_init() < 0) %s" % code.error_goto(self.pos))
         code.putln("#endif")
 
+        code.putln("#ifdef __Pyx_Generator_USED")
+        code.putln("if (__pyx_Generator_init() < 0) %s" % code.error_goto(self.pos))
+        code.putln("#endif")
+
         code.putln("/*--- Library function declarations ---*/")
         env.generate_library_function_declarations(code)
 

Cython/Compiler/Naming.py

@@ -51,6 +51,7 @@ lambda_func_prefix = pyrex_prefix + "lambda_"
 module_is_main   = pyrex_prefix + "module_is_main_"
 
 args_cname       = pyrex_prefix + "args"
+generator_cname  = pyrex_prefix + "generator"
 sent_value_cname = pyrex_prefix + "sent_value"
 pykwdlist_cname  = pyrex_prefix + "pyargnames"
 obj_base_cname   = pyrex_prefix + "base"

Cython/Compiler/Nodes.py

@@ -3777,19 +3777,22 @@ class GeneratorDefNode(DefNode):
 
     def generate_function_body(self, env, code):
         body_cname = self.gbody.entry.func_cname
-        generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
 
-        code.putln('%s.resume_label = 0;' % generator_cname)
-        code.putln('%s.body = (__pyx_generator_body_t) %s;' % (generator_cname, body_cname))
-        code.put_giveref(Naming.cur_scope_cname)
+        code.putln('{')
+        code.putln('__pyx_GeneratorObject *gen = __Pyx_Generator_New('
+                   '(__pyx_generator_body_t) %s, (PyObject *) %s); %s' % (
+                       body_cname, Naming.cur_scope_cname,
+                       code.error_goto_if_null('gen', self.pos)))
+        code.put_decref(Naming.cur_scope_cname, py_object_type)
         if self.requires_classobj:
-            classobj_cname = '%s->classobj' % Naming.cur_scope_cname
+            classobj_cname = 'gen->classobj'
             code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                 classobj_cname, Naming.self_cname))
             code.put_incref(classobj_cname, py_object_type)
             code.put_giveref(classobj_cname)
         code.put_finish_refcount_context()
-        code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
+        code.putln('return (PyObject *) gen;');
+        code.putln('}')
 
     def generate_function_definitions(self, env, code):
         from ExprNodes import generator_utility_code
@@ -3807,9 +3810,9 @@ class GeneratorBodyDefNode(DefNode):
     is_generator_body = True
 
     def __init__(self, pos=None, name=None, body=None):
-        super(GeneratorBodyDefNode, self).__init__(pos=pos, body=body, name=name, doc=None,
-                                                   args=[],
-                                                   star_arg=None, starstar_arg=None)
+        super(GeneratorBodyDefNode, self).__init__(
+            pos=pos, body=body, name=name, doc=None,
+            args=[], star_arg=None, starstar_arg=None)
 
     def declare_generator_body(self, env):
         prefix = env.next_id(env.scope_prefix)
@@ -3826,9 +3829,9 @@ class GeneratorBodyDefNode(DefNode):
         self.declare_generator_body(env)
 
     def generate_function_header(self, code, proto=False):
-        header = "static PyObject *%s(%s, PyObject *%s)" % (
+        header = "static PyObject *%s(__pyx_GeneratorObject *%s, PyObject *%s)" % (
             self.entry.func_cname,
-            self.local_scope.scope_class.type.declaration_code(Naming.cur_scope_cname),
+            Naming.generator_cname,
             Naming.sent_value_cname)
         if proto:
             code.putln('%s; /* proto */' % header)
@@ -3851,6 +3854,7 @@ class GeneratorBodyDefNode(DefNode):
         # ----- Function header
         code.putln("")
         self.generate_function_header(code)
+        closure_init_code = code.insertion_point()
         # ----- Local variables
         code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
         tempvardecl_code = code.insertion_point()
@@ -3868,6 +3872,12 @@ class GeneratorBodyDefNode(DefNode):
 
         # ----- Function body
         self.generate_function_body(env, code)
+        # ----- Closure initialization
+        if lenv.scope_class.type.scope.entries:
+            closure_init_code.putln('%s = %s;' % (
+                lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
+                lenv.scope_class.type.cast_code('%s->closure' %
+                                                Naming.generator_cname)))
         code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
         # ----- Error cleanup
         if code.error_label in code.labels_used:
@@ -3880,7 +3890,7 @@ class GeneratorBodyDefNode(DefNode):
         # ----- Non-error return cleanup
         code.put_label(code.return_label)
         code.put_xdecref(Naming.retval_cname, py_object_type)
-        code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
+        code.putln('%s->resume_label = -1;' % Naming.generator_cname)
         code.put_finish_refcount_context()
         code.putln('return NULL;');
         code.putln("}")
@@ -3888,14 +3898,16 @@ class GeneratorBodyDefNode(DefNode):
         # ----- Go back and insert temp variable declarations
         tempvardecl_code.put_temp_declarations(code.funcstate)
         # ----- Generator resume code
-        resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
+        resume_code.putln("switch (%s->resume_label) {" % (
+                       Naming.generator_cname));
         resume_code.putln("case 0: goto %s;" % first_run_label)
 
         from ParseTreeTransforms import YieldNodeCollector
         collector = YieldNodeCollector()
         collector.visitchildren(self)
         for yield_expr in collector.yields:
-            resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
+            resume_code.putln("case %d: goto %s;" % (
+                yield_expr.label_num, yield_expr.label_name));
         resume_code.putln("default: /* CPython raises the right error here */");
         resume_code.put_finish_refcount_context()
         resume_code.putln("return NULL;");
@@ -8123,7 +8135,8 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject
 """,
 impl = """
 #if PY_MAJOR_VERSION < 3
-static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) {
+static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb,
+                        CYTHON_UNUSED PyObject *cause) {
     /* cause is unused */
     Py_XINCREF(type);
     Py_XINCREF(value);

Cython/Compiler/ParseTreeTransforms.py

@@ -2043,63 +2043,12 @@ class CreateClosureClasses(CythonTransform):
         super(CreateClosureClasses, self).__init__(context)
         self.path = []
         self.in_lambda = False
-        self.generator_class = None
 
     def visit_ModuleNode(self, node):
         self.module_scope = node.scope
         self.visitchildren(node)
         return node
 
-    def create_generator_class(self, target_module_scope, pos):
-        if self.generator_class:
-            return self.generator_class
-        # XXX: make generator class creation cleaner
-        entry = target_module_scope.declare_c_class(name='__pyx_Generator',
-                    objstruct_cname='__pyx_Generator_object',
-                    typeobj_cname='__pyx_Generator_type',
-                    pos=pos, defining=True, implementing=True)
-        entry.type.is_final_type = True
-        klass = entry.type.scope
-        klass.is_internal = True
-
-        body_type = PyrexTypes.create_typedef_type('generator_body',
-                                                   PyrexTypes.c_void_ptr_type,
-                                                   '__pyx_generator_body_t')
-        klass.declare_var(pos=pos, name='body', cname='body',
-                          type=body_type, is_cdef=True)
-        klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
-                          is_cdef=True)
-        klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
-                          is_cdef=True)
-        klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
-                          type=PyrexTypes.py_object_type, is_cdef=True)
-        klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
-                          type=PyrexTypes.py_object_type, is_cdef=True)
-        klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
-                          type=PyrexTypes.py_object_type, is_cdef=True)
-
-        import TypeSlots
-        e = klass.declare_pyfunction('send', pos)
-        e.func_cname = '__Pyx_Generator_Send'
-        e.signature = TypeSlots.binaryfunc
-
-        e = klass.declare_pyfunction('close', pos)
-        e.func_cname = '__Pyx_Generator_Close'
-        e.signature = TypeSlots.unaryfunc
-
-        e = klass.declare_pyfunction('throw', pos)
-        e.func_cname = '__Pyx_Generator_Throw'
-        e.signature = TypeSlots.pyfunction_signature
-
-        e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
-        e.func_cname = 'PyObject_SelfIter'
-
-        e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
-        e.func_cname = '__Pyx_Generator_Next'
-
-        self.generator_class = entry.type
-        return self.generator_class
-
     def find_entries_used_in_closures(self, node):
         from_closure = []
         in_closure = []
@@ -2140,9 +2089,8 @@ class CreateClosureClasses(CythonTransform):
             inner_node.needs_self_code = False
             node.needs_outer_scope = False
 
-        base_type = None
         if node.is_generator:
-            base_type = self.create_generator_class(target_module_scope, node.pos)
+            pass
         elif not in_closure and not from_closure:
             return
         elif not in_closure:
@@ -2151,23 +2099,19 @@ class CreateClosureClasses(CythonTransform):
             node.needs_outer_scope = True
             return
 
-        as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
+        as_name = '%s_%s' % (
+            target_module_scope.next_id(Naming.closure_class_prefix),
+            node.entry.cname)
 
         entry = target_module_scope.declare_c_class(
             name=as_name, pos=node.pos, defining=True,
-            implementing=True, base_type=base_type)
+            implementing=True)
         entry.type.is_final_type = True
 
         func_scope.scope_class = entry
         class_scope = entry.type.scope
         class_scope.is_internal = True
 
-        if node.is_generator and node.requires_classobj:
-            class_scope.declare_var(pos=node.pos, name='classobj',
-                                    cname='classobj',
-                                    type=PyrexTypes.py_object_type,
-                                    is_cdef=True)
-
         if from_closure:
             assert cscope.is_closure_scope
             class_scope.declare_var(pos=node.pos,

Cython/Utility/Generator.c

+//////////////////// Generator.proto ////////////////////
+#define __Pyx_Generator_USED
+#include <structmember.h>
+
+typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
+
+typedef struct {
+    PyObject_HEAD
+    __pyx_generator_body_t body;
+    PyObject *closure;
+    int is_running;
+    int resume_label;
+    PyObject *exc_type;
+    PyObject *exc_value;
+    PyObject *exc_traceback;
+    PyObject *gi_weakreflist;
+    PyObject *classobj;
+} __pyx_GeneratorObject;
+
+static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
+                                                  PyObject *closure);
+static int __pyx_Generator_init(void);
+
+//////////////////// Generator ////////////////////
+static PyObject *__Pyx_Generator_Next(PyObject *self);
+static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value);
+static PyObject *__Pyx_Generator_Close(PyObject *self);
+static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args);
+
+static CYTHON_INLINE
+void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self)
+{
+    Py_XDECREF(self->exc_type);
+    Py_XDECREF(self->exc_value);
+    Py_XDECREF(self->exc_traceback);
+
+    self->exc_type = NULL;
+    self->exc_value = NULL;
+    self->exc_traceback = NULL;
+}
+
+static CYTHON_INLINE
+PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value)
+{
+    PyObject *retval;
+
+    if (self->is_running) {
+        PyErr_SetString(PyExc_ValueError,
+                        "generator already executing");
+        return NULL;
+    }
+
+    if (self->resume_label == 0) {
+        if (value && value != Py_None) {
+            PyErr_SetString(PyExc_TypeError,
+                            "can't send non-None value to a "
+                            "just-started generator");
+            return NULL;
+        }
+    }
+
+    if (self->resume_label == -1) {
+        PyErr_SetNone(PyExc_StopIteration);
+        return NULL;
+    }
+
+
+    if (value)
+        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
+    else
+        __Pyx_Generator_ExceptionClear(self);
+
+    self->is_running = 1;
+    retval = self->body((PyObject *) self, value);
+    self->is_running = 0;
+
+    if (retval)
+        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
+    else
+        __Pyx_Generator_ExceptionClear(self);
+
+    return retval;
+}
+
+static PyObject *__Pyx_Generator_Next(PyObject *self)
+{
+    return __Pyx_Generator_SendEx((__pyx_GeneratorObject *) self, Py_None);
+}
+
+static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value)
+{
+    return __Pyx_Generator_SendEx((__pyx_GeneratorObject *) self, value);
+}
+
+static PyObject *__Pyx_Generator_Close(PyObject *self)
+{
+    __pyx_GeneratorObject *generator = (__pyx_GeneratorObject *) self;
+    PyObject *retval;
+#if PY_VERSION_HEX < 0x02050000
+    PyErr_SetNone(PyExc_StopIteration);
+#else
+    PyErr_SetNone(PyExc_GeneratorExit);
+#endif
+    retval = __Pyx_Generator_SendEx(generator, NULL);
+    if (retval) {
+        Py_DECREF(retval);
+        PyErr_SetString(PyExc_RuntimeError,
+                        "generator ignored GeneratorExit");
+        return NULL;
+    }
+#if PY_VERSION_HEX < 0x02050000
+    if (PyErr_ExceptionMatches(PyExc_StopIteration))
+#else
+    if (PyErr_ExceptionMatches(PyExc_StopIteration)
+        || PyErr_ExceptionMatches(PyExc_GeneratorExit))
+#endif
+    {
+        PyErr_Clear();          /* ignore these errors */
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
+    return NULL;
+}
+
+static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args)
+{
+    __pyx_GeneratorObject *generator = (__pyx_GeneratorObject *) self;
+    PyObject *typ;
+    PyObject *tb = NULL;
+    PyObject *val = NULL;
+
+    if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
+        return NULL;
+    __Pyx_Raise(typ, val, tb, NULL);
+    return __Pyx_Generator_SendEx(generator, NULL);
+}
+
+static int
+__Pyx_Generator_traverse(PyObject *self, visitproc visit, void *arg)
+{
+    __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
+
+    Py_VISIT(gen->closure);
+    Py_VISIT(gen->classobj);
+    Py_VISIT(gen->exc_type);
+    Py_VISIT(gen->exc_value);
+    Py_VISIT(gen->exc_traceback);
+    return 0;
+}
+
+static void
+__Pyx_Generator_dealloc(PyObject *self)
+{
+    __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
+
+    PyObject_GC_UnTrack(gen);
+    if (gen->gi_weakreflist != NULL)
+        PyObject_ClearWeakRefs(self);
+    PyObject_GC_Track(self);
+
+    if (gen->resume_label > 0) {
+        /* Generator is paused, so we need to close */
+        Py_TYPE(gen)->tp_del(self);
+        if (self->ob_refcnt > 0)
+            return;                     /* resurrected.  :( */
+    }
+
+    PyObject_GC_UnTrack(self);
+    Py_CLEAR(gen->closure);
+    Py_CLEAR(gen->classobj);
+    Py_CLEAR(gen->exc_type);
+    Py_CLEAR(gen->exc_value);
+    Py_CLEAR(gen->exc_traceback);
+    PyObject_GC_Del(gen);
+}
+
+static void
+__Pyx_Generator_del(PyObject *self)
+{
+    PyObject *res;
+    PyObject *error_type, *error_value, *error_traceback;
+    __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
+
+    if (gen->resume_label <= 0)
+        return ;
+
+    /* Temporarily resurrect the object. */
+    assert(self->ob_refcnt == 0);
+    self->ob_refcnt = 1;
+
+    /* Save the current exception, if any. */
+    __Pyx_ErrFetch(&error_type, &error_value, &error_traceback);
+
+    res = __Pyx_Generator_Close(self);
+
+    if (res == NULL)
+        PyErr_WriteUnraisable(self);
+    else
+        Py_DECREF(res);
+
+    /* Restore the saved exception. */
+    __Pyx_ErrRestore(error_type, error_value, error_traceback);
+
+    /* Undo the temporary resurrection; can't use DECREF here, it would
+     * cause a recursive call.
+     */
+    assert(self->ob_refcnt > 0);
+    if (--self->ob_refcnt == 0)
+        return; /* this is the normal path out */
+
+    /* close() resurrected it!  Make it look like the original Py_DECREF
+     * never happened.
+     */
+    {
+        Py_ssize_t refcnt = self->ob_refcnt;
+        _Py_NewReference(self);
+        self->ob_refcnt = refcnt;
+    }
+    assert(PyType_IS_GC(self->ob_type) &&
+           _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED);
+
+    /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
+     * we need to undo that. */
+    _Py_DEC_REFTOTAL;
+    /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
+     * chain, so no more to do there.
+     * If COUNT_ALLOCS, the original decref bumped tp_frees, and
+     * _Py_NewReference bumped tp_allocs:  both of those need to be
+     * undone.
+     */
+#ifdef COUNT_ALLOCS
+    --self->ob_type->tp_frees;
+    --self->ob_type->tp_allocs;
+#endif
+}
+
+static PyMemberDef __pyx_Generator_memberlist[] = {
+    {(char *) "gi_running",
+     T_INT,
+     offsetof(__pyx_GeneratorObject, is_running),
+     READONLY,
+     NULL},
+};
+
+static PyMethodDef __pyx_Generator_methods[] = {
+    {__Pyx_NAMESTR("send"), (PyCFunction) __Pyx_Generator_Send, METH_O, 0},
+    {__Pyx_NAMESTR("throw"), (PyCFunction) __Pyx_Generator_Throw, METH_VARARGS, 0},
+    {__Pyx_NAMESTR("close"), (PyCFunction) __Pyx_Generator_Close, METH_NOARGS, 0},
+    {0, 0, 0, 0}
+};
+
+static PyTypeObject __pyx_GeneratorType = {
+    PyVarObject_HEAD_INIT(0, 0)
+    __Pyx_NAMESTR("generator"),         /*tp_name*/
+    sizeof(__pyx_GeneratorObject),      /*tp_basicsize*/
+    0,                                  /*tp_itemsize*/
+    (destructor) __Pyx_Generator_dealloc,/*tp_dealloc*/
+    0,                                  /*tp_print*/
+    0,                                  /*tp_getattr*/
+    0,                                  /*tp_setattr*/
+#if PY_MAJOR_VERSION < 3
+    0,                                  /*tp_compare*/
+#else
+    0,                                  /*reserved*/
+#endif
+    0,                                   /*tp_repr*/
+    0,                                  /*tp_as_number*/
+    0,                                  /*tp_as_sequence*/
+    0,                                  /*tp_as_mapping*/
+    0,                                  /*tp_hash*/
+    0,                                  /*tp_call*/
+    0,                                  /*tp_str*/
+    PyObject_GenericGetAttr,            /*tp_getattro*/
+    0,                                  /*tp_setattro*/
+    0,                                  /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags*/
+    0,                                  /*tp_doc*/
+    (traverseproc) __Pyx_Generator_traverse,   /*tp_traverse*/
+    0,                                  /*tp_clear*/
+    0,                                  /*tp_richcompare*/
+    offsetof(__pyx_GeneratorObject, gi_weakreflist), /* tp_weaklistoffse */
+    PyObject_SelfIter,                  /*tp_iter*/
+    (iternextfunc) __Pyx_Generator_Next, /*tp_iternext*/
+    __pyx_Generator_methods,            /*tp_methods*/
+    __pyx_Generator_memberlist,         /*tp_members*/
+    0,                                  /*tp_getset*/
+    0,                                  /*tp_base*/
+    0,                                  /*tp_dict*/
+    0,                                  /*tp_descr_get*/
+    0,                                  /*tp_descr_set*/
+    0,                                  /*tp_dictoffset*/
+    0,                                  /*tp_init*/
+    0,                                  /*tp_alloc*/
+    0,                                  /*tp_new*/
+    0,                                  /*tp_free*/
+    0,                                  /*tp_is_gc*/
+    0,                                  /*tp_bases*/
+    0,                                  /*tp_mro*/
+    0,                                  /*tp_cache*/
+    0,                                  /*tp_subclasses*/
+    0,                                  /*tp_weaklist*/
+    __Pyx_Generator_del,                /*tp_del*/
+#if PY_VERSION_HEX >= 0x02060000
+    0,                                  /*tp_version_tag*/
+#endif
+};
+
+static
+__pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
+                                           PyObject *closure)
+{
+    __pyx_GeneratorObject *gen =
+        PyObject_GC_New(__pyx_GeneratorObject, &__pyx_GeneratorType);
+
+    if (gen == NULL)
+        return NULL;
+
+    gen->body = body;
+    gen->closure = closure;
+    Py_XINCREF(closure);
+    gen->is_running = 0;
+    gen->resume_label = 0;
+    gen->classobj = NULL;
+    gen->exc_type = NULL;
+    gen->exc_value = NULL;
+    gen->exc_traceback = NULL;
+    gen->gi_weakreflist = NULL;
+
+    PyObject_GC_Track(gen);
+    return gen;
+}
+
+static int __pyx_Generator_init(void)
+{
+    return PyType_Ready(&__pyx_GeneratorType);
+}

tests/run/generators_py.py

@@ -320,3 +320,18 @@ def test_lambda(n):
     """
     for i in range(n):
         yield lambda : i
+
+def test_generator_cleanup():
+    """
+    >>> g = test_generator_cleanup()
+    >>> del g
+    >>> g = test_generator_cleanup()
+    >>> next(g)
+    1
+    >>> del g
+    cleanup
+    """
+    try:
+        yield 1
+    finally:
+        print('cleanup')