Merge pull request #73 from vitek/_generators_cleanup

 generators cleanup

Cython/Compiler/ExprNodes.py

@@ -5830,7 +5830,7 @@ class ClassCellNode(ExprNode):
                 Naming.self_cname))
         else:
             code.putln('%s =  %s->classobj;' % (
-                self.result(), Naming.cur_scope_cname))
+                self.result(), Naming.generator_cname))
         code.putln(
             'if (!%s) { PyErr_SetString(PyExc_SystemError, '
             '"super(): empty __class__ cell"); %s }' % (
@@ -6252,7 +6252,8 @@ class YieldExprNode(ExprNode):
         code.put_xgiveref(Naming.retval_cname)
         code.put_finish_refcount_context()
         code.putln("/* return from generator, yielding value */")
-        code.putln("%s->%s.resume_label = %d;" % (Naming.cur_scope_cname, Naming.obj_base_cname, self.label_num))
+        code.putln("%s->resume_label = %d;" % (
+            Naming.generator_cname, self.label_num))
         code.putln("return %s;" % Naming.retval_cname);
         code.put_label(self.label_name)
         for cname, save_cname, type in saved:
@@ -9814,122 +9815,8 @@ cyfunction_class_cell_utility_code = UtilityCode.load(
     "CythonFunction.c",
     requires=[binding_cfunc_utility_code])
 
-generator_utility_code = UtilityCode(
-proto="""
-static PyObject *__Pyx_Generator_Next(PyObject *self);
-static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value);
-static PyObject *__Pyx_Generator_Close(PyObject *self);
-static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args, CYTHON_UNUSED PyObject *kwds);
-
-typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
-""",
-impl="""
-static CYTHON_INLINE void __Pyx_Generator_ExceptionClear(struct __pyx_Generator_object *self)
-{
-    Py_XDECREF(self->exc_type);
-    Py_XDECREF(self->exc_value);
-    Py_XDECREF(self->exc_traceback);
-
-    self->exc_type = NULL;
-    self->exc_value = NULL;
-    self->exc_traceback = NULL;
-}
-
-static CYTHON_INLINE PyObject *__Pyx_Generator_SendEx(struct __pyx_Generator_object *self, PyObject *value)
-{
-    PyObject *retval;
-
-    if (self->is_running) {
-        PyErr_SetString(PyExc_ValueError,
-                        "generator already executing");
-        return NULL;
-    }
-
-    if (self->resume_label == 0) {
-        if (value && value != Py_None) {
-            PyErr_SetString(PyExc_TypeError,
-                            "can't send non-None value to a "
-                            "just-started generator");
-            return NULL;
-        }
-    }
-
-    if (self->resume_label == -1) {
-        PyErr_SetNone(PyExc_StopIteration);
-        return NULL;
-    }
-
-
-    if (value)
-        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
-    else
-        __Pyx_Generator_ExceptionClear(self);
-
-    self->is_running = 1;
-    retval = self->body((PyObject *) self, value);
-    self->is_running = 0;
-
-    if (retval)
-        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
-    else
-        __Pyx_Generator_ExceptionClear(self);
-
-    return retval;
-}
-
-static PyObject *__Pyx_Generator_Next(PyObject *self)
-{
-    return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, Py_None);
-}
-
-static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value)
-{
-    return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, value);
-}
-
-static PyObject *__Pyx_Generator_Close(PyObject *self)
-{
-    struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
-    PyObject *retval;
-#if PY_VERSION_HEX < 0x02050000
-    PyErr_SetNone(PyExc_StopIteration);
-#else
-    PyErr_SetNone(PyExc_GeneratorExit);
-#endif
-    retval = __Pyx_Generator_SendEx(generator, NULL);
-    if (retval) {
-        Py_DECREF(retval);
-        PyErr_SetString(PyExc_RuntimeError,
-                        "generator ignored GeneratorExit");
-        return NULL;
-    }
-#if PY_VERSION_HEX < 0x02050000
-    if (PyErr_ExceptionMatches(PyExc_StopIteration))
-#else
-    if (PyErr_ExceptionMatches(PyExc_StopIteration)
-        || PyErr_ExceptionMatches(PyExc_GeneratorExit))
-#endif
-    {
-        PyErr_Clear();          /* ignore these errors */
-        Py_INCREF(Py_None);
-        return Py_None;
-    }
-    return NULL;
-}
-
-static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args, CYTHON_UNUSED PyObject *kwds)
-{
-    struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
-    PyObject *typ;
-    PyObject *tb = NULL;
-    PyObject *val = NULL;
-
-    if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
-        return NULL;
-    __Pyx_Raise(typ, val, tb, NULL);
-    return __Pyx_Generator_SendEx(generator, NULL);
-}
-""",
-proto_block='utility_code_proto_before_types',
-requires=[Nodes.raise_utility_code, Nodes.swap_exception_utility_code],
+generator_utility_code = UtilityCode.load(
+    "Generator",
+    "Generator.c",
+    requires=[Nodes.raise_utility_code, Nodes.swap_exception_utility_code],
 )

Cython/Compiler/ModuleNode.py

@@ -1767,6 +1767,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("if (__pyx_FusedFunction_init() < 0) %s" % code.error_goto(self.pos))
         code.putln("#endif")
 
+        code.putln("#ifdef __Pyx_Generator_USED")
+        code.putln("if (__pyx_Generator_init() < 0) %s" % code.error_goto(self.pos))
+        code.putln("#endif")
+
         code.putln("/*--- Library function declarations ---*/")
         env.generate_library_function_declarations(code)
 

Cython/Compiler/Naming.py

@@ -51,6 +51,7 @@ lambda_func_prefix = pyrex_prefix + "lambda_"
 module_is_main   = pyrex_prefix + "module_is_main_"
 
 args_cname       = pyrex_prefix + "args"
+generator_cname  = pyrex_prefix + "generator"
 sent_value_cname = pyrex_prefix + "sent_value"
 pykwdlist_cname  = pyrex_prefix + "pyargnames"
 obj_base_cname   = pyrex_prefix + "base"

Cython/Compiler/Nodes.py

@@ -3777,19 +3777,22 @@ class GeneratorDefNode(DefNode):
 
     def generate_function_body(self, env, code):
         body_cname = self.gbody.entry.func_cname
-        generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
 
-        code.putln('%s.resume_label = 0;' % generator_cname)
-        code.putln('%s.body = (__pyx_generator_body_t) %s;' % (generator_cname, body_cname))
-        code.put_giveref(Naming.cur_scope_cname)
+        code.putln('{')
+        code.putln('__pyx_GeneratorObject *gen = __Pyx_Generator_New('
+                   '(__pyx_generator_body_t) %s, (PyObject *) %s); %s' % (
+                       body_cname, Naming.cur_scope_cname,
+                       code.error_goto_if_null('gen', self.pos)))
+        code.put_decref(Naming.cur_scope_cname, py_object_type)
         if self.requires_classobj:
-            classobj_cname = '%s->classobj' % Naming.cur_scope_cname
+            classobj_cname = 'gen->classobj'
             code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                 classobj_cname, Naming.self_cname))
             code.put_incref(classobj_cname, py_object_type)
             code.put_giveref(classobj_cname)
         code.put_finish_refcount_context()
-        code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
+        code.putln('return (PyObject *) gen;');
+        code.putln('}')
 
     def generate_function_definitions(self, env, code):
         from ExprNodes import generator_utility_code
@@ -3807,9 +3810,9 @@ class GeneratorBodyDefNode(DefNode):
     is_generator_body = True
 
     def __init__(self, pos=None, name=None, body=None):
-        super(GeneratorBodyDefNode, self).__init__(pos=pos, body=body, name=name, doc=None,
-                                                   args=[],
-                                                   star_arg=None, starstar_arg=None)
+        super(GeneratorBodyDefNode, self).__init__(
+            pos=pos, body=body, name=name, doc=None,
+            args=[], star_arg=None, starstar_arg=None)
 
     def declare_generator_body(self, env):
         prefix = env.next_id(env.scope_prefix)
@@ -3826,9 +3829,9 @@ class GeneratorBodyDefNode(DefNode):
         self.declare_generator_body(env)
 
     def generate_function_header(self, code, proto=False):
-        header = "static PyObject *%s(%s, PyObject *%s)" % (
+        header = "static PyObject *%s(__pyx_GeneratorObject *%s, PyObject *%s)" % (
             self.entry.func_cname,
-            self.local_scope.scope_class.type.declaration_code(Naming.cur_scope_cname),
+            Naming.generator_cname,
             Naming.sent_value_cname)
         if proto:
             code.putln('%s; /* proto */' % header)
@@ -3851,6 +3854,7 @@ class GeneratorBodyDefNode(DefNode):
         # ----- Function header
         code.putln("")
         self.generate_function_header(code)
+        closure_init_code = code.insertion_point()
         # ----- Local variables
         code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
         tempvardecl_code = code.insertion_point()
@@ -3868,6 +3872,12 @@ class GeneratorBodyDefNode(DefNode):
 
         # ----- Function body
         self.generate_function_body(env, code)
+        # ----- Closure initialization
+        if lenv.scope_class.type.scope.entries:
+            closure_init_code.putln('%s = %s;' % (
+                lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
+                lenv.scope_class.type.cast_code('%s->closure' %
+                                                Naming.generator_cname)))
         code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
         # ----- Error cleanup
         if code.error_label in code.labels_used:
@@ -3880,7 +3890,7 @@ class GeneratorBodyDefNode(DefNode):
         # ----- Non-error return cleanup
         code.put_label(code.return_label)
         code.put_xdecref(Naming.retval_cname, py_object_type)
-        code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
+        code.putln('%s->resume_label = -1;' % Naming.generator_cname)
         code.put_finish_refcount_context()
         code.putln('return NULL;');
         code.putln("}")
@@ -3888,14 +3898,16 @@ class GeneratorBodyDefNode(DefNode):
         # ----- Go back and insert temp variable declarations
         tempvardecl_code.put_temp_declarations(code.funcstate)
         # ----- Generator resume code
-        resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
+        resume_code.putln("switch (%s->resume_label) {" % (
+                       Naming.generator_cname));
         resume_code.putln("case 0: goto %s;" % first_run_label)
 
         from ParseTreeTransforms import YieldNodeCollector
         collector = YieldNodeCollector()
         collector.visitchildren(self)
         for yield_expr in collector.yields:
-            resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
+            resume_code.putln("case %d: goto %s;" % (
+                yield_expr.label_num, yield_expr.label_name));
         resume_code.putln("default: /* CPython raises the right error here */");
         resume_code.put_finish_refcount_context()
         resume_code.putln("return NULL;");
@@ -8123,7 +8135,8 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject
 """,
 impl = """
 #if PY_MAJOR_VERSION < 3
-static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) {
+static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb,
+                        CYTHON_UNUSED PyObject *cause) {
     /* cause is unused */
     Py_XINCREF(type);
     Py_XINCREF(value);

Cython/Compiler/ParseTreeTransforms.py

@@ -2043,63 +2043,12 @@ class CreateClosureClasses(CythonTransform):
         super(CreateClosureClasses, self).__init__(context)
         self.path = []
         self.in_lambda = False
-        self.generator_class = None
 
     def visit_ModuleNode(self, node):
         self.module_scope = node.scope
         self.visitchildren(node)
         return node
 
-    def create_generator_class(self, target_module_scope, pos):
-        if self.generator_class:
-            return self.generator_class
-        # XXX: make generator class creation cleaner
-        entry = target_module_scope.declare_c_class(name='__pyx_Generator',
-                    objstruct_cname='__pyx_Generator_object',
-                    typeobj_cname='__pyx_Generator_type',
-                    pos=pos, defining=True, implementing=True)
-        entry.type.is_final_type = True
-        klass = entry.type.scope
-        klass.is_internal = True
-
-        body_type = PyrexTypes.create_typedef_type('generator_body',
-                                                   PyrexTypes.c_void_ptr_type,
-                                                   '__pyx_generator_body_t')
-        klass.declare_var(pos=pos, name='body', cname='body',
-                          type=body_type, is_cdef=True)
-        klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
-                          is_cdef=True)
-        klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
-                          is_cdef=True)
-        klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
-                          type=PyrexTypes.py_object_type, is_cdef=True)
-        klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
-                          type=PyrexTypes.py_object_type, is_cdef=True)
-        klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
-                          type=PyrexTypes.py_object_type, is_cdef=True)
-
-        import TypeSlots
-        e = klass.declare_pyfunction('send', pos)
-        e.func_cname = '__Pyx_Generator_Send'
-        e.signature = TypeSlots.binaryfunc
-
-        e = klass.declare_pyfunction('close', pos)
-        e.func_cname = '__Pyx_Generator_Close'
-        e.signature = TypeSlots.unaryfunc
-
-        e = klass.declare_pyfunction('throw', pos)
-        e.func_cname = '__Pyx_Generator_Throw'
-        e.signature = TypeSlots.pyfunction_signature
-
-        e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
-        e.func_cname = 'PyObject_SelfIter'
-
-        e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
-        e.func_cname = '__Pyx_Generator_Next'
-
-        self.generator_class = entry.type
-        return self.generator_class
-
     def find_entries_used_in_closures(self, node):
         from_closure = []
         in_closure = []
@@ -2140,9 +2089,8 @@ class CreateClosureClasses(CythonTransform):
             inner_node.needs_self_code = False
             node.needs_outer_scope = False
 
-        base_type = None
         if node.is_generator:
-            base_type = self.create_generator_class(target_module_scope, node.pos)
+            pass
         elif not in_closure and not from_closure:
             return
         elif not in_closure:
@@ -2151,23 +2099,19 @@ class CreateClosureClasses(CythonTransform):
             node.needs_outer_scope = True
             return
 
-        as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
+        as_name = '%s_%s' % (
+            target_module_scope.next_id(Naming.closure_class_prefix),
+            node.entry.cname)
 
         entry = target_module_scope.declare_c_class(
             name=as_name, pos=node.pos, defining=True,
-            implementing=True, base_type=base_type)
+            implementing=True)
         entry.type.is_final_type = True
 
         func_scope.scope_class = entry
         class_scope = entry.type.scope
         class_scope.is_internal = True
 
-        if node.is_generator and node.requires_classobj:
-            class_scope.declare_var(pos=node.pos, name='classobj',
-                                    cname='classobj',
-                                    type=PyrexTypes.py_object_type,
-                                    is_cdef=True)
-
         if from_closure:
             assert cscope.is_closure_scope
             class_scope.declare_var(pos=node.pos,

tests/run/generators_py.py

@@ -320,3 +320,18 @@ def test_lambda(n):
     """
     for i in range(n):
         yield lambda : i
+
+def test_generator_cleanup():
+    """
+    >>> g = test_generator_cleanup()
+    >>> del g
+    >>> g = test_generator_cleanup()
+    >>> next(g)
+    1
+    >>> del g
+    cleanup
+    """
+    try:
+        yield 1
+    finally:
+        print('cleanup')