merged in Vitek's generators branch

Cython/Compiler/Code.pxd

@@ -34,6 +34,8 @@ cdef class FunctionState:
     cdef public dict temps_used_type
     cdef public size_t temp_counter
 
+    cdef public object closure_temps
+
     @cython.locals(n=size_t)
     cpdef new_label(self, name=*)
     cpdef tuple get_loop_labels(self)

Cython/Compiler/Code.py

@@ -117,6 +117,7 @@ class FunctionState(object):
         self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
         self.temps_used_type = {} # name -> (type, manage_ref)
         self.temp_counter = 0
+        self.closure_temps = None
 
     # labels
 
@@ -270,6 +271,9 @@ class FunctionState(object):
                 if manage_ref
                 for cname in freelist]
 
+    def init_closure_temps(self, scope):
+        self.closure_temps = ClosureTempAllocator(scope)
+
 
 class IntConst(object):
     """Global info about a Python integer constant held by GlobalState.
@@ -475,6 +479,7 @@ class GlobalState(object):
         w.enter_cfunc_scope()
         w.putln("")
         w.putln("static int __Pyx_InitCachedConstants(void) {")
+        w.put_declare_refcount_context()
         w.put_setup_refcount_context("__Pyx_InitCachedConstants")
 
         w = self.parts['init_globals']
@@ -1297,6 +1302,8 @@ class CCodeWriter(object):
         #if entry.type.is_extension_type:
         #    code = "((PyObject*)%s)" % code
         self.put_init_to_py_none(code, entry.type, nanny)
+        if entry.in_closure:
+            self.put_giveref('Py_None')
 
     def put_pymethoddef(self, entry, term, allow_skip=True):
         if entry.is_special or entry.name == '__getattribute__':
@@ -1366,6 +1373,9 @@ class CCodeWriter(object):
     def lookup_filename(self, filename):
         return self.globalstate.lookup_filename(filename)
 
+    def put_declare_refcount_context(self):
+        self.putln('__Pyx_RefNannyDeclareContext;')
+
     def put_setup_refcount_context(self, name):
         self.putln('__Pyx_RefNannySetupContext("%s");' % name)
 
@@ -1402,3 +1412,26 @@ class PyrexCodeWriter(object):
     def dedent(self):
         self.level -= 1
 
+
+class ClosureTempAllocator(object):
+    def __init__(self, klass):
+        self.klass = klass
+        self.temps_allocated = {}
+        self.temps_free = {}
+        self.temps_count = 0
+
+    def reset(self):
+        for type, cnames in self.temps_allocated.items():
+            self.temps_free[type] = list(cnames)
+
+    def allocate_temp(self, type):
+        if not type in self.temps_allocated:
+            self.temps_allocated[type] = []
+            self.temps_free[type] = []
+        elif self.temps_free[type]:
+            return self.temps_free[type].pop(0)
+        cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count)
+        self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True)
+        self.temps_allocated[type].append(cname)
+        self.temps_count += 1
+        return cname

Cython/Compiler/ExprNodes.py

@@ -77,12 +77,15 @@ class ExprNode(Node):
     #               [ExprNode or [ExprNode or None] or None]
     #                            Cached result of subexpr_nodes()
     #  use_managed_ref boolean   use ref-counted temps/assignments/etc.
+    #  result_is_used  boolean   indicates that the result will be dropped and the
+    #                            result_code/temp_result can safely be set to None
 
     result_ctype = None
     type = None
     temp_code = None
     old_temp = None # error checker for multiple frees etc.
     use_managed_ref = True # can be set by optimisation transforms
+    result_is_used = True
 
     #  The Analyse Expressions phase for expressions is split
     #  into two sub-phases:
@@ -452,6 +455,9 @@ class ExprNode(Node):
 
     def release_temp_result(self, code):
         if not self.temp_code:
+            if not self.result_is_used:
+                # not used anyway, so ignore if not set up
+                return
             if self.old_temp:
                 raise RuntimeError("temp %s released multiple times in %s" % (
                         self.old_temp, self.__class__.__name__))
@@ -497,7 +503,7 @@ class ExprNode(Node):
 
     def generate_disposal_code(self, code):
         if self.is_temp:
-            if self.type.is_pyobject:
+            if self.type.is_pyobject and self.result():
                 code.put_decref_clear(self.result(), self.ctype())
         else:
             # Already done if self.is_temp
@@ -1628,9 +1634,10 @@ class NameNode(AtomicExprNode):
                 #print "...RHS type", rhs.type, "ctype", rhs.ctype() ###
                 if self.use_managed_ref:
                     rhs.make_owned_reference(code)
-                    if entry.is_cglobal:
-                        code.put_gotref(self.py_result())
+                    is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure
                     if not self.lhs_of_first_assignment:
+                        if is_external_ref:
+                            code.put_gotref(self.py_result())
                         if entry.is_local and not Options.init_local_none:
                             initialized = entry.scope.control_flow.get_state((entry.name, 'initialized'), self.pos)
                             if initialized is True:
@@ -1639,7 +1646,7 @@ class NameNode(AtomicExprNode):
                                 code.put_xdecref(self.result(), self.ctype())
                         else:
                             code.put_decref(self.result(), self.ctype())
-                    if entry.is_cglobal:
+                    if is_external_ref:
                         code.put_giveref(rhs.py_result())
 
             code.putln('%s = %s;' % (self.result(),
@@ -4442,15 +4449,18 @@ class DictComprehensionAppendNode(ComprehensionAppendNode):
         self.value_expr.annotate(code)
 
 
-class GeneratorExpressionNode(ScopedExprNode):
-    # A generator expression, e.g.  (i for i in range(10))
-    #
-    # Result is a generator.
+class InlinedGeneratorExpressionNode(ScopedExprNode):
+    # An inlined generator expression for which the result is
+    # calculated inside of the loop.  This will only be created by
+    # transforms when replacing builtin calls on generator
+    # expressions.
     #
-    # loop      ForStatNode   the for-loop, containing a YieldExprNode
+    # loop           ForStatNode      the for-loop, not containing any YieldExprNodes
+    # result_node    ResultRefNode    the reference to the result value temp
+    # orig_func      String           the name of the builtin function this node replaces
 
     child_attrs = ["loop"]
-
+    loop_analysed = False
     type = py_object_type
 
     def analyse_scoped_declarations(self, env):
@@ -4461,30 +4471,12 @@ class GeneratorExpressionNode(ScopedExprNode):
             self.loop.analyse_expressions(env)
         self.is_temp = True
 
-    def analyse_scoped_expressions(self, env):
-        if self.has_local_scope:
-            self.loop.analyse_expressions(env)
-
     def may_be_none(self):
         return False
 
     def annotate(self, code):
         self.loop.annotate(code)
 
-
-class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
-    # An inlined generator expression for which the result is
-    # calculated inside of the loop.  This will only be created by
-    # transforms when replacing builtin calls on generator
-    # expressions.
-    #
-    # loop           ForStatNode      the for-loop, not containing any YieldExprNodes
-    # result_node    ResultRefNode    the reference to the result value temp
-    # orig_func      String           the name of the builtin function this node replaces
-
-    child_attrs = ["loop"]
-    loop_analysed = False
-
     def infer_type(self, env):
         return self.result_node.infer_type(env)
 
@@ -4497,7 +4489,8 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
 
     def analyse_scoped_expressions(self, env):
         self.loop_analysed = True
-        GeneratorExpressionNode.analyse_scoped_expressions(self, env)
+        if self.has_local_scope:
+            self.loop.analyse_expressions(env)
 
     def coerce_to(self, dst_type, env):
         if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed:
@@ -4508,7 +4501,7 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
             # assignments.
             self.result_node.type = self.type = dst_type
             return self
-        return GeneratorExpressionNode.coerce_to(self, dst_type, env)
+        return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env)
 
     def generate_result_code(self, code):
         self.result_node.result_code = self.result()
@@ -5055,32 +5048,98 @@ class LambdaNode(InnerFunctionNode):
         self.pymethdef_cname = self.def_node.entry.pymethdef_cname
         env.add_lambda_def(self.def_node)
 
+
+class GeneratorExpressionNode(LambdaNode):
+    # A generator expression, e.g.  (i for i in range(10))
+    #
+    # Result is a generator.
+    #
+    # loop      ForStatNode   the for-loop, containing a YieldExprNode
+    # def_node  DefNode       the underlying generator 'def' node
+
+    name = StringEncoding.EncodedString('genexpr')
+    binding = False
+
+    def analyse_declarations(self, env):
+        self.def_node.no_assignment_synthesis = True
+        self.def_node.analyse_declarations(env)
+        env.add_lambda_def(self.def_node)
+
+    def generate_result_code(self, code):
+        code.putln(
+            '%s = %s(%s, NULL); %s' % (
+                self.result(),
+                self.def_node.entry.func_cname,
+                self.self_result_code(),
+                code.error_goto_if_null(self.result(), self.pos)))
+        code.put_gotref(self.py_result())
+
+
 class YieldExprNode(ExprNode):
     # Yield expression node
     #
     # arg         ExprNode   the value to return from the generator
     # label_name  string     name of the C label used for this yield
+    # label_num   integer    yield label number
 
     subexprs = ['arg']
     type = py_object_type
+    label_num = 0
 
     def analyse_types(self, env):
+        if not self.label_num:
+            error(self.pos, "'yield' not supported here")
         self.is_temp = 1
         if self.arg is not None:
             self.arg.analyse_types(env)
             if not self.arg.type.is_pyobject:
                 self.arg = self.arg.coerce_to_pyobject(env)
-        error(self.pos, "Generators are not supported")
+        env.use_utility_code(generator_utility_code)
 
-    def generate_result_code(self, code):
+    def generate_evaluation_code(self, code):
         self.label_name = code.new_label('resume_from_yield')
         code.use_label(self.label_name)
-        code.putln("/* FIXME: save temporary variables */")
-        code.putln("/* FIXME: return from function, yielding value */")
+        if self.arg:
+            self.arg.generate_evaluation_code(code)
+            self.arg.make_owned_reference(code)
+            code.putln(
+                "%s = %s;" % (
+                    Naming.retval_cname,
+                    self.arg.result_as(py_object_type)))
+            self.arg.generate_post_assignment_code(code)
+            #self.arg.generate_disposal_code(code)
+            self.arg.free_temps(code)
+        else:
+            code.put_init_to_py_none(Naming.retval_cname, py_object_type)
+        saved = []
+        code.funcstate.closure_temps.reset()
+        for cname, type, manage_ref in code.funcstate.temps_in_use():
+            save_cname = code.funcstate.closure_temps.allocate_temp(type)
+            saved.append((cname, save_cname, type))
+            if type.is_pyobject:
+                code.put_xgiveref(cname)
+            code.putln('%s->%s = %s;' % (Naming.cur_scope_cname, save_cname, cname))
+
+        code.put_xgiveref(Naming.retval_cname)
+        code.put_finish_refcount_context()
+        code.putln("/* return from generator, yielding value */")
+        code.putln("%s->%s.resume_label = %d;" % (Naming.cur_scope_cname, Naming.obj_base_cname, self.label_num))
+        code.putln("return %s;" % Naming.retval_cname);
         code.put_label(self.label_name)
-        code.putln("/* FIXME: restore temporary variables and  */")
-        code.putln("/* FIXME: extract sent value from closure */")
-
+        for cname, save_cname, type in saved:
+            code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname))
+            if type.is_pyobject:
+                code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname))
+            if type.is_pyobject:
+                code.put_xgotref(cname)
+        if self.result_is_used:
+            self.allocate_temp_result(code)
+            code.putln('%s = %s; %s' %
+                       (self.result(), Naming.sent_value_cname,
+                        code.error_goto_if_null(self.result(), self.pos)))
+            code.put_incref(self.result(), py_object_type)
+        else:
+            code.putln(code.error_goto_if_null(Naming.sent_value_cname, self.pos))
 
 #-------------------------------------------------------------------
 #
@@ -8403,3 +8462,101 @@ static int %(binding_cfunc)s_init(void) {
 
 }
 """ % Naming.__dict__)
+
+generator_utility_code = UtilityCode(
+proto="""
+static PyObject *__Pyx_Generator_Next(PyObject *self);
+static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value);
+static PyObject *__Pyx_Generator_Close(PyObject *self);
+static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args, CYTHON_UNUSED PyObject *kwds);
+
+typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
+""",
+impl="""
+static CYTHON_INLINE PyObject *__Pyx_Generator_SendEx(struct __pyx_Generator_object *self, PyObject *value)
+{
+    PyObject *retval;
+
+    if (self->is_running) {
+        PyErr_SetString(PyExc_ValueError,
+                        "generator already executing");
+        return NULL;
+    }
+
+    if (self->resume_label == 0) {
+        if (value && value != Py_None) {
+            PyErr_SetString(PyExc_TypeError,
+                            "can't send non-None value to a "
+                            "just-started generator");
+            return NULL;
+        }
+    }
+
+    if (self->resume_label == -1) {
+        PyErr_SetNone(PyExc_StopIteration);
+        return NULL;
+    }
+
+    self->is_running = 1;
+    retval = self->body((PyObject *) self, value);
+    self->is_running = 0;
+
+    return retval;
+}
+
+static PyObject *__Pyx_Generator_Next(PyObject *self)
+{
+    return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, Py_None);
+}
+
+static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value)
+{
+    return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, value);
+}
+
+static PyObject *__Pyx_Generator_Close(PyObject *self)
+{
+    struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
+    PyObject *retval;
+#if PY_VERSION_HEX < 0x02050000
+    PyErr_SetNone(PyExc_StopIteration);
+#else
+    PyErr_SetNone(PyExc_GeneratorExit);
+#endif
+    retval = __Pyx_Generator_SendEx(generator, NULL);
+    if (retval) {
+        Py_DECREF(retval);
+        PyErr_SetString(PyExc_RuntimeError,
+                        "generator ignored GeneratorExit");
+        return NULL;
+    }
+#if PY_VERSION_HEX < 0x02050000
+    if (PyErr_ExceptionMatches(PyExc_StopIteration))
+#else
+    if (PyErr_ExceptionMatches(PyExc_StopIteration)
+        || PyErr_ExceptionMatches(PyExc_GeneratorExit))
+#endif
+    {
+        PyErr_Clear();          /* ignore these errors */
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
+    return NULL;
+}
+
+static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args, CYTHON_UNUSED PyObject *kwds)
+{
+    struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
+    PyObject *typ;
+    PyObject *tb = NULL;
+    PyObject *val = NULL;
+
+    if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb))
+        return NULL;
+    __Pyx_Raise(typ, val, tb);
+    return __Pyx_Generator_SendEx(generator, NULL);
+}
+""",
+proto_block='utility_code_proto_before_types',
+requires=[Nodes.raise_utility_code],
+)

Cython/Compiler/ModuleNode.py

@@ -964,9 +964,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     type.vtabstruct_cname,
                     type.vtabslot_cname))
         for attr in type.scope.var_entries:
+            if attr.is_declared_generic:
+                attr_type = py_object_type
+            else:
+                attr_type = attr.type
             code.putln(
                 "%s;" %
-                    attr.type.declaration_code(attr.cname))
+                    attr_type.declaration_code(attr.cname))
         code.putln(footer)
         if type.objtypedef_cname is not None:
             # Only for exposing public typedef name.
@@ -1265,7 +1269,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         for entry in py_attrs:
             name = "p->%s" % entry.cname
             code.putln("tmp = ((PyObject*)%s);" % name)
-            code.put_init_to_py_none(name, entry.type, nanny=False)
+            if entry.is_declared_generic:
+                code.put_init_to_py_none(name, py_object_type, nanny=False)
+            else:
+                code.put_init_to_py_none(name, entry.type, nanny=False)
             code.putln("Py_XDECREF(tmp);")
         code.putln(
             "return 0;")
@@ -2762,8 +2769,9 @@ refnanny_utility_code = UtilityCode(proto="""
     Py_XDECREF(m);
     return (__Pyx_RefNannyAPIStruct *)r;
   }
+  #define __Pyx_RefNannyDeclareContext void *__pyx_refnanny;
   #define __Pyx_RefNannySetupContext(name) \
-          void *__pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__)
+          __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__)
   #define __Pyx_RefNannyFinishContext() \
           __Pyx_RefNanny->FinishContext(&__pyx_refnanny)
   #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
@@ -2772,6 +2780,7 @@ refnanny_utility_code = UtilityCode(proto="""
   #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
   #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r);} } while(0)
 #else
+  #define __Pyx_RefNannyDeclareContext
   #define __Pyx_RefNannySetupContext(name)
   #define __Pyx_RefNannyFinishContext()
   #define __Pyx_INCREF(r) Py_INCREF(r)

Cython/Compiler/Naming.py

@@ -19,6 +19,7 @@ funcdoc_prefix    = pyrex_prefix + "doc_"
 enum_prefix       = pyrex_prefix + "e_"
 func_prefix       = pyrex_prefix + "f_"
 pyfunc_prefix     = pyrex_prefix + "pf_"
+genbody_prefix    = pyrex_prefix + "gb_"
 gstab_prefix      = pyrex_prefix + "getsets_"
 prop_get_prefix   = pyrex_prefix + "getprop_"
 const_prefix      = pyrex_prefix + "k_"
@@ -51,6 +52,7 @@ lambda_func_prefix = pyrex_prefix + "lambda_"
 module_is_main   = pyrex_prefix + "module_is_main_"
 
 args_cname       = pyrex_prefix + "args"
+sent_value_cname = pyrex_prefix + "sent_value"
 pykwdlist_cname  = pyrex_prefix + "pyargnames"
 obj_base_cname   = pyrex_prefix + "base"
 builtins_cname   = pyrex_prefix + "b"
@@ -107,10 +109,6 @@ exc_lineno_name = pyrex_prefix + "exc_lineno"
 
 exc_vars = (exc_type_name, exc_value_name, exc_tb_name)
 
-exc_save_vars = (pyrex_prefix + 'save_exc_type',
-                 pyrex_prefix + 'save_exc_value',
-                 pyrex_prefix + 'save_exc_tb')
-
 api_name        = pyrex_prefix + "capi__"
 
 h_guard_prefix   = "__PYX_HAVE__"

Cython/Compiler/Nodes.py

@@ -23,7 +23,7 @@ from PyrexTypes import py_object_type, error_type, CFuncType
 from Symtab import ModuleScope, LocalScope, ClosureScope, \
     StructOrUnionScope, PyClassScope, CClassScope, CppClassScope
 from Cython.Utils import open_new_file, replace_suffix
-from Code import UtilityCode
+from Code import UtilityCode, ClosureTempAllocator
 from StringEncoding import EncodedString, escape_byte_string, split_string_literal
 import Options
 import ControlFlow
@@ -1177,6 +1177,8 @@ class FuncDefNode(StatNode, BlockNode):
     assmt = None
     needs_closure = False
     needs_outer_scope = False
+    is_generator = False
+    is_generator_body = False
     modifiers = []
 
     def analyse_default_values(self, env):
@@ -1236,6 +1238,9 @@ class FuncDefNode(StatNode, BlockNode):
         lenv.directives = env.directives
         return lenv
 
+    def generate_function_body(self, env, code):
+        self.body.generate_execution_code(code)
+
     def generate_function_definitions(self, env, code):
         import Buffer
 
@@ -1323,6 +1328,8 @@ class FuncDefNode(StatNode, BlockNode):
                     (self.return_type.declaration_code(Naming.retval_cname),
                      init))
         tempvardecl_code = code.insertion_point()
+        if not lenv.nogil:
+            code.put_declare_refcount_context()
         self.generate_keyword_list(code)
         if profile:
             code.put_trace_declarations()
@@ -1402,7 +1409,7 @@ class FuncDefNode(StatNode, BlockNode):
         # -------------------------
         # ----- Function body -----
         # -------------------------
-        self.body.generate_execution_code(code)
+        self.generate_function_body(env, code)
 
         # ----- Default return value
         code.putln("")
@@ -1484,14 +1491,10 @@ class FuncDefNode(StatNode, BlockNode):
             if entry.type.is_pyobject:
                 if entry.used and not entry.in_closure:
                     code.put_var_decref(entry)
-                elif entry.in_closure and self.needs_closure:
-                    code.put_giveref(entry.cname)
         # Decref any increfed args
         for entry in lenv.arg_entries:
             if entry.type.is_pyobject:
-                if entry.in_closure:
-                    code.put_var_giveref(entry)
-                elif acquire_gil or entry.assignments:
+                if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_decref(entry)
         if self.needs_closure:
             code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
@@ -1920,6 +1923,7 @@ class DefNode(FuncDefNode):
     num_required_kw_args = 0
     reqd_kw_flags_cname = "0"
     is_wrapper = 0
+    no_assignment_synthesis = 0
     decorators = None
     return_type_annotation = None
     entry = None
@@ -2250,6 +2254,8 @@ class DefNode(FuncDefNode):
             self.synthesize_assignment_node(env)
 
     def needs_assignment_synthesis(self, env, code=None):
+        if self.no_assignment_synthesis:
+            return False
         # Should enable for module level as well, that will require more testing...
         if self.entry.is_anonymous:
             return True
@@ -2353,8 +2359,8 @@ class DefNode(FuncDefNode):
             code.putln("0};")
 
     def generate_argument_parsing_code(self, env, code):
-        # Generate PyArg_ParseTuple call for generic
-        # arguments, if any.
+        # Generate fast equivalent of PyArg_ParseTuple call for
+        # generic arguments, if any, including args/kwargs
         if self.entry.signature.has_dummy_arg and not self.self_in_stararg:
             # get rid of unused argument warning
             code.putln("%s = %s;" % (Naming.self_cname, Naming.self_cname))
@@ -2431,14 +2437,24 @@ class DefNode(FuncDefNode):
         if code.label_used(end_label):
             code.put_label(end_label)
 
+        # fix refnanny view on closure variables here, instead of
+        # doing it separately for each arg parsing special case
+        if self.star_arg and self.star_arg.entry.in_closure:
+            code.put_var_giveref(self.star_arg.entry)
+        if self.starstar_arg and self.starstar_arg.entry.in_closure:
+            code.put_var_giveref(self.starstar_arg.entry)
+        for arg in self.args:
+            if arg.type.is_pyobject and arg.entry.in_closure:
+                code.put_var_giveref(arg.entry)
+
     def generate_arg_assignment(self, arg, item, code):
         if arg.type.is_pyobject:
             if arg.is_generic:
                 item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
             entry = arg.entry
-            code.putln("%s = %s;" % (entry.cname, item))
             if entry.in_closure:
-                code.put_var_incref(entry)
+                code.put_incref(item, PyrexTypes.py_object_type)
+            code.putln("%s = %s;" % (entry.cname, item))
         else:
             func = arg.type.from_py_function
             if func:
@@ -2657,19 +2673,18 @@ class DefNode(FuncDefNode):
             code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % (
                     Naming.args_cname,
                     max_positional_args))
-            code.put('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s)); ' % (
+            code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % (
                     self.star_arg.entry.cname, Naming.args_cname,
                     max_positional_args, Naming.args_cname))
-            code.put_gotref(self.star_arg.entry.cname)
+            code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
             if self.starstar_arg:
-                code.putln("")
-                code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
                 code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
-                code.putln('return %s;' % self.error_value())
-                code.putln('}')
-            else:
-                code.putln("if (unlikely(!%s)) return %s;" % (
-                        self.star_arg.entry.cname, self.error_value()))
+            if self.needs_closure:
+                code.put_decref(Naming.cur_scope_cname, self.local_scope.scope_class.type)
+            code.put_finish_refcount_context()
+            code.putln('return %s;' % self.error_value())
+            code.putln('}')
+            code.put_gotref(self.star_arg.entry.cname)
             code.putln('} else {')
             code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple))
             code.put_incref(Naming.empty_tuple, py_object_type)
@@ -2839,9 +2854,9 @@ class DefNode(FuncDefNode):
             if arg.needs_conversion:
                 self.generate_arg_conversion(arg, code)
             elif arg.entry.in_closure:
-                code.putln('%s = %s;' % (arg.entry.cname, arg.hdr_cname))
                 if arg.type.is_pyobject:
-                    code.put_var_incref(arg.entry)
+                    code.put_incref(arg.hdr_cname, py_object_type)
+                code.putln('%s = %s;' % (arg.entry.cname, arg.hdr_cname))
 
     def generate_arg_conversion(self, arg, code):
         # Generate conversion code for one argument.
@@ -2914,6 +2929,146 @@ class DefNode(FuncDefNode):
     def caller_will_check_exceptions(self):
         return 1
 
+
+class GeneratorDefNode(DefNode):
+    # Generator DefNode.
+    #
+    # gbody          GeneratorBodyDefNode
+    #
+
+    is_generator = True
+    needs_closure = True
+
+    child_attrs = DefNode.child_attrs + ["gbody"]
+
+    def __init__(self, **kwargs):
+        # XXX: don't actually needs a body
+        kwargs['body'] = StatListNode(kwargs['pos'], stats=[])
+        super(GeneratorDefNode, self).__init__(**kwargs)
+
+    def analyse_declarations(self, env):
+        super(GeneratorDefNode, self).analyse_declarations(env)
+        self.gbody.local_scope = self.local_scope
+        self.gbody.analyse_declarations(env)
+
+    def generate_function_body(self, env, code):
+        body_cname = self.gbody.entry.func_cname
+        generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
+
+        code.putln('%s.resume_label = 0;' % generator_cname)
+        code.putln('%s.body = (__pyx_generator_body_t) %s;' % (generator_cname, body_cname))
+        code.put_giveref(Naming.cur_scope_cname)
+        code.put_finish_refcount_context()
+        code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
+
+    def generate_function_definitions(self, env, code):
+        self.gbody.generate_function_header(code, proto=True)
+        super(GeneratorDefNode, self).generate_function_definitions(env, code)
+        self.gbody.generate_function_definitions(env, code)
+
+
+class GeneratorBodyDefNode(DefNode):
+    # Generator body DefNode.
+    #
+
+    is_generator_body = True
+
+    def __init__(self, pos=None, name=None, body=None):
+        super(GeneratorBodyDefNode, self).__init__(pos=pos, body=body, name=name, doc=None,
+                                                   args=[],
+                                                   star_arg=None, starstar_arg=None)
+
+    def declare_generator_body(self, env):
+        prefix = env.next_id(env.scope_prefix)
+        name = env.next_id('generator')
+        entry = env.declare_var(prefix + name, py_object_type, self.pos, visibility='private')
+        entry.func_cname = Naming.genbody_prefix + prefix + name
+        entry.qualified_name = EncodedString(self.name)
+        self.entry = entry
+
+    def analyse_declarations(self, env):
+        self.analyse_argument_types(env)
+        self.declare_generator_body(env)
+
+    def generate_function_header(self, code, proto=False):
+        header = "static PyObject *%s(%s, PyObject *%s)" % (
+            self.entry.func_cname,
+            self.local_scope.scope_class.type.declaration_code(Naming.cur_scope_cname),
+            Naming.sent_value_cname)
+        if proto:
+            code.putln('%s; /* proto */' % header)
+        else:
+            code.putln('%s /* generator body */\n{' % header);
+
+    def generate_function_definitions(self, env, code):
+        lenv = self.local_scope
+
+        # Generate closure function definitions
+        self.body.generate_function_definitions(lenv, code)
+
+        # Generate C code for header and body of function
+        code.enter_cfunc_scope()
+        code.return_from_error_cleanup_label = code.new_label()
+
+        # ----- Top-level constants used by this function
+        code.mark_pos(self.pos)
+        self.generate_cached_builtins_decls(lenv, code)
+        # ----- Function header
+        code.putln("")
+        self.generate_function_header(code)
+        # ----- Local variables
+        code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
+        tempvardecl_code = code.insertion_point()
+        code.put_declare_refcount_context()
+        code.put_setup_refcount_context(self.entry.name)
+
+        # ----- Resume switch point.
+        code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
+        resume_code = code.insertion_point()
+        first_run_label = code.new_label('first_run')
+        code.use_label(first_run_label)
+        code.put_label(first_run_label)
+        code.putln('%s' %
+                   (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))
+
+        # ----- Function body
+        self.generate_function_body(env, code)
+        code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
+        # ----- Error cleanup
+        if code.error_label in code.labels_used:
+            code.put_goto(code.return_label)
+            code.put_label(code.error_label)
+            for cname, type in code.funcstate.all_managed_temps():
+                code.put_xdecref(cname, type)
+            code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
+
+        # ----- Non-error return cleanup
+        code.put_label(code.return_label)
+        code.put_xdecref(Naming.retval_cname, py_object_type)
+        code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
+        code.put_finish_refcount_context()
+        code.putln('return NULL;');
+        code.putln("}")
+
+        # ----- Go back and insert temp variable declarations
+        tempvardecl_code.put_temp_declarations(code.funcstate)
+        # ----- Generator resume code
+        resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
+        resume_code.putln("case 0: goto %s;" % first_run_label)
+
+        from ParseTreeTransforms import YieldNodeCollector
+        collector = YieldNodeCollector()
+        collector.visitchildren(self)
+        for yield_expr in collector.yields:
+            resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
+        resume_code.putln("default: /* CPython raises the right error here */");
+        resume_code.put_finish_refcount_context()
+        resume_code.putln("return NULL;");
+        resume_code.putln("}");
+
+        code.exit_cfunc_scope()
+
+
 class OverrideCheckNode(StatNode):
     # A Node for dispatching to the def method if it
     # is overriden.
@@ -3352,6 +3507,24 @@ class GlobalNode(StatNode):
         pass
 
 
+class NonlocalNode(StatNode):
+    # Nonlocal variable declaration via the 'nonlocal' keyword.
+    #
+    # names    [string]
+
+    child_attrs = []
+
+    def analyse_declarations(self, env):
+        for name in self.names:
+            env.declare_nonlocal(name, self.pos)
+
+    def analyse_expressions(self, env):
+        pass
+
+    def generate_execution_code(self, code):
+        pass
+
+
 class ExprStatNode(StatNode):
     #  Expression used as a statement.
     #
@@ -3376,6 +3549,7 @@ class ExprStatNode(StatNode):
                 self.__class__ = PassStatNode
 
     def analyse_expressions(self, env):
+        self.expr.result_is_used = False # hint that .result() may safely be left empty
         self.expr.analyse_expressions(env)
 
     def nogil_check(self, env):
@@ -4705,12 +4879,12 @@ class TryExceptStatNode(StatNode):
         try_continue_label = code.new_label('try_continue')
         try_end_label = code.new_label('try_end')
 
+        exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
+                         for i in xrange(3)]
         code.putln("{")
-        code.putln("PyObject %s;" %
-                   ', '.join(['*%s' % var for var in Naming.exc_save_vars]))
         code.putln("__Pyx_ExceptionSave(%s);" %
-                   ', '.join(['&%s' % var for var in Naming.exc_save_vars]))
-        for var in Naming.exc_save_vars:
+                   ', '.join(['&%s' % var for var in exc_save_vars]))
+        for var in exc_save_vars:
             code.put_xgotref(var)
         code.putln(
             "/*try:*/ {")
@@ -4729,14 +4903,15 @@ class TryExceptStatNode(StatNode):
             self.else_clause.generate_execution_code(code)
             code.putln(
                 "}")
-        for var in Naming.exc_save_vars:
+        for var in exc_save_vars:
             code.put_xdecref_clear(var, py_object_type)
         code.put_goto(try_end_label)
         if code.label_used(try_return_label):
             code.put_label(try_return_label)
-            for var in Naming.exc_save_vars: code.put_xgiveref(var)
+            for var in exc_save_vars:
+                code.put_xgiveref(var)
             code.putln("__Pyx_ExceptionReset(%s);" %
-                       ', '.join(Naming.exc_save_vars))
+                       ', '.join(exc_save_vars))
             code.put_goto(old_return_label)
         code.put_label(our_error_label)
         for temp_name, type in temps_to_clean_up:
@@ -4748,9 +4923,10 @@ class TryExceptStatNode(StatNode):
         if error_label_used or not self.has_default_clause:
             if error_label_used:
                 code.put_label(except_error_label)
-            for var in Naming.exc_save_vars: code.put_xgiveref(var)
+            for var in exc_save_vars:
+                code.put_xgiveref(var)
             code.putln("__Pyx_ExceptionReset(%s);" %
-                       ', '.join(Naming.exc_save_vars))
+                       ', '.join(exc_save_vars))
             code.put_goto(old_error_label)
 
         for exit_label, old_label in zip(
@@ -4759,19 +4935,24 @@ class TryExceptStatNode(StatNode):
 
             if code.label_used(exit_label):
                 code.put_label(exit_label)
-                for var in Naming.exc_save_vars: code.put_xgiveref(var)
+                for var in exc_save_vars:
+                    code.put_xgiveref(var)
                 code.putln("__Pyx_ExceptionReset(%s);" %
-                           ', '.join(Naming.exc_save_vars))
+                           ', '.join(exc_save_vars))
                 code.put_goto(old_label)
 
         if code.label_used(except_end_label):
             code.put_label(except_end_label)
-            for var in Naming.exc_save_vars: code.put_xgiveref(var)
+            for var in exc_save_vars:
+                code.put_xgiveref(var)
             code.putln("__Pyx_ExceptionReset(%s);" %
-                       ', '.join(Naming.exc_save_vars))
+                       ', '.join(exc_save_vars))
         code.put_label(try_end_label)
         code.putln("}")
 
+        for cname in exc_save_vars:
+            code.funcstate.release_temp(cname)
+
         code.return_label = old_return_label
         code.break_label = old_break_label
         code.continue_label = old_continue_label

Cython/Compiler/Optimize.py

@@ -1170,11 +1170,12 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform):
             self.yield_nodes = []
 
         visit_Node = Visitor.TreeVisitor.visitchildren
-        def visit_YieldExprNode(self, node):
+        # XXX: disable inlining while it's not back supported
+        def __visit_YieldExprNode(self, node):
             self.yield_nodes.append(node)
             self.visitchildren(node)
 
-        def visit_ExprStatNode(self, node):
+        def __visit_ExprStatNode(self, node):
             self.visitchildren(node)
             if node.expr in self.yield_nodes:
                 self.yield_stat_nodes[node.expr] = node

Cython/Compiler/ParseTreeTransforms.pxd

@@ -19,6 +19,7 @@ cdef class NormalizeTree(CythonTransform):
 cdef class PostParse(ScopeTrackingTransform):
     cdef dict specialattribute_handlers
     cdef size_t lambda_counter
+    cdef size_t genexpr_counter
     cdef _visit_assignment_node(self, node, list expr_list)
 
 
@@ -45,6 +46,11 @@ cdef class AlignFunctionDefinitions(CythonTransform):
     cdef dict directives
     cdef scope
 
+cdef class YieldNodeCollector(TreeVisitor):
+    cdef public list yields
+    cdef public list returns
+    cdef public bint has_return_value
+
 cdef class MarkClosureVisitor(CythonTransform):
     cdef bint needs_closure
 
@@ -52,6 +58,7 @@ cdef class CreateClosureClasses(CythonTransform):
     cdef list path
     cdef bint in_lambda
     cdef module_scope
+    cdef generator_class
 
 cdef class GilCheck(VisitorTransform):
     cdef list env_stack

Cython/Compiler/ParseTreeTransforms.py

@@ -182,6 +182,7 @@ class PostParse(ScopeTrackingTransform):
 
     def visit_ModuleNode(self, node):
         self.lambda_counter = 1
+        self.genexpr_counter = 1
         return super(PostParse, self).visit_ModuleNode(node)
 
     def visit_LambdaNode(self, node):
@@ -189,14 +190,34 @@ class PostParse(ScopeTrackingTransform):
         lambda_id = self.lambda_counter
         self.lambda_counter += 1
         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
-
-        body = Nodes.ReturnStatNode(
-            node.result_expr.pos, value = node.result_expr)
+        collector = YieldNodeCollector()
+        collector.visitchildren(node.result_expr)
+        if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
+            body = ExprNodes.YieldExprNode(
+                node.result_expr.pos, arg=node.result_expr)
+            body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
+        else:
+            body = Nodes.ReturnStatNode(
+                node.result_expr.pos, value=node.result_expr)
         node.def_node = Nodes.DefNode(
             node.pos, name=node.name, lambda_name=node.lambda_name,
             args=node.args, star_arg=node.star_arg,
             starstar_arg=node.starstar_arg,
-            body=body)
+            body=body, doc=None)
+        self.visitchildren(node)
+        return node
+
+    def visit_GeneratorExpressionNode(self, node):
+        # unpack a generator expression into the corresponding DefNode
+        genexpr_id = self.genexpr_counter
+        self.genexpr_counter += 1
+        node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
+
+        node.def_node = Nodes.DefNode(node.pos, name=node.name,
+                                      doc=None,
+                                      args=[], star_arg=None,
+                                      starstar_arg=None,
+                                      body=node.loop)
         self.visitchildren(node)
         return node
 
@@ -1408,6 +1429,42 @@ class AlignFunctionDefinitions(CythonTransform):
         return node
 
 
+class YieldNodeCollector(TreeVisitor):
+
+    def __init__(self):
+        super(YieldNodeCollector, self).__init__()
+        self.yields = []
+        self.returns = []
+        self.has_return_value = False
+
+    def visit_Node(self, node):
+        return self.visitchildren(node)
+
+    def visit_YieldExprNode(self, node):
+        if self.has_return_value:
+            error(node.pos, "'yield' outside function")
+        self.yields.append(node)
+        self.visitchildren(node)
+
+    def visit_ReturnStatNode(self, node):
+        if node.value:
+            self.has_return_value = True
+            if self.yields:
+                error(node.pos, "'return' with argument inside generator")
+        self.returns.append(node)
+
+    def visit_ClassDefNode(self, node):
+        pass
+
+    def visit_DefNode(self, node):
+        pass
+
+    def visit_LambdaNode(self, node):
+        pass
+
+    def visit_GeneratorExpressionNode(self, node):
+        pass
+
 class MarkClosureVisitor(CythonTransform):
 
     def visit_ModuleNode(self, node):
@@ -1420,6 +1477,27 @@ class MarkClosureVisitor(CythonTransform):
         self.visitchildren(node)
         node.needs_closure = self.needs_closure
         self.needs_closure = True
+
+        collector = YieldNodeCollector()
+        collector.visitchildren(node)
+
+        if collector.yields:
+            for i, yield_expr in enumerate(collector.yields):
+                yield_expr.label_num = i + 1
+
+            gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
+                                               name=node.name,
+                                               body=node.body)
+            generator = Nodes.GeneratorDefNode(pos=node.pos,
+                                               name=node.name,
+                                               args=node.args,
+                                               star_arg=node.star_arg,
+                                               starstar_arg=node.starstar_arg,
+                                               doc=node.doc,
+                                               decorators=node.decorators,
+                                               gbody=gbody,
+                                               lambda_name=node.lambda_name)
+            return generator
         return node
 
     def visit_CFuncDefNode(self, node):
@@ -1440,7 +1518,6 @@ class MarkClosureVisitor(CythonTransform):
         self.needs_closure = True
         return node
 
-
 class CreateClosureClasses(CythonTransform):
     # Output closure classes in module scope for all functions
     # that really need it.
@@ -1449,24 +1526,78 @@ class CreateClosureClasses(CythonTransform):
         super(CreateClosureClasses, self).__init__(context)
         self.path = []
         self.in_lambda = False
+        self.generator_class = None
 
     def visit_ModuleNode(self, node):
         self.module_scope = node.scope
         self.visitchildren(node)
         return node
 
-    def get_scope_use(self, node):
+    def create_generator_class(self, target_module_scope, pos):
+        if self.generator_class:
+            return self.generator_class
+        # XXX: make generator class creation cleaner
+        entry = target_module_scope.declare_c_class(name='__pyx_Generator',
+                    objstruct_cname='__pyx_Generator_object',
+                    typeobj_cname='__pyx_Generator_type',
+                    pos=pos, defining=True, implementing=True)
+        klass = entry.type.scope
+        klass.is_internal = True
+        klass.directives = {'final': True}
+
+        body_type = PyrexTypes.create_typedef_type('generator_body',
+                                                   PyrexTypes.c_void_ptr_type,
+                                                   '__pyx_generator_body_t')
+        klass.declare_var(pos=pos, name='body', cname='body',
+                          type=body_type, is_cdef=True)
+        klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
+                          is_cdef=True)
+        klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
+                          is_cdef=True)
+
+        import TypeSlots
+        e = klass.declare_pyfunction('send', pos)
+        e.func_cname = '__Pyx_Generator_Send'
+        e.signature = TypeSlots.binaryfunc
+
+        e = klass.declare_pyfunction('close', pos)
+        e.func_cname = '__Pyx_Generator_Close'
+        e.signature = TypeSlots.unaryfunc
+
+        e = klass.declare_pyfunction('throw', pos)
+        e.func_cname = '__Pyx_Generator_Throw'
+        e.signature = TypeSlots.pyfunction_signature
+
+        e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
+        e.func_cname = 'PyObject_SelfIter'
+
+        e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
+        e.func_cname = '__Pyx_Generator_Next'
+
+        self.generator_class = entry.type
+        return self.generator_class
+
+    def find_entries_used_in_closures(self, node):
         from_closure = []
         in_closure = []
         for name, entry in node.local_scope.entries.items():
             if entry.from_closure:
                 from_closure.append((name, entry))
-            elif entry.in_closure and not entry.from_closure:
+            elif entry.in_closure:
                 in_closure.append((name, entry))
         return from_closure, in_closure
 
     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
-        from_closure, in_closure = self.get_scope_use(node)
+        # skip generator body
+        if node.is_generator_body:
+            return
+        # move local variables into closure
+        if node.is_generator:
+            for entry in node.local_scope.entries.values():
+                if not entry.from_closure:
+                    entry.in_closure = True
+
+        from_closure, in_closure = self.find_entries_used_in_closures(node)
         in_closure.sort()
 
         # Now from the begining
@@ -1485,8 +1616,11 @@ class CreateClosureClasses(CythonTransform):
                 inner_node = node.assmt.rhs
             inner_node.needs_self_code = False
             node.needs_outer_scope = False
-        # Simple cases
-        if not in_closure and not from_closure:
+
+        base_type = None
+        if node.is_generator:
+            base_type = self.create_generator_class(target_module_scope, node.pos)
+        elif not in_closure and not from_closure:
             return
         elif not in_closure:
             func_scope.is_passthrough = True
@@ -1496,8 +1630,10 @@ class CreateClosureClasses(CythonTransform):
 
         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
 
-        entry = target_module_scope.declare_c_class(name = as_name,
-            pos = node.pos, defining = True, implementing = True)
+        entry = target_module_scope.declare_c_class(
+            name=as_name, pos=node.pos, defining=True,
+            implementing=True, base_type=base_type)
+
         func_scope.scope_class = entry
         class_scope = entry.type.scope
         class_scope.is_internal = True
@@ -1512,11 +1648,13 @@ class CreateClosureClasses(CythonTransform):
                                     is_cdef=True)
             node.needs_outer_scope = True
         for name, entry in in_closure:
-            class_scope.declare_var(pos=entry.pos,
+            closure_entry = class_scope.declare_var(pos=entry.pos,
                                     name=entry.name,
                                     cname=entry.cname,
                                     type=entry.type,
                                     is_cdef=True)
+            if entry.is_declared_generic:
+                closure_entry.is_declared_generic = 1
         node.needs_closure = True
         # Do it here because other classes are already checked
         target_module_scope.check_c_class(func_scope.scope_class)

Cython/Compiler/Parsing.pxd

@@ -84,6 +84,7 @@ cdef p_genexp(PyrexScanner s, expr)
 #-------------------------------------------------------
 
 cdef p_global_statement(PyrexScanner s)
+cdef p_nonlocal_statement(PyrexScanner s)
 cdef p_expression_or_assignment(PyrexScanner s)
 cdef p_print_statement(PyrexScanner s)
 cdef p_exec_statement(PyrexScanner s)

Cython/Compiler/Parsing.py

@@ -1045,6 +1045,12 @@ def p_global_statement(s):
     names = p_ident_list(s)
     return Nodes.GlobalNode(pos, names = names)
 
+def p_nonlocal_statement(s):
+    pos = s.position()
+    s.next()
+    names = p_ident_list(s)
+    return Nodes.NonlocalNode(pos, names = names)
+
 def p_expression_or_assignment(s):
     expr_list = [p_testlist_star_expr(s)]
     while s.sy == '=':
@@ -1598,6 +1604,8 @@ def p_simple_statement(s, first_statement = 0):
     #print "p_simple_statement:", s.sy, s.systring ###
     if s.sy == 'global':
         node = p_global_statement(s)
+    elif s.sy == 'nonlocal':
+        node = p_nonlocal_statement(s)
     elif s.sy == 'print':
         node = p_print_statement(s)
     elif s.sy == 'exec':

Cython/Compiler/Scanning.py

@@ -36,7 +36,7 @@ def get_lexicon():
 #------------------------------------------------------------------
 
 py_reserved_words = [
-    "global", "def", "class", "print", "del", "pass", "break",
+    "global", "nonlocal", "def", "class", "print", "del", "pass", "break",
     "continue", "return", "raise", "import", "exec", "try",
     "except", "finally", "while", "if", "elif", "else", "for",
     "in", "assert", "and", "or", "not", "is", "in", "lambda",

Cython/Compiler/Symtab.py

@@ -1309,6 +1309,16 @@ class LocalScope(Scope):
             entry = self.global_scope().lookup_target(name)
             self.entries[name] = entry
 
+    def declare_nonlocal(self, name, pos):
+        # Pull entry from outer scope into local scope
+        orig_entry = self.lookup_here(name)
+        if orig_entry and orig_entry.scope is self and not orig_entry.from_closure:
+            error(pos, "'%s' redeclared as nonlocal" % name)
+        else:
+            entry = self.lookup(name)
+            if entry is None or not entry.from_closure:
+                error(pos, "no binding for nonlocal '%s' found" % name)
+
     def lookup(self, name):
         # Look up name in this scope or an enclosing one.
         # Return None if not found.
@@ -1326,6 +1336,7 @@ class LocalScope(Scope):
                 inner_entry.is_variable = True
                 inner_entry.outer_entry = entry
                 inner_entry.from_closure = True
+                inner_entry.is_declared_generic = entry.is_declared_generic
                 self.entries[name] = inner_entry
                 return inner_entry
         return entry
@@ -1479,6 +1490,20 @@ class PyClassScope(ClassScope):
         entry.is_pyclass_attr = 1
         return entry
 
+    def declare_nonlocal(self, name, pos):
+        # Pull entry from outer scope into local scope
+        orig_entry = self.lookup_here(name)
+        if orig_entry and orig_entry.scope is self and not orig_entry.from_closure:
+            error(pos, "'%s' redeclared as nonlocal" % name)
+        else:
+            entry = self.lookup(name)
+            if entry is None:
+                error(pos, "no binding for nonlocal '%s' found" % name)
+            else:
+                # FIXME: this works, but it's unclear if it's the
+                # right thing to do
+                self.entries[name] = entry
+
     def add_default_value(self, type):
         return self.outer_scope.add_default_value(type)
 

Cython/Compiler/TypeInference.py

@@ -221,7 +221,8 @@ class SimpleAssignmentTypeInferer(object):
     # TODO: Implement a real type inference algorithm.
     # (Something more powerful than just extending this one...)
     def infer_types(self, scope):
-        enabled = not scope.is_closure_scope and scope.directives['infer_types']
+        closure_or_inner = scope.is_closure_scope or (scope.outer_scope and scope.outer_scope.is_closure_scope)
+        enabled = not closure_or_inner and scope.directives['infer_types']
         verbose = scope.directives['infer_types.verbose']
         if enabled == True:
             spanning_type = aggressive_spanning_type

tests/bugs.txt

@@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408
 compile.cpp_operators
 cpp_templated_ctypedef
 cpp_structs
-genexpr_T491
 with_statement_module_level_T536
 function_as_method_T494
 closure_inside_cdef_T554
@@ -19,6 +18,7 @@ genexpr_iterable_lookup_T600
 for_from_pyvar_loop_T601
 decorators_T593
 temp_sideeffects_T654
+generator_type_inference
 
 # CPython regression tests that don't current work:
 pyregr.test_threadsignals

tests/errors/e_generators.pyx

+def foo():
+    yield
+    return 0
+
+def bar(a):
+    return 0
+    yield
+
+yield
+
+class Foo:
+    yield
+
+_ERRORS = u"""
+3:4: 'return' with argument inside generator
+7:4: 'yield' outside function
+9:0: 'yield' not supported here
+12:4: 'yield' not supported here
+"""

tests/errors/e_nonlocal_T490.pyx

+
+def test_non_existant():
+    nonlocal no_such_name
+    no_such_name = 1
+
+def redef():
+    x = 1
+    def f():
+        x = 2
+        nonlocal x
+
+global_name = 5
+
+def ref_to_global():
+    nonlocal global_name
+    global_name = 6
+
+def global_in_class_scope():
+    class Test():
+        nonlocal global_name
+        global_name = 6
+
+def redef_in_class_scope():
+    x = 1
+    class Test():
+        x = 2
+        nonlocal x
+
+
+_ERRORS = u"""
+3:4: no binding for nonlocal 'no_such_name' found
+10:8: 'x' redeclared as nonlocal
+15:4: no binding for nonlocal 'global_name' found
+27:8: 'x' redeclared as nonlocal
+"""

tests/run/closure_self.pyx

+cdef class Test:
+    cdef int x
+
+cdef class SelfInClosure(object):
+    cdef Test _t
+    cdef int x
+
+    def plain(self):
+        """
+        >>> o = SelfInClosure()
+        >>> o.plain()
+        1
+        """
+        self.x = 1
+        return self.x
+
+    def closure_method(self):
+        """
+        >>> o = SelfInClosure()
+        >>> o.closure_method()() == o
+        True
+        """
+        def nested():
+            return self
+        return nested
+
+    def closure_method_cdef_attr(self, Test t):
+        """
+        >>> o = SelfInClosure()
+        >>> o.closure_method_cdef_attr(Test())()
+        (1, 2)
+        """
+        t.x = 2
+        self._t = t
+        self.x = 1
+        def nested():
+            return self.x, t.x
+        return nested

tests/run/generator_type_inference.pyx

+# mode: run
+# tag: typeinference, generators
+
+cimport cython
+
+def test_type_inference():
+    """
+    >>> [ item for item in test_type_inference() ]
+    [(2.0, 'double'), (2.0, 'double'), (2.0, 'double')]
+    """
+    x = 1.0
+    for i in range(3):
+        yield x * 2.0, cython.typeof(x)

tests/run/generators.pyx

+
+try:
+    from builtins import next # Py3k
+except ImportError:
+    def next(it):
+        return it.next()
+
+if hasattr(__builtins__, 'GeneratorExit'):
+    GeneratorExit = __builtins__.GeneratorExit
+else: # < 2.5
+    GeneratorExit = StopIteration
+
+def very_simple():
+    """
+    >>> x = very_simple()
+    >>> next(x)
+    1
+    >>> next(x)
+    Traceback (most recent call last):
+    StopIteration
+    >>> next(x)
+    Traceback (most recent call last):
+    StopIteration
+    >>> x = very_simple()
+    >>> x.send(1)
+    Traceback (most recent call last):
+    TypeError: can't send non-None value to a just-started generator
+    """
+    yield 1
+
+
+def simple():
+    """
+    >>> x = simple()
+    >>> list(x)
+    [1, 2, 3]
+    """
+    yield 1
+    yield 2
+    yield 3
+
+def simple_seq(seq):
+    """
+    >>> x = simple_seq("abc")
+    >>> list(x)
+    ['a', 'b', 'c']
+    """
+    for i in seq:
+        yield i
+
+def simple_send():
+    """
+    >>> x = simple_send()
+    >>> next(x)
+    >>> x.send(1)
+    1
+    >>> x.send(2)
+    2
+    >>> x.send(3)
+    3
+    """
+    i = None
+    while True:
+        i = yield i
+
+def raising():
+    """
+    >>> x = raising()
+    >>> next(x)
+    Traceback (most recent call last):
+    KeyError: 'foo'
+    >>> next(x)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    yield {}['foo']
+
+def with_outer(*args):
+    """
+    >>> x = with_outer(1, 2, 3)
+    >>> list(x())
+    [1, 2, 3]
+    """
+    def generator():
+        for i in args:
+            yield i
+    return generator
+
+def with_outer_raising(*args):
+    """
+    >>> x = with_outer_raising(1, 2, 3)
+    >>> list(x())
+    [1, 2, 3]
+    """
+    def generator():
+        for i in args:
+            yield i
+        raise StopIteration
+    return generator
+
+def test_close():
+    """
+    >>> x = test_close()
+    >>> x.close()
+    >>> x = test_close()
+    >>> next(x)
+    >>> x.close()
+    >>> next(x)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    while True:
+        yield
+
+def test_ignore_close():
+    """
+    >>> x = test_ignore_close()
+    >>> x.close()
+    >>> x = test_ignore_close()
+    >>> next(x)
+    >>> x.close()
+    Traceback (most recent call last):
+    RuntimeError: generator ignored GeneratorExit
+    """
+    try:
+        yield
+    except GeneratorExit:
+        yield
+
+def check_throw():
+    """
+    >>> x = check_throw()
+    >>> x.throw(ValueError)
+    Traceback (most recent call last):
+    ValueError
+    >>> next(x)
+    Traceback (most recent call last):
+    StopIteration
+    >>> x = check_throw()
+    >>> next(x)
+    >>> x.throw(ValueError)
+    >>> next(x)
+    >>> x.throw(IndexError, "oops")
+    Traceback (most recent call last):
+    IndexError: oops
+    >>> next(x)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    while True:
+        try:
+            yield
+        except ValueError:
+            pass
+
+def test_first_assignment():
+    """
+    >>> gen = test_first_assignment()
+    >>> next(gen)
+    5
+    >>> next(gen)
+    10
+    >>> next(gen)
+    (5, 10)
+    """
+    cdef x = 5 # first
+    yield x
+    cdef y = 10 # first
+    yield y
+    yield (x,y)
+
+def test_swap_assignment():
+    """
+    >>> gen = test_swap_assignment()
+    >>> next(gen)
+    (5, 10)
+    >>> next(gen)
+    (10, 5)
+    """
+    x,y = 5,10
+    yield (x,y)
+    x,y = y,x   # no ref-counting here
+    yield (x,y)
+
+
+class Foo(object):
+    """
+    >>> obj = Foo()
+    >>> list(obj.simple(1, 2, 3))
+    [1, 2, 3]
+    """
+    def simple(self, *args):
+        for i in args:
+            yield i
+
+def generator_nonlocal():
+    """
+    >>> g = generator_nonlocal()
+    >>> list(g(5))
+    [2, 3, 4, 5, 6]
+    """
+    def f(x):
+        def g(y):
+            nonlocal x
+            for i in range(y):
+                x += 1
+                yield x
+        return g
+    return f(1)
+
+def test_nested(a, b, c):
+    """
+    >>> obj = test_nested(1, 2, 3)
+    >>> [i() for i in obj]
+    [1, 2, 3, 4]
+    """
+    def one():
+        return a
+    def two():
+        return b
+    def three():
+        return c
+    def new_closure(a, b):
+        def sum():
+            return a + b
+        return sum
+    yield one
+    yield two
+    yield three
+    yield new_closure(a, c)
+
+
+def tolist(func):
+    def wrapper(*args, **kwargs):
+        return list(func(*args, **kwargs))
+    return wrapper
+
+@tolist
+def test_decorated(*args):
+    """
+    >>> test_decorated(1, 2, 3)
+    [1, 2, 3]
+    """
+    for i in args:
+        yield i
+
+def test_return(a):
+    """
+    >>> d = dict()
+    >>> obj = test_return(d)
+    >>> next(obj)
+    1
+    >>> next(obj)
+    Traceback (most recent call last):
+    StopIteration
+    >>> d['i_was_here']
+    True
+    """
+    yield 1
+    a['i_was_here'] = True
+    return
+
+def test_copied_yield(foo):
+    """
+    >>> class Manager(object):
+    ...    def __enter__(self):
+    ...        return self
+    ...    def __exit__(self, type, value, tb):
+    ...        pass
+    >>> list(test_copied_yield(Manager()))
+    [1]
+    """
+    with foo:
+        yield 1
+
+def test_nested_yield():
+    """
+    >>> obj = test_nested_yield()
+    >>> next(obj)
+    1
+    >>> obj.send(2)
+    2
+    >>> obj.send(3)
+    3
+    >>> obj.send(4)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    yield (yield (yield 1))
+
+def test_inside_lambda():
+    """
+    >>> obj = test_inside_lambda()()
+    >>> next(obj)
+    1
+    >>> obj.send('a')
+    2
+    >>> obj.send('b')
+    ('a', 'b')
+    """
+    return lambda:((yield 1), (yield 2))
+
+def test_nested_gen(int n):
+    """
+    >>> [list(a) for a in test_nested_gen(5)]
+    [[], [0], [0, 1], [0, 1, 2], [0, 1, 2, 3]]
+    """
+    for a in range(n):
+        yield (b for b in range(a))
+
+def test_lambda(n):
+    """
+    >>> [i() for i in test_lambda(3)]
+    [0, 1, 2]
+    """
+    for i in range(n):
+        yield lambda : i

tests/run/nonlocal_T490.pyx

+def simple():
+    """
+    >>> simple()
+    1
+    2
+    """
+    x = 1
+    y = 2
+    def f():
+        nonlocal x
+        nonlocal x, y
+        print(x)
+        print(y)
+    f()
+
+def assign():
+    """
+    >>> assign()
+    1
+    """
+    xx = 0
+    def ff():
+        nonlocal xx
+        xx += 1
+        print(xx)
+    ff()
+
+def nested():
+    """
+    >>> nested()
+    1
+    """
+    x = 0
+    def fx():
+        def gx():
+            nonlocal x
+            x=1
+            print(x)
+        return gx
+    fx()()
+
+def arg(x):
+    """
+    >>> arg('x')
+    xyy
+    """
+    def appendy():
+        nonlocal x
+        x += 'y'
+    x+='y'
+    appendy()
+    print x
+    return
+
+def argtype(int n):
+    """
+    >>> argtype(0)
+    1
+    """
+    def inc():
+        nonlocal n
+        n += 1
+    inc()
+    print n
+    return
+
+def ping_pong():
+    """
+    >>> f = ping_pong()
+    >>> inc, dec = f(0)
+    >>> inc()
+    1
+    >>> inc()
+    2
+    >>> dec()
+    1
+    >>> inc()
+    2
+    >>> dec()
+    1
+    >>> dec()
+    0
+    """
+    def f(x):
+        def inc():
+            nonlocal x
+            x += 1
+            return x
+        def dec():
+            nonlocal x
+            x -= 1
+            return x
+        return inc, dec
+    return f
+
+def methods():
+    """
+    >>> f = methods()
+    >>> c = f(0)
+    >>> c.inc()
+    1
+    >>> c.inc()
+    2
+    >>> c.dec()
+    1
+    >>> c.dec()
+    0
+    """
+    def f(x):
+        class c:
+            def inc(self):
+                nonlocal x
+                x += 1
+                return x
+            def dec(self):
+                nonlocal x
+                x -= 1
+                return x
+        return c()
+    return f
+
+def class_body(int x, y):
+    """
+    >>> c = class_body(2,99)
+    >>> c.z
+    (3, 2)
+    >>> c.x     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    AttributeError: ...
+    >>> c.y     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    AttributeError: ...
+    """
+    class c(object):
+        nonlocal x
+        nonlocal y
+        y = 2
+        x += 1
+        z = x,y
+    return c()
+
+def nested_nonlocals(x):
+    """
+    >>> g = nested_nonlocals(1)
+    >>> h = g()
+    >>> h()
+    3
+    """
+    def g():
+        nonlocal x
+        x -= 2
+        def h():
+            nonlocal x
+            x += 4
+            return x
+        return h
+    return g