Merge branch 'master' of github.com:vitek/cython

Cython/Compiler/CmdLine.py

@@ -10,15 +10,15 @@ usage = """\
 Cython (http://cython.org) is a compiler for code written in the
 Cython language.  Cython is based on Pyrex by Greg Ewing.
 
-Usage: cython [options] sourcefile.pyx ...
+Usage: cython [options] sourcefile.{pyx,py} ...
 
 Options:
   -V, --version                  Display version number of cython compiler
   -l, --create-listing           Write error messages to a listing file
   -I, --include-dir <directory>  Search for include files in named directory
-                                 (multiply include directories are allowed).
+                                 (multiple include directories are allowed).
   -o, --output-file <filename>   Specify name of generated C file
-  -t, --timestamps               Only compile newer source files (implied with -r)
+  -t, --timestamps               Only compile newer source files
   -f, --force                    Compile all source files (overrides implied -t)
   -q, --quiet                    Don't print module names in recursive mode
   -v, --verbose                  Be verbose, print file names on multiple compilation
@@ -30,11 +30,11 @@ Options:
                                  are searched from)
   --gdb                          Output debug information for cygdb
 
-  -D, --no-docstrings            Remove docstrings.
+  -D, --no-docstrings            Strip docstrings from the compiled module.
   -a, --annotate                 Produce a colorized HTML version of the source.
   --line-directives              Produce #line directives pointing to the .pyx source
-  --cplus                        Output a c++ rather than c file.
-  --embed                        Embed the Python interpreter in a main() method.
+  --cplus                        Output a C++ rather than C file.
+  --embed                        Generate a main() function that embeds the Python interpreter.
   -2                             Compile based on Python-2 syntax and code semantics.
   -3                             Compile based on Python-3 syntax and code semantics.
   --fast-fail                    Abort the compilation on the first error
@@ -42,7 +42,7 @@ Options:
 """
 
 # The following is broken http://trac.cython.org/cython_trac/ticket/379
-#  -r, --recursive                Recursively find and compile dependencies
+#  -r, --recursive                Recursively find and compile dependencies (implies -t)
 
 
 #The following experimental options are supported only on MacOSX:
@@ -143,6 +143,9 @@ def parse_command_line(args):
                 else:
                     sys.stderr.write("Unknown debug flag: %s\n" % option)
                     bad_usage()
+            elif option in ('-h', '--help'):
+                sys.stdout.write(usage)
+                sys.exit(0)
             else:
                 sys.stderr.write("Unknown compiler flag: %s\n" % option)
                 sys.exit(1)

Cython/Compiler/Code.py

@@ -1361,7 +1361,7 @@ class CCodeWriter(object):
         return self.globalstate.lookup_filename(filename)
 
     def put_declare_refcount_context(self):
-        self.putln('__Pyx_RefNannyDeclareContext')
+        self.putln('__Pyx_RefNannyDeclareContext;')
 
     def put_setup_refcount_context(self, name):
         self.putln('__Pyx_RefNannySetupContext("%s");' % name)

Cython/Compiler/ExprNodes.py

@@ -954,7 +954,8 @@ class BytesNode(ConstNode):
     #
     # value      BytesLiteral
 
-    type = PyrexTypes.c_char_ptr_type
+    # start off as Python 'bytes' to support len() in O(1)
+    type = bytes_type
 
     def compile_time_value(self, denv):
         return self.value
@@ -975,11 +976,13 @@ class BytesNode(ConstNode):
         return len(self.value) == 1
 
     def coerce_to_boolean(self, env):
-        # This is special because we start off as a C char*.  Testing
-        # that for truth directly would yield the wrong result.
+        # This is special because testing a C char* for truth directly
+        # would yield the wrong result.
         return BoolNode(self.pos, value=bool(self.value))
 
     def coerce_to(self, dst_type, env):
+        if self.type == dst_type:
+            return self
         if dst_type.is_int:
             if not self.can_coerce_to_char_literal():
                 error(self.pos, "Only single-character string literals can be coerced into ints.")
@@ -990,21 +993,20 @@ class BytesNode(ConstNode):
             return CharNode(self.pos, value=self.value)
 
         node = BytesNode(self.pos, value=self.value)
-        if dst_type == PyrexTypes.c_char_ptr_type:
-            node.type = PyrexTypes.c_char_ptr_type
+        if dst_type.is_pyobject:
+            if dst_type in (py_object_type, Builtin.bytes_type):
+                node.type = Builtin.bytes_type
+            else:
+                self.check_for_coercion_error(dst_type, fail=True)
+                return node
+        elif dst_type == PyrexTypes.c_char_ptr_type:
+            node.type = dst_type
             return node
         elif dst_type == PyrexTypes.c_uchar_ptr_type:
             node.type = PyrexTypes.c_char_ptr_type
             return CastNode(node, PyrexTypes.c_uchar_ptr_type)
-
-        if not self.type.is_pyobject:
-            if dst_type in (py_object_type, Builtin.bytes_type):
-                node.type = Builtin.bytes_type
-            elif dst_type.is_pyobject:
-                self.fail_assignment(dst_type)
-                return self
-        elif dst_type.is_pyobject and dst_type is not py_object_type:
-            self.check_for_coercion_error(dst_type, fail=True)
+        elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type):
+            node.type = dst_type
             return node
 
         # We still need to perform normal coerce_to processing on the
@@ -1012,11 +1014,6 @@ class BytesNode(ConstNode):
         # in which case a type test node will be needed.
         return ConstNode.coerce_to(node, dst_type, env)
 
-    def as_py_string_node(self, env):
-        # Return a new BytesNode with the same value as this node
-        # but whose type is a Python type instead of a C type.
-        return BytesNode(self.pos, value = self.value, type = Builtin.bytes_type)
-
     def generate_evaluation_code(self, code):
         if self.type.is_pyobject:
             self.result_code = code.get_py_string_const(self.value)
@@ -2043,7 +2040,7 @@ class IndexNode(ExprNode):
         return None
 
     def type_dependencies(self, env):
-        return self.base.type_dependencies(env)
+        return self.base.type_dependencies(env) + self.index.type_dependencies(env)
 
     def infer_type(self, env):
         base_type = self.base.infer_type(env)
@@ -2969,9 +2966,14 @@ class SimpleCallNode(CallNode):
                 arg = arg.coerce_to_temp(env)
             self.args[i] = arg
         for i in range(max_nargs, actual_nargs):
-            if self.args[i].type.is_pyobject:
+            arg = self.args[i]
+            if arg.type.is_pyobject:
+                arg_ctype = arg.type.default_coerced_ctype()
+                if arg_ctype is None:
                     error(self.args[i].pos,
                           "Python object cannot be passed as a varargs parameter")
+                else:
+                    self.args[i] = arg.coerce_to(arg_ctype, env)
         # Calc result type and code fragment
         if isinstance(self.function, NewExprNode):
             self.type = PyrexTypes.CPtrType(self.function.class_type)
@@ -4341,15 +4343,18 @@ class DictComprehensionAppendNode(ComprehensionAppendNode):
         self.value_expr.annotate(code)
 
 
-class GeneratorExpressionNode(ScopedExprNode):
-    # A generator expression, e.g.  (i for i in range(10))
-    #
-    # Result is a generator.
+class InlinedGeneratorExpressionNode(ScopedExprNode):
+    # An inlined generator expression for which the result is
+    # calculated inside of the loop.  This will only be created by
+    # transforms when replacing builtin calls on generator
+    # expressions.
     #
-    # loop      ForStatNode   the for-loop, containing a YieldExprNode
+    # loop           ForStatNode      the for-loop, not containing any YieldExprNodes
+    # result_node    ResultRefNode    the reference to the result value temp
+    # orig_func      String           the name of the builtin function this node replaces
 
     child_attrs = ["loop"]
-
+    loop_analysed = False
     type = py_object_type
 
     def analyse_scoped_declarations(self, env):
@@ -4360,30 +4365,12 @@ class GeneratorExpressionNode(ScopedExprNode):
             self.loop.analyse_expressions(env)
         self.is_temp = True
 
-    def analyse_scoped_expressions(self, env):
-        if self.has_local_scope:
-            self.loop.analyse_expressions(env)
-
     def may_be_none(self):
         return False
 
     def annotate(self, code):
         self.loop.annotate(code)
 
-
-class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
-    # An inlined generator expression for which the result is
-    # calculated inside of the loop.  This will only be created by
-    # transforms when replacing builtin calls on generator
-    # expressions.
-    #
-    # loop           ForStatNode      the for-loop, not containing any YieldExprNodes
-    # result_node    ResultRefNode    the reference to the result value temp
-    # orig_func      String           the name of the builtin function this node replaces
-
-    child_attrs = ["loop"]
-    loop_analysed = False
-
     def infer_type(self, env):
         return self.result_node.infer_type(env)
 
@@ -4396,7 +4383,8 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
 
     def analyse_scoped_expressions(self, env):
         self.loop_analysed = True
-        GeneratorExpressionNode.analyse_scoped_expressions(self, env)
+        if self.has_local_scope:
+            self.loop.analyse_expressions(env)
 
     def coerce_to(self, dst_type, env):
         if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed:
@@ -4407,7 +4395,7 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
             # assignments.
             self.result_node.type = self.type = dst_type
             return self
-        return GeneratorExpressionNode.coerce_to(self, dst_type, env)
+        return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env)
 
     def generate_result_code(self, code):
         self.result_node.result_code = self.result()
@@ -4955,6 +4943,34 @@ class LambdaNode(InnerFunctionNode):
         env.add_lambda_def(self.def_node)
 
 
+class GeneratorExpressionNode(LambdaNode):
+    # A generator expression, e.g.  (i for i in range(10))
+    #
+    # Result is a generator.
+    #
+    # loop      ForStatNode   the for-loop, containing a YieldExprNode
+    # def_node  DefNode       the underlying generator 'def' node
+
+    name = StringEncoding.EncodedString('genexpr')
+    binding = False
+
+    def analyse_declarations(self, env):
+        # XXX: dirty hack to disable assignment synthesis
+        self.def_node.needs_assignment_synthesis = lambda *args, **kwargs: False
+        self.def_node.analyse_declarations(env)
+        #super(GeneratorExpressionNode, self).analyse_declarations(env)
+        env.add_lambda_def(self.def_node)
+
+    def generate_result_code(self, code):
+        code.putln(
+            '%s = %s(%s, NULL); %s' % (
+                self.result(),
+                self.def_node.entry.func_cname,
+                self.self_result_code(),
+                code.error_goto_if_null(self.result(), self.pos)))
+        code.put_gotref(self.py_result())
+
+
 class YieldExprNode(ExprNode):
     # Yield expression node
     #

Cython/Compiler/ModuleNode.py

@@ -2051,12 +2051,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                                        code.error_goto_if_null(type.typeptr_cname, pos))
         self.use_type_import_utility_code(env)
         if type.vtabptr_cname:
-            code.putln(
-                "if (__Pyx_GetVtable(%s->tp_dict, &%s) < 0) %s" % (
-                    type.typeptr_cname,
-                    type.vtabptr_cname,
-                    code.error_goto(pos)))
             env.use_utility_code(Nodes.get_vtable_utility_code)
+            code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % (
+                type.vtabptr_cname,
+                type.vtabstruct_cname,
+                type.typeptr_cname,
+                code.error_goto_if_null(type.vtabptr_cname, pos)))
         env.types_imported[type] = 1
 
     py3_type_name_map = {'str' : 'bytes', 'unicode' : 'str'}
@@ -2109,24 +2109,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 # a significant performance hit. (See trac #561.)
                 for func in entry.type.scope.pyfunc_entries:
                     if func.is_special and Options.docstrings and func.wrapperbase_cname:
-                        code.putln("{");
+                        code.putln("{")
                         code.putln(
                             'PyObject *wrapper = __Pyx_GetAttrString((PyObject *)&%s, "%s"); %s' % (
                                 typeobj_cname,
                                 func.name,
-                                code.error_goto_if_null('wrapper', entry.pos)));
+                                code.error_goto_if_null('wrapper', entry.pos)))
                         code.putln(
-                            "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {");
+                            "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {")
                         code.putln(
                             "%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % (
-                                func.wrapperbase_cname));
+                                func.wrapperbase_cname))
                         code.putln(
-                            "%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname));
+                            "%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname))
                         code.putln(
                             "((PyWrapperDescrObject *)wrapper)->d_base = &%s;" % (
-                                func.wrapperbase_cname));
-                        code.putln("}");
-                        code.putln("}");
+                                func.wrapperbase_cname))
+                        code.putln("}")
+                        code.putln("}")
                 if type.vtable_cname:
                     code.putln(
                         "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % (

Cython/Compiler/Nodes.py

@@ -1167,6 +1167,7 @@ class FuncDefNode(StatNode, BlockNode):
     needs_closure = False
     needs_outer_scope = False
     is_generator = False
+    is_generator_body = False
     modifiers = []
 
     def analyse_default_values(self, env):
@@ -1210,6 +1211,9 @@ class FuncDefNode(StatNode, BlockNode):
         lenv.directives = env.directives
         return lenv
 
+    def generate_function_body(self, env, code):
+        self.body.generate_execution_code(code)
+
     def generate_function_definitions(self, env, code):
         import Buffer
 
@@ -1297,7 +1301,6 @@ class FuncDefNode(StatNode, BlockNode):
                      init))
         tempvardecl_code = code.insertion_point()
         code.put_declare_refcount_context()
-        if not self.is_generator:
         self.generate_keyword_list(code)
         if profile:
             code.put_trace_declarations()
@@ -1317,14 +1320,7 @@ class FuncDefNode(StatNode, BlockNode):
         if is_getbuffer_slot:
             self.getbuffer_init(code)
         # ----- Create closure scope object
-        if self.is_generator:
-            code.putln("%s = (%s) %s;" % (
-                Naming.cur_scope_cname,
-                lenv.scope_class.type.declaration_code(''),
-                Naming.self_cname))
-            gotref_code = code.insertion_point()
-
-        elif self.needs_closure:
+        if self.needs_closure:
             code.putln("%s = (%s)%s->tp_new(%s, %s, NULL);" % (
                 Naming.cur_scope_cname,
                 lenv.scope_class.type.declaration_code(''),
@@ -1341,7 +1337,7 @@ class FuncDefNode(StatNode, BlockNode):
             code.putln("}")
             code.put_gotref(Naming.cur_scope_cname)
             # Note that it is unsafe to decref the scope at this point.
-        if self.needs_outer_scope and not self.is_generator:
+        if self.needs_outer_scope:
             code.putln("%s = (%s)%s;" % (
                             outer_scope_cname,
                             cenv.scope_class.type.declaration_code(''),
@@ -1358,23 +1354,33 @@ class FuncDefNode(StatNode, BlockNode):
             # fatal error before hand, it's not really worth tracing
             code.put_trace_call(self.entry.name, self.pos)
         # ----- Fetch arguments
-        if not self.is_generator:
-            self.generate_preamble(env, code)
-        if self.is_generator:
-            code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
-            resume_code = code.insertion_point()
-            first_run_label = code.new_label('first_run')
-            code.use_label(first_run_label)
-            code.put_label(first_run_label)
-            code.putln('%s' %
-                       (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))
+        self.generate_argument_parsing_code(env, code)
+        # If an argument is assigned to in the body, we must
+        # incref it to properly keep track of refcounts.
+        for entry in lenv.arg_entries:
+            if entry.type.is_pyobject:
+                if (acquire_gil or entry.assignments) and not entry.in_closure:
+                    code.put_var_incref(entry)
+        # ----- Initialise local variables
+        for entry in lenv.var_entries:
+            if entry.type.is_pyobject and entry.init_to_none and entry.used:
+                code.put_init_var_to_py_none(entry)
+        # ----- Initialise local buffer auxiliary variables
+        for entry in lenv.var_entries + lenv.arg_entries:
+            if entry.type.is_buffer and entry.buffer_aux.buffer_info_var.used:
+                code.putln("%s.buf = NULL;" %
+                           entry.buffer_aux.buffer_info_var.cname)
+        # ----- Check and convert arguments
+        self.generate_argument_type_tests(code)
+        # ----- Acquire buffer arguments
+        for entry in lenv.arg_entries:
+            if entry.type.is_buffer:
+                Buffer.put_acquire_arg_buffer(entry, code, self.pos)
+
         # -------------------------
         # ----- Function body -----
         # -------------------------
-        self.body.generate_execution_code(code)
-
-        if self.is_generator:
-            code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
+        self.generate_function_body(env, code)
 
         # ----- Default return value
         code.putln("")
@@ -1461,10 +1467,8 @@ class FuncDefNode(StatNode, BlockNode):
             if entry.type.is_pyobject:
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_decref(entry)
-        if self.needs_closure and not self.is_generator:
+        if self.needs_closure:
             code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
-        if self.is_generator:
-            code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
 
         # ----- Return
         # This code is duplicated in ModuleNode.generate_module_init_func
@@ -1502,56 +1506,15 @@ class FuncDefNode(StatNode, BlockNode):
 
         if preprocessor_guard:
             code.putln("#endif /*!(%s)*/" % preprocessor_guard)
+
         # ----- Go back and insert temp variable declarations
         tempvardecl_code.put_temp_declarations(code.funcstate)
-        # ----- Generator resume code
-        if self.is_generator:
-            resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
-            resume_code.putln("case 0: goto %s;" % first_run_label)
-            for yield_expr in self.yields:
-                resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
-            resume_code.putln("default: /* CPython raises the right error here */");
-            resume_code.putln("return NULL;");
-            resume_code.putln("}");
         # ----- Python version
         code.exit_cfunc_scope()
         if self.py_func:
             self.py_func.generate_function_definitions(env, code)
         self.generate_wrapper_functions(code)
 
-        if self.is_generator:
-            self.generator.generate_function_body(self.local_scope, code)
-
-    def generate_preamble(self, env, code):
-        """Parse arguments and prepare scope"""
-        import Buffer
-
-        lenv = self.local_scope
-        acquire_gil = self.acquire_gil
-
-        self.generate_argument_parsing_code(env, code)
-        # If an argument is assigned to in the body, we must
-        # incref it to properly keep track of refcounts.
-        for entry in lenv.arg_entries:
-            if entry.type.is_pyobject:
-                if (acquire_gil or entry.assignments) and not entry.in_closure:
-                    code.put_var_incref(entry)
-        # ----- Initialise local variables
-        for entry in lenv.var_entries:
-            if entry.type.is_pyobject and entry.init_to_none and entry.used:
-                code.put_init_var_to_py_none(entry)
-        # ----- Initialise local buffer auxiliary variables
-        for entry in lenv.var_entries + lenv.arg_entries:
-            if entry.type.is_buffer and entry.buffer_aux.buffer_info_var.used:
-                code.putln("%s.buf = NULL;" %
-                           entry.buffer_aux.buffer_info_var.cname)
-        # ----- Check and convert arguments
-        self.generate_argument_type_tests(code)
-        # ----- Acquire buffer arguments
-        for entry in lenv.arg_entries:
-            if entry.type.is_buffer:
-                Buffer.put_acquire_arg_buffer(entry, code, self.pos)
-
     def declare_argument(self, env, arg):
         if arg.type.is_void:
             error(arg.pos, "Invalid use of 'void'")
@@ -1902,61 +1865,6 @@ class DecoratorNode(Node):
     child_attrs = ['decorator']
 
 
-class GeneratorWrapperNode(object):
-    # Wrapper
-    def __init__(self, def_node, func_cname=None, body_cname=None, header=None):
-        self.def_node = def_node
-        self.func_cname = func_cname
-        self.body_cname = body_cname
-        self.header = header
-
-    def generate_function_body(self, env, code):
-        code.mark_pos(self.def_node.pos)
-        cenv = env.outer_scope # XXX: correct?
-        while cenv.is_py_class_scope or cenv.is_c_class_scope:
-            cenv = cenv.outer_scope
-        lenv = self.def_node.local_scope
-        code.enter_cfunc_scope()
-        code.putln()
-        code.putln('%s {' % self.header)
-        code.put_declare_refcount_context()
-        self.def_node.generate_keyword_list(code)
-        code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
-        code.putln(";")
-        code.put_setup_refcount_context(self.def_node.entry.name)
-        code.putln("%s = (%s)%s->tp_new(%s, %s, NULL);" % (
-            Naming.cur_scope_cname,
-            lenv.scope_class.type.declaration_code(''),
-            lenv.scope_class.type.typeptr_cname,
-            lenv.scope_class.type.typeptr_cname,
-            Naming.empty_tuple))
-        code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
-        code.put_finish_refcount_context()
-        code.putln("return NULL;");
-        code.putln("}");
-        code.put_gotref(Naming.cur_scope_cname)
-
-        if self.def_node.needs_outer_scope:
-            outer_scope_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.outer_scope_cname)
-            code.putln("%s = (%s)%s;" % (
-                            outer_scope_cname,
-                            cenv.scope_class.type.declaration_code(''),
-                            Naming.self_cname))
-            code.put_incref(outer_scope_cname, cenv.scope_class.type)
-            code.put_giveref(outer_scope_cname)
-
-        self.def_node.generate_preamble(env, code)
-
-        generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
-
-        code.putln('%s.resume_label = 0;' % generator_cname)
-        code.putln('%s.body = %s;' % (generator_cname, self.body_cname))
-        code.put_giveref(Naming.cur_scope_cname)
-        code.put_finish_refcount_context()
-        code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
-        code.putln('}\n')
-        code.exit_cfunc_scope()
-
 class DefNode(FuncDefNode):
     # A Python function definition.
     #
@@ -2250,10 +2158,6 @@ class DefNode(FuncDefNode):
             Naming.pyfunc_prefix + prefix + name
         entry.pymethdef_cname = \
             Naming.pymethdef_prefix + prefix + name
-
-        if self.is_generator:
-            self.generator_body_cname = Naming.genbody_prefix + env.next_id(env.scope_prefix) + name
-
         if Options.docstrings:
             entry.doc = embed_position(self.pos, self.doc)
             entry.doc_cname = \
@@ -2401,14 +2305,6 @@ class DefNode(FuncDefNode):
                 "static PyMethodDef %s = " %
                     self.entry.pymethdef_cname)
             code.put_pymethoddef(self.entry, ";", allow_skip=False)
-        if self.is_generator:
-            code.putln("static PyObject *%s(PyObject *%s, PyObject *%s) /* generator body */\n{" %
-                       (self.generator_body_cname, Naming.self_cname, Naming.sent_value_cname))
-            self.generator = GeneratorWrapperNode(self,
-                                                  func_cname=self.entry.func_cname,
-                                                  body_cname=self.generator_body_cname,
-                                                  header=header)
-        else:
         code.putln("%s {" % header)
 
     def generate_argument_declarations(self, env, code):
@@ -2556,8 +2452,6 @@ class DefNode(FuncDefNode):
                     self.name, Naming.args_cname, self.error_value()))
             code.putln("}")
 
-        code.globalstate.use_utility_code(keyword_string_check_utility_code)
-
         if self.starstar_arg:
             if self.star_arg:
                 kwarg_check = "unlikely(%s)" % Naming.kwds_cname
@@ -2566,6 +2460,7 @@ class DefNode(FuncDefNode):
         else:
             kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
                 Naming.kwds_cname, Naming.kwds_cname)
+        code.globalstate.use_utility_code(keyword_string_check_utility_code)
         code.putln(
             "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
                 kwarg_check, Naming.kwds_cname, self.name,
@@ -2629,8 +2524,6 @@ class DefNode(FuncDefNode):
         has_fixed_positional_count = not self.star_arg and \
             min_positional_args == max_positional_args
 
-        code.globalstate.use_utility_code(raise_double_keywords_utility_code)
-        code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
         if self.num_required_kw_args:
             code.globalstate.use_utility_code(raise_keyword_required_utility_code)
 
@@ -2721,6 +2614,7 @@ class DefNode(FuncDefNode):
         if code.label_used(argtuple_error_label):
             code.put_goto(success_label)
             code.put_label(argtuple_error_label)
+            code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
             code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
                     self.name, has_fixed_positional_count,
                     min_positional_args, max_positional_args,
@@ -2842,6 +2736,7 @@ class DefNode(FuncDefNode):
                             # kwargs) that were passed into positional
                             # arguments up to this point
                             code.putln('else {')
+                            code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
                             code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
                                     self.name, has_fixed_positional_count,
                                     min_positional_args, max_positional_args, i))
@@ -3003,6 +2898,151 @@ class DefNode(FuncDefNode):
     def caller_will_check_exceptions(self):
         return 1
 
+
+class GeneratorDefNode(DefNode):
+    # Generator DefNode.
+    #
+    # gbody          GeneratorBodyDefNode
+    #
+
+    is_generator = True
+    needs_closure = True
+
+    child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "gbody"]
+
+    def __init__(self, **kwargs):
+        # XXX: don't actually needs a body
+        kwargs['body'] = StatListNode(kwargs['pos'], stats=[])
+        super(GeneratorDefNode, self).__init__(**kwargs)
+
+    def analyse_declarations(self, env):
+        super(GeneratorDefNode, self).analyse_declarations(env)
+        self.gbody.local_scope = self.local_scope
+        self.gbody.analyse_declarations(env)
+
+    def generate_function_body(self, env, code):
+        body_cname = self.gbody.entry.func_cname
+        generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
+
+        code.putln('%s.resume_label = 0;' % generator_cname)
+        code.putln('%s.body = (__pyx_generator_body_t) %s;' % (generator_cname, body_cname))
+        code.put_giveref(Naming.cur_scope_cname)
+        code.put_finish_refcount_context()
+        code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
+
+    def generate_function_definitions(self, env, code):
+        self.gbody.generate_function_header(code, proto=True)
+        super(GeneratorDefNode, self).generate_function_definitions(env, code)
+        self.gbody.generate_function_definitions(env, code)
+
+
+class GeneratorBodyDefNode(DefNode):
+    # Generator body DefNode.
+    #
+
+    is_generator_body = True
+
+    child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators"]
+
+    def __init__(self, pos=None, name=None, body=None):
+        super(GeneratorBodyDefNode, self).__init__(pos=pos, body=body, name=name, doc=None,
+                                                   args=[],
+                                                   star_arg=None, starstar_arg=None)
+
+    def declare_generator_body(self, env):
+        prefix = env.next_id(env.scope_prefix)
+        name = env.next_id('generator')
+        entry = env.declare_var(prefix + name, py_object_type, self.pos, visibility='private')
+        entry.func_cname = Naming.genbody_prefix + prefix + name
+        entry.qualified_name = EncodedString(self.name)
+        self.entry = entry
+
+    def analyse_declarations(self, env):
+        self.analyse_argument_types(env)
+        self.declare_generator_body(env)
+
+    def generate_function_header(self, code, proto=False):
+        header = "static PyObject *%s(%s, PyObject *%s)" % (
+            self.entry.func_cname,
+            self.local_scope.scope_class.type.declaration_code(Naming.cur_scope_cname),
+            Naming.sent_value_cname)
+        if proto:
+            code.putln('%s; /* proto */' % header)
+        else:
+            code.putln('%s /* generator body */\n{' % header);
+
+    def generate_function_definitions(self, env, code):
+        lenv = self.local_scope
+
+        # Generate closure function definitions
+        self.body.generate_function_definitions(lenv, code)
+        # generate lambda function definitions
+        self.generate_lambda_definitions(lenv, code)
+
+        # Generate C code for header and body of function
+        code.enter_cfunc_scope()
+        code.return_from_error_cleanup_label = code.new_label()
+
+        # ----- Top-level constants used by this function
+        code.mark_pos(self.pos)
+        self.generate_cached_builtins_decls(lenv, code)
+        # ----- Function header
+        code.putln("")
+        self.generate_function_header(code)
+        # ----- Local variables
+        code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
+        tempvardecl_code = code.insertion_point()
+        code.put_declare_refcount_context()
+        code.put_setup_refcount_context(self.entry.name)
+
+        # ----- Resume switch point.
+        code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
+        resume_code = code.insertion_point()
+        first_run_label = code.new_label('first_run')
+        code.use_label(first_run_label)
+        code.put_label(first_run_label)
+        code.putln('%s' %
+                   (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))
+
+        # ----- Function body
+        self.generate_function_body(env, code)
+        code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
+        # ----- Error cleanup
+        if code.error_label in code.labels_used:
+            code.put_goto(code.return_label)
+            code.put_label(code.error_label)
+            for cname, type in code.funcstate.all_managed_temps():
+                code.put_xdecref(cname, type)
+            code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
+            # XXX: ^^^ is this enough?
+
+        # ----- Non-error return cleanup
+        code.put_label(code.return_label)
+        code.put_xdecref(Naming.retval_cname, py_object_type)
+        code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
+        code.put_finish_refcount_context()
+        code.putln('return NULL;');
+        code.putln("}")
+
+        # ----- Go back and insert temp variable declarations
+        tempvardecl_code.put_temp_declarations(code.funcstate)
+        # ----- Generator resume code
+        resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
+        resume_code.putln("case 0: goto %s;" % first_run_label)
+
+        from ParseTreeTransforms import YieldNodeCollector
+        collector = YieldNodeCollector()
+        collector.visitchildren(self)
+        for yield_expr in collector.yields:
+            resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
+        resume_code.putln("default: /* CPython raises the right error here */");
+        resume_code.put_finish_refcount_context()
+        resume_code.putln("return NULL;");
+        resume_code.putln("}");
+
+        code.exit_cfunc_scope()
+
+
 class OverrideCheckNode(StatNode):
     # A Node for dispatching to the def method if it
     # is overriden.
@@ -6237,7 +6277,8 @@ invalid_keyword:
 bad:
     return -1;
 }
-""")
+""",
+requires=[raise_double_keywords_utility_code])
 
 #------------------------------------------------------------------------------------
 
@@ -6381,25 +6422,26 @@ bad:
 
 get_vtable_utility_code = UtilityCode(
 proto = """
-static int __Pyx_GetVtable(PyObject *dict, void *vtabptr); /*proto*/
+static void* __Pyx_GetVtable(PyObject *dict); /*proto*/
 """,
 impl = r"""
-static int __Pyx_GetVtable(PyObject *dict, void *vtabptr) {
+static void* __Pyx_GetVtable(PyObject *dict) {
+    void* ptr;
     PyObject *ob = PyMapping_GetItemString(dict, (char *)"__pyx_vtable__");
     if (!ob)
         goto bad;
 #if PY_VERSION_HEX >= 0x02070000 && !(PY_MAJOR_VERSION==3&&PY_MINOR_VERSION==0)
-    *(void **)vtabptr = PyCapsule_GetPointer(ob, 0);
+    ptr = PyCapsule_GetPointer(ob, 0);
 #else
-    *(void **)vtabptr = PyCObject_AsVoidPtr(ob);
+    ptr = PyCObject_AsVoidPtr(ob);
 #endif
-    if (!*(void **)vtabptr)
-        goto bad;
+    if (!ptr && !PyErr_Occurred())
+        PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type");
     Py_DECREF(ob);
-    return 0;
+    return ptr;
 bad:
     Py_XDECREF(ob);
-    return -1;
+    return NULL;
 }
 """)
 

Cython/Compiler/Optimize.py

@@ -1167,11 +1167,12 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform):
             self.yield_nodes = []
 
         visit_Node = Visitor.TreeVisitor.visitchildren
-        def visit_YieldExprNode(self, node):
+        # XXX: disable inlining while it's not back supported
+        def __visit_YieldExprNode(self, node):
             self.yield_nodes.append(node)
             self.visitchildren(node)
 
-        def visit_ExprStatNode(self, node):
+        def __visit_ExprStatNode(self, node):
             self.visitchildren(node)
             if node.expr in self.yield_nodes:
                 self.yield_stat_nodes[node.expr] = node

Cython/Compiler/ParseTreeTransforms.py

@@ -183,6 +183,7 @@ class PostParse(ScopeTrackingTransform):
 
     def visit_ModuleNode(self, node):
         self.lambda_counter = 1
+        self.genexpr_counter = 1
         return super(PostParse, self).visit_ModuleNode(node)
 
     def visit_LambdaNode(self, node):
@@ -190,14 +191,34 @@ class PostParse(ScopeTrackingTransform):
         lambda_id = self.lambda_counter
         self.lambda_counter += 1
         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
-
+        collector = YieldNodeCollector()
+        collector.visitchildren(node.result_expr)
+        if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
+            body = ExprNodes.YieldExprNode(
+                node.result_expr.pos, arg=node.result_expr)
+            body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
+        else:
             body = Nodes.ReturnStatNode(
-            node.result_expr.pos, value = node.result_expr)
+                node.result_expr.pos, value=node.result_expr)
         node.def_node = Nodes.DefNode(
             node.pos, name=node.name, lambda_name=node.lambda_name,
             args=node.args, star_arg=node.star_arg,
             starstar_arg=node.starstar_arg,
-            body=body)
+            body=body, doc=None)
+        self.visitchildren(node)
+        return node
+
+    def visit_GeneratorExpressionNode(self, node):
+        # unpack a generator expression into the corresponding DefNode
+        genexpr_id = self.genexpr_counter
+        self.genexpr_counter += 1
+        node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
+
+        node.def_node = Nodes.DefNode(node.pos, name=node.name,
+                                      doc=None,
+                                      args=[], star_arg=None,
+                                      starstar_arg=None,
+                                      body=node.loop)
         self.visitchildren(node)
         return node
 
@@ -1317,7 +1338,7 @@ class YieldNodeCollector(TreeVisitor):
         if self.has_return_value:
             error(node.pos, "'yield' outside function")
         self.yields.append(node)
-        node.label_num = len(self.yields)
+        self.visitchildren(node)
 
     def visit_ReturnStatNode(self, node):
         if node.value:
@@ -1332,6 +1353,9 @@ class YieldNodeCollector(TreeVisitor):
     def visit_DefNode(self, node):
         pass
 
+    def visit_LambdaNode(self, node):
+        pass
+
     def visit_GeneratorExpressionNode(self, node):
         pass
 
@@ -1352,11 +1376,22 @@ class MarkClosureVisitor(CythonTransform):
         collector.visitchildren(node)
 
         if collector.yields:
-            if collector.returns and not collector.has_return_value:
-                error(collector.returns[0].pos, "'return' inside generators not yet supported ")
-            node.is_generator = True
-            node.needs_closure = True
-            node.yields = collector.yields
+            for i, yield_expr in enumerate(collector.yields):
+                yield_expr.label_num = i + 1
+
+            gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
+                                               name=node.name,
+                                               body=node.body)
+            generator = Nodes.GeneratorDefNode(pos=node.pos,
+                                               name=node.name,
+                                               args=node.args,
+                                               star_arg=node.star_arg,
+                                               starstar_arg=node.starstar_arg,
+                                               doc=node.doc,
+                                               decorators=node.decorators,
+                                               gbody=gbody,
+                                               lambda_name=node.lambda_name)
+            return generator
         return node
 
     def visit_CFuncDefNode(self, node):
@@ -1447,6 +1482,9 @@ class CreateClosureClasses(CythonTransform):
         return from_closure, in_closure
 
     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
+        # skip generator body
+        if node.is_generator_body:
+            return
         # move local variables into closure
         if node.is_generator:
             for entry in node.local_scope.entries.values():

Cython/Compiler/PyrexTypes.py

@@ -353,6 +353,10 @@ class PyObjectType(PyrexType):
     def can_coerce_to_pyobject(self, env):
         return True
 
+    def default_coerced_ctype(self):
+        "The default C type that this Python type coerces to, or None."
+        return None
+
     def assignable_from(self, src_type):
         # except for pointers, conversion will be attempted
         return not src_type.is_ptr or src_type.is_string
@@ -404,6 +408,15 @@ class BuiltinObjectType(PyObjectType):
     def __repr__(self):
         return "<%s>"% self.cname
 
+    def default_coerced_ctype(self):
+        if self.name == 'bytes':
+            return c_char_ptr_type
+        elif self.name == 'bool':
+            return c_bint_type
+        elif self.name == 'float':
+            return c_double_type
+        return None
+
     def assignable_from(self, src_type):
         if isinstance(src_type, BuiltinObjectType):
             return src_type.name == self.name

tests/bugs.txt

@@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408
 compile.cpp_operators
 cpp_templated_ctypedef
 cpp_structs
-genexpr_T491
 with_statement_module_level_T536
 function_as_method_T494
 closure_inside_cdef_T554

tests/errors/e_generators.pyx

@@ -6,10 +6,6 @@ def bar(a):
     return 0
     yield
 
-def xxx():
-    yield
-    return
-
 yield
 
 class Foo:
@@ -18,7 +14,6 @@ class Foo:
 _ERRORS = u"""
 3:4: 'return' with argument inside generator
 7:4: 'yield' outside function
-11:4: 'return' inside generators not yet supported
-13:0: 'yield' not supported here
-16:4: 'yield' not supported here
+9:0: 'yield' not supported here
+12:4: 'yield' not supported here
 """

tests/errors/string_assignments.pyx

@@ -58,18 +58,18 @@ _ERRORS = u"""
 30:22: Cannot convert Unicode string to 'bytes' implicitly, encoding required.
 31:22: Cannot convert 'str' to 'bytes' implicitly. This is not portable.
 
-33:17: Cannot assign type 'char *' to 'str object'
+33:17: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
 34:19: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
 35:17: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
 36:19: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
 
 38:20: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
 39:22: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
-40:20: Cannot assign type 'char *' to 'unicode object'
+40:20: Cannot convert 'bytes' object to unicode implicitly, decoding required
 41:22: Cannot convert 'bytes' object to unicode implicitly, decoding required
 42:22: Cannot convert 'char*' to unicode implicitly, decoding required
 
 44:19: Cannot assign type 'str object' to 'tuple object'
 45:18: Cannot assign type 'unicode object' to 'tuple object'
-46:18: Cannot assign type 'char *' to 'tuple object'
+46:18: Cannot assign type 'bytes object' to 'tuple object'
 """

tests/run/generators.pyx

@@ -207,3 +207,95 @@ def generator_nonlocal():
                 yield x
         return g
     return f(1)
+
+def test_nested(a, b, c):
+    """
+    >>> obj = test_nested(1, 2, 3)
+    >>> [i() for i in obj]
+    [1, 2, 3, 4]
+    """
+    def one():
+        return a
+    def two():
+        return b
+    def three():
+        return c
+    def new_closure(a, b):
+        def sum():
+            return a + b
+        return sum
+    yield one
+    yield two
+    yield three
+    yield new_closure(a, c)
+
+
+def tolist(func):
+    def wrapper(*args, **kwargs):
+        return list(func(*args, **kwargs))
+    return wrapper
+
+@tolist
+def test_decorated(*args):
+    """
+    >>> test_decorated(1, 2, 3)
+    [1, 2, 3]
+    """
+    for i in args:
+        yield i
+
+def test_return(a):
+    """
+    >>> d = dict()
+    >>> obj = test_return(d)
+    >>> next(obj)
+    1
+    >>> next(obj)
+    Traceback (most recent call last):
+    StopIteration
+    >>> d['i_was_here']
+    True
+    """
+    yield 1
+    a['i_was_here'] = True
+    return
+
+def test_copied_yield(foo):
+    """
+    >>> class Manager(object):
+    ...    def __enter__(self):
+    ...        return self
+    ...    def __exit__(self, type, value, tb):
+    ...        pass
+    >>> list(test_copied_yield(Manager()))
+    [1]
+    """
+    with foo:
+        yield 1
+
+def test_nested_yield():
+    """
+    >>> obj = test_nested_yield()
+    >>> next(obj)
+    1
+    >>> obj.send(2)
+    2
+    >>> obj.send(3)
+    3
+    >>> obj.send(4)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    yield (yield (yield 1))
+
+def test_inside_lambda():
+    """
+    >>> obj = test_inside_lambda()()
+    >>> next(obj)
+    1
+    >>> obj.send('a')
+    2
+    >>> obj.send('b')
+    ('a', 'b')
+    """
+    return lambda:((yield 1), (yield 2))

tests/run/type_inference.pyx

@@ -25,7 +25,7 @@ def simple():
     xptrptr = &xptr
     assert typeof(xptrptr) == "double **", typeof(xptrptr)
     b = b"abc"
-    assert typeof(b) == "char *", typeof(b)
+    assert typeof(b) == "bytes object", typeof(b)
     s = "abc"
     assert typeof(s) == "str object", typeof(s)
     u = u"xyz"
@@ -57,7 +57,7 @@ def slicing():
     >>> slicing()
     """
     b = b"abc"
-    assert typeof(b) == "char *", typeof(b)
+    assert typeof(b) == "bytes object", typeof(b)
     b1 = b[1:2]
     assert typeof(b1) == "bytes object", typeof(b1)
     b2 = b[1:2:2]
@@ -92,9 +92,9 @@ def indexing():
     >>> indexing()
     """
     b = b"abc"
-    assert typeof(b) == "char *", typeof(b)
+    assert typeof(b) == "bytes object", typeof(b)
     b1 = b[1]
-    assert typeof(b1) == "char", typeof(b1)  # FIXME: Python object ??
+    assert typeof(b1) == "Python object", typeof(b1)
     u = u"xyz"
     assert typeof(u) == "unicode object", typeof(u)
     u1 = u[1]
@@ -479,3 +479,10 @@ def large_literals():
     c, d = 10, 100000000000000000000000000000000
     assert typeof(c) == "long", typeof(c)
     assert typeof(d) == "Python object", typeof(d)
+
+
+# Regression test for trac #638.
+
+def bar(foo):
+    qux = foo
+    quux = foo[qux.baz]