merged in latest cython-devel

Cython/Compiler/Annotate.py

@@ -44,7 +44,7 @@ class AnnotationCCodeWriter(CCodeWriter):
         if pos is not None:
             CCodeWriter.mark_pos(self, pos)
         if self.last_pos:
-            pos_code = self.code.setdefault(self.last_pos[0].get_description(),{})
+            pos_code = self.code.setdefault(self.last_pos[0].filename,{})
             code = pos_code.get(self.last_pos[1], "")
             pos_code[self.last_pos[1]] = code + self.annotation_buffer.getvalue()
         self.annotation_buffer = StringIO()

Cython/Compiler/Main.py

@@ -311,7 +311,10 @@ class Context(object):
                 try:
                     if debug_find_module:
                         print("Context.find_module: Parsing %s" % pxd_pathname)
-                    source_desc = FileSourceDescriptor(pxd_pathname)
+                    rel_path = module_name.replace('.', os.sep) + os.path.splitext(pxd_pathname)[1]
+                    if not pxd_pathname.endswith(rel_path):
+                        rel_path = pxd_pathname # safety measure to prevent printing incorrect paths
+                    source_desc = FileSourceDescriptor(pxd_pathname, rel_path)
                     err, result = self.process_pxd(source_desc, scope, module_name)
                     if err:
                         raise err
@@ -584,15 +587,23 @@ def run_pipeline(source, options, full_module_name = None):
 
     # Set up source object
     cwd = os.getcwd()
-    source_desc = FileSourceDescriptor(os.path.join(cwd, source))
+    abs_path = os.path.abspath(source)
+    source_ext = os.path.splitext(source)[1]
     full_module_name = full_module_name or context.extract_module_name(source, options)
+    if options.relative_path_in_code_position_comments:
+        rel_path = full_module_name.replace('.', os.sep) + source_ext
+        if not abs_path.endswith(rel_path):
+            rel_path = source # safety measure to prevent printing incorrect paths
+    else:
+        rel_path = abs_path
+    source_desc = FileSourceDescriptor(abs_path, rel_path)
     source = CompilationSource(source_desc, full_module_name, cwd)
 
     # Set up result object
     result = create_default_resultobj(source, options)
 
     # Get pipeline
-    if source_desc.filename.endswith(".py"):
+    if source_ext.lower() == '.py':
         pipeline = context.create_py_pipeline(options, result)
     else:
         pipeline = context.create_pyx_pipeline(options, result)
@@ -820,6 +831,7 @@ default_options = dict(
     compiler_directives = {},
     evaluate_tree_assertions = False,
     emit_linenums = False,
+    relative_path_in_code_position_comments = True,
     language_level = 2,
     gdb_debug = False,
 )

Cython/Compiler/ModuleNode.py

@@ -139,7 +139,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             h_code.putln("")
             h_code.putln("#endif")
 
-            h_code.copyto(open_new_file(result.h_file))
+            f = open_new_file(result.h_file)
+            try:
+                h_code.copyto(f)
+            finally:
+                f.close()
 
     def generate_public_declaration(self, entry, h_code, i_code):
         h_code.putln("%s %s;" % (
@@ -222,7 +226,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             h_code.putln("")
             h_code.putln("#endif")
 
-            h_code.copyto(open_new_file(result.api_file))
+            f = open_new_file(result.api_file)
+            try:
+                h_code.copyto(f)
+            finally:
+                f.close()
 
     def generate_cclass_header_code(self, type, h_code):
         h_code.putln("%s DL_IMPORT(PyTypeObject) %s;" % (

Cython/Compiler/Nodes.py

@@ -1975,6 +1975,7 @@ class DefNode(FuncDefNode):
     #  when the def statement is inside a Python class definition.
     #
     #  assmt   AssignmentNode   Function construction/assignment
+    #  py_cfunc_node  PyCFunctionNode/InnerFunctionNode   The PyCFunction to create and assign
 
     child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators"]
 
@@ -1989,6 +1990,7 @@ class DefNode(FuncDefNode):
     entry = None
     acquire_gil = 0
     self_in_stararg = 0
+    py_cfunc_node = None
 
     def __init__(self, pos, **kwds):
         FuncDefNode.__init__(self, pos, **kwds)
@@ -2340,15 +2342,23 @@ class DefNode(FuncDefNode):
             genv = genv.outer_scope
 
         if genv.is_closure_scope:
-            rhs = ExprNodes.InnerFunctionNode(
+            self.py_cfunc_node = ExprNodes.InnerFunctionNode(
                 self.pos, pymethdef_cname = self.entry.pymethdef_cname)
         else:
-            rhs = ExprNodes.PyCFunctionNode(
+            self.py_cfunc_node = ExprNodes.PyCFunctionNode(
                 self.pos, pymethdef_cname = self.entry.pymethdef_cname, binding = env.directives['binding'])
 
         if env.is_py_class_scope:
             if not self.is_staticmethod and not self.is_classmethod:
-                rhs.binding = True
+                self.py_cfunc_node.binding = True
+
+        rhs = self.py_cfunc_node
+        if self.decorators:
+            for decorator in self.decorators[::-1]:
+                rhs = ExprNodes.SimpleCallNode(
+                    decorator.pos,
+                    function = decorator.decorator,
+                    args = [rhs])
 
         self.assmt = SingleAssignmentNode(self.pos,
             lhs = ExprNodes.NameNode(self.pos, name = self.name),
@@ -3082,8 +3092,9 @@ class PyClassDefNode(ClassDefNode):
     #  classobj ClassNode  Class object
     #  target   NameNode   Variable to assign class object to
 
-    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "classobj", "target"]
+    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result", "target"]
     decorators = None
+    class_result = None
     py3_style_class = False # Python3 style class (bases+kwargs)
 
     def __init__(self, pos, name, bases, doc, body, decorators = None,
@@ -3186,6 +3197,16 @@ class PyClassDefNode(ClassDefNode):
         return cenv
 
     def analyse_declarations(self, env):
+        class_result = self.classobj
+        if self.decorators:
+            from ExprNodes import SimpleCallNode
+            for decorator in self.decorators[::-1]:
+                class_result = SimpleCallNode(
+                    decorator.pos,
+                    function = decorator.decorator,
+                    args = [class_result])
+        self.class_result = class_result
+        self.class_result.analyse_declarations(env)
         self.target.analyse_target_declaration(env)
         cenv = self.create_scope(env)
         cenv.directives = env.directives
@@ -3198,7 +3219,7 @@ class PyClassDefNode(ClassDefNode):
             self.metaclass.analyse_expressions(env)
             self.mkw.analyse_expressions(env)
         self.dict.analyse_expressions(env)
-        self.classobj.analyse_expressions(env)
+        self.class_result.analyse_expressions(env)
         genv = env.global_scope()
         cenv = self.scope
         self.body.analyse_expressions(cenv)
@@ -3218,9 +3239,9 @@ class PyClassDefNode(ClassDefNode):
         self.dict.generate_evaluation_code(code)
         cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
         self.body.generate_execution_code(code)
-        self.classobj.generate_evaluation_code(code)
+        self.class_result.generate_evaluation_code(code)
         cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
-        self.target.generate_assignment_code(self.classobj, code)
+        self.target.generate_assignment_code(self.class_result, code)
         self.dict.generate_disposal_code(code)
         self.dict.free_temps(code)
         if self.py3_style_class:
@@ -3279,6 +3300,9 @@ class CClassDefNode(ClassDefNode):
         if env.in_cinclude and not self.objstruct_name:
             error(self.pos, "Object struct name specification required for "
                 "C class defined in 'extern from' block")
+        if self.decorators:
+            error(self.pos,
+                  "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
         self.base_type = None
         # Now that module imports are cached, we need to
         # import the modules for extern classes.

Cython/Compiler/ParseTreeTransforms.py

@@ -945,39 +945,23 @@ class WithTransform(CythonTransform, SkipDeclarations):
         return node
 
 
-class DecoratorTransform(CythonTransform, SkipDeclarations):
+class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations):
+    """Originally, this was the only place where decorators were
+    transformed into the corresponding calling code.  Now, this is
+    done directly in DefNode and PyClassDefNode to avoid reassignments
+    to the function/class name - except for cdef class methods.  For
+    those, the reassignment is required as methods are originally
+    defined in the PyMethodDef struct.
+    """
 
     def visit_DefNode(self, func_node):
-        self.visitchildren(func_node)
-        if not func_node.decorators:
+        scope_type = self.scope_type
+        func_node = self.visit_FuncDefNode(func_node)
+        if scope_type != 'cclass' or not func_node.decorators:
             return func_node
         return self._handle_decorators(
             func_node, func_node.name)
 
-    def visit_CClassDefNode(self, class_node):
-        # This doesn't currently work, so it's disabled.
-        #
-        # Problem: assignments to cdef class names do not work.  They
-        # would require an additional check anyway, as the extension
-        # type must not change its C type, so decorators cannot
-        # replace an extension type, just alter it and return it.
-
-        self.visitchildren(class_node)
-        if not class_node.decorators:
-            return class_node
-        error(class_node.pos,
-              "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
-        return class_node
-        #return self._handle_decorators(
-        #    class_node, class_node.class_name)
-
-    def visit_ClassDefNode(self, class_node):
-        self.visitchildren(class_node)
-        if not class_node.decorators:
-            return class_node
-        return self._handle_decorators(
-            class_node, class_node.name)
-
     def _handle_decorators(self, node, name):
         decorator_result = ExprNodes.NameNode(node.pos, name = name)
         for decorator in node.decorators[::-1]:
@@ -1467,9 +1451,9 @@ class CreateClosureClasses(CythonTransform):
 
         if not from_closure and (self.path or inner_node):
             if not inner_node:
-                if not node.assmt:
+                if not node.py_cfunc_node:
                     raise InternalError, "DefNode does not have assignment node"
-                inner_node = node.assmt.rhs
+                inner_node = node.py_cfunc_node
             inner_node.needs_self_code = False
             node.needs_outer_scope = False
 

Cython/Compiler/Scanning.py

@@ -166,8 +166,9 @@ class FileSourceDescriptor(SourceDescriptor):
     optional name argument and will be passed back when asking for
     the position()-tuple.
     """
-    def __init__(self, filename):
+    def __init__(self, filename, path_description=None):
         filename = Utils.decode_filename(filename)
+        self.path_description = path_description or filename
         self.filename = filename
         self.set_file_type_from_name(filename)
         self._cmp_name = filename
@@ -180,7 +181,7 @@ class FileSourceDescriptor(SourceDescriptor):
             require_normalised_newlines=False)
 
     def get_description(self):
-        return self.filename
+        return self.path_description
 
     def get_filenametable_entry(self):
         return self.filename
@@ -199,6 +200,8 @@ class StringSourceDescriptor(SourceDescriptor):
     Instances of this class can be used instead of a filenames if the
     code originates from a string object.
     """
+    filename = None
+
     def __init__(self, name, code):
         self.name = name
         #self.set_file_type_from_name(name)

Cython/Compiler/Tests/TestDecorators.py

-import unittest
-from Cython.TestUtils import TransformTest
-from Cython.Compiler.ParseTreeTransforms import DecoratorTransform
-
-class TestDecorator(TransformTest):
-
-    def test_decorator(self):
-        t = self.run_pipeline([DecoratorTransform(None)], u"""
-        def decorator(fun):
-            return fun
-        @decorator
-        def decorated():
-            pass
-        """)
-
-        self.assertCode(u"""
-        def decorator(fun):
-            return fun
-        def decorated():
-            pass
-        decorated = decorator(decorated)
-        """, t)
-
-if __name__ == '__main__':
-    unittest.main()

tests/bugs.txt

@@ -18,6 +18,7 @@ ipow_crash_T562
 pure_mode_cmethod_inheritance_T583
 genexpr_iterable_lookup_T600
 for_from_pyvar_loop_T601
+decorators_T593
 
 # CPython regression tests that don't current work:
 pyregr.test_threadsignals

tests/run/decorators_T593.pyx

+"""
+>>> am_i_buggy
+False
+>>> Foo
+False
+"""
+def testme(func):
+    try:
+        am_i_buggy
+        return True
+    except NameError:
+        return False
+@testme
+def am_i_buggy():
+    pass
+
+def testclass(klass):
+    try:
+        Foo
+        return True
+    except NameError:
+        return False
+@testclass
+class Foo:
+    pass
+
+class ODict(dict):
+   def __init__(self):
+       dict.__init__(self)
+       self._order = []
+       dict.__setitem__(self, '_order', self._order)
+   def __setitem__(self, key, value):
+       dict.__setitem__(self, key, value)
+       self._order.append(key)
+
+class Base(type):
+   @staticmethod
+   def __prepare__(*args, **kwargs):
+       return ODict()
+
+class Bar(metaclass=Base):
+   """
+   >>> Bar._order
+   ['__module__', '__doc__', 'bar']
+   """
+   @property
+   def bar(self):
+       return 0