HTML source annotation module

Cython/Compiler/CmdLine.py

@@ -25,6 +25,7 @@ Options:
   --incref-local-binop           Force local an extra incref on local variables before
                                  performing any binary operations.
   -D, --no-docstrings            Remove docstrings.
+  -a, --annotate                 Produce an colorized version of the source.
 """  
 #The following experimental options are supported only on MacOSX:
 #  -C, --compile    Compile generated .c file to .o file
@@ -85,6 +86,8 @@ def parse_command_line(args):
                 Options.generate_cleanup_code = int(pop_arg())
             elif option in ("-D", "--no-docstrings"):
                 Options.docstrings = False
+            elif option in ("-a", "--annotate"):
+                Options.annotate = True
             else:
                 bad_usage()
         else:

Cython/Compiler/Code.py

@@ -96,6 +96,8 @@ class CCodeWriter:
         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
 
     def mark_pos(self, pos):
+#        if self.marker is not None:
+#            print "new marker"
         file, line, col = pos
         contents = self.file_contents(file)
 

Cython/Compiler/ExprNodes.py

@@ -12,6 +12,7 @@ import PyrexTypes
 from PyrexTypes import py_object_type, c_long_type, typecast, error_type
 import Symtab
 import Options
+from Annotate import AnnotationItem
 
 from Cython.Debugging import print_call_chain
 from DebugFlags import debug_disposal_code, debug_temp_alloc, \
@@ -404,6 +405,7 @@ class ExprNode(Node):
             code.put_incref(self.result_code, self.ctype())
     
     def generate_evaluation_code(self, code):
+        code.mark_pos(self.pos)
         #  Generate code to evaluate this node and
         #  its sub-expressions, and dispose of any
         #  temporary results of its sub-expressions.
@@ -456,6 +458,12 @@ class ExprNode(Node):
         #  will have been reported earlier.
         pass
     
+    # ---------------- Annotation ---------------------
+    
+    def annotate(self, code):
+        for node in self.subexpr_nodes():
+            node.annotate(code)
+    
     # ----------------- Coercion ----------------------
     
     def coerce_to(self, dst_type, env):
@@ -969,6 +977,14 @@ class NameNode(AtomicExprNode):
             'PyObject_DelAttrString(%s, "%s")' % (
                 Naming.module_cname,
                 self.entry.name))
+                
+    def annotate(self, code):
+        if hasattr(self, 'is_called') and self.is_called:
+            pos = (self.pos[0], self.pos[1], self.pos[2] - len(self.name) - 1)
+            if self.type.is_pyobject:
+                code.annotate(pos, AnnotationItem('py_call', 'python function', size=len(self.name)))
+            else:
+                code.annotate(pos, AnnotationItem('c_call', 'c function', size=len(self.name)))
             
 
 class BackquoteNode(ExprNode):
@@ -1970,6 +1986,12 @@ class AttributeNode(ExprNode):
         else:
             error(self.pos, "Cannot delete C attribute of extension type")
         self.obj.generate_disposal_code(code)
+        
+    def annotate(self, code):
+        if self.is_py_attr:
+            code.annotate(self.pos, AnnotationItem('py_attr', 'python attribute', size=len(self.attribute)))
+        else:
+            code.annotate(self.pos, AnnotationItem('c_attr', 'c attribute', size=len(self.attribute)))
 
 #-------------------------------------------------------------------
 #
@@ -2089,6 +2111,15 @@ class SequenceNode(ExprNode):
         self.iterator.generate_disposal_code(code)
 
         code.putln("}")
+        
+    def annotate(self, code):
+        for arg in self.args:
+            arg.annotate(code)
+        if self.unpacked_items:
+            for arg in self.unpacked_items:
+                arg.annotate(code)
+            for arg in self.coerced_unpacked_items:
+                arg.annotate(code)
 
 
 class TupleNode(SequenceNode):
@@ -2176,6 +2207,9 @@ class ListComprehensionNode(SequenceNode):
             0,
             code.error_goto_if_null(self.result_code, self.pos)))
         self.loop.generate_execution_code(code)
+        
+    def annotate(self, code):
+        self.loop.annotate(code)
 
 
 class ListComprehensionAppendNode(ExprNode):
@@ -2249,7 +2283,11 @@ class DictNode(ExprNode):
                     value.py_result()))
             key.generate_disposal_code(code)
             value.generate_disposal_code(code)
-    
+            
+    def annotate(self, code):
+        for key, value in self.key_value_pairs:
+            key.annotate(code)
+            value.annotate(code)
 
 class ClassNode(ExprNode):
     #  Helper class used in the implementation of Python
@@ -3321,6 +3359,12 @@ class PrimaryCmpNode(ExprNode, CmpNode):
         #  so only need to dispose of the two main operands.
         self.operand1.generate_disposal_code(code)
         self.operand2.generate_disposal_code(code)
+        
+    def annotate(self, code):
+        self.operand1.annotate(code)
+        self.operand2.annotate(code)
+        if self.cascade:
+            self.cascade.annotate(code)
 
 
 class CascadedCmpNode(Node, CmpNode):
@@ -3385,6 +3429,11 @@ class CascadedCmpNode(Node, CmpNode):
         self.operand2.generate_disposal_code(code)
         code.putln("}")
 
+    def annotate(self, code):
+        self.operand2.annotate(code)
+        if self.cascade:
+            self.cascade.annotate(code)
+
 
 binop_node_classes = {
     "or":		BoolBinopNode,
@@ -3434,6 +3483,12 @@ class CoercionNode(ExprNode):
         self.arg = arg
         if debug_coercion:
             print self, "Coercing", self.arg
+            
+    def annotate(self, code):
+        self.arg.annotate(code)
+        if self.arg.type != self.type:
+            file, line, col = self.pos
+            code.annotate((file, line, col-1), AnnotationItem(style='coerce', tag='coerce', text='[%s] to [%s]' % (self.arg.type, self.type)))
 
 
 class CastNode(CoercionNode):

Cython/Compiler/ModuleNode.py

@@ -6,6 +6,7 @@ import os, time
 from cStringIO import StringIO
 from PyrexTypes import CPtrType
 
+import Annotate
 import Code
 import Naming
 import Nodes
@@ -200,7 +201,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     
     def generate_c_code(self, env, result):
         modules = self.referenced_modules
-        code = Code.CCodeWriter(StringIO())
+        if Options.annotate:
+            code = Annotate.AnnotationCCodeWriter(StringIO())
+        else:
+            code = Code.CCodeWriter(StringIO())
         code.h = Code.CCodeWriter(StringIO())
         code.init_labels()
         self.generate_module_preamble(env, modules, code.h)
@@ -233,6 +237,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         f.write(code.f.getvalue())
         f.close()
         result.c_file_generated = 1
+        if Options.annotate:
+            self.annotate(code)
+            code.save_annotation(result.c_file[:-1] + "pyx") # change?
     
     def find_referenced_modules(self, env, module_list, modules_seen):
         if env not in modules_seen:
@@ -388,6 +395,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         return header, footer
     
     def generate_struct_union_definition(self, entry, code):
+        code.mark_pos(entry.pos)
         type = entry.type
         scope = type.scope
         if scope:
@@ -407,6 +415,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln(footer)
 
     def generate_enum_definition(self, entry, code):
+        code.mark_pos(entry.pos)
         type = entry.type
         name = entry.cname or entry.name or ""
         header, footer = \
@@ -450,6 +459,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             #	code.putln("staticforward PyTypeObject %s;" % name)
     
     def generate_exttype_vtable_struct(self, entry, code):
+        code.mark_pos(entry.pos)
         # Generate struct declaration for an extension type's vtable.
         type = entry.type
         scope = type.scope
@@ -470,6 +480,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 "};")
     
     def generate_exttype_vtabptr_declaration(self, entry, code):
+        code.mark_pos(entry.pos)
         # Generate declaration of pointer to an extension type's vtable.
         type = entry.type
         if type.vtabptr_cname:
@@ -478,6 +489,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 type.vtabptr_cname))
     
     def generate_obj_struct_definition(self, type, code):
+        code.mark_pos(type.pos)
         # Generate object struct definition for an
         # extension type.
         if not type.scope:

Cython/Compiler/Options.py

@@ -20,3 +20,5 @@ incref_local_binop = 0
 # Decref global variables in this module on exit for garbage collection. 
 # 0: None, 1+: interned objects, 2+: cdef globals, 3+: types objects
 generate_cleanup_code = 1
+
+annotate = 0
\ No newline at end of file

Cython/Compiler/Symtab.py

@@ -847,6 +847,7 @@ class ModuleScope(Scope):
         #
         if not entry:
             type = PyExtensionType(name, typedef_flag, base_type)
+            type.pos = pos
             if visibility == 'extern':
                 type.module_name = module_name
             else: