HTML source annotation module

Cython/Compiler/CmdLine.py

@@ -25,6 +25,7 @@ Options:
   --incref-local-binop           Force local an extra incref on local variables before
                                  performing any binary operations.
   -D, --no-docstrings            Remove docstrings.
+  -a, --annotate                 Produce an colorized version of the source.
 """  
 #The following experimental options are supported only on MacOSX:
 #  -C, --compile    Compile generated .c file to .o file
@@ -85,6 +86,8 @@ def parse_command_line(args):
                 Options.generate_cleanup_code = int(pop_arg())
             elif option in ("-D", "--no-docstrings"):
                 Options.docstrings = False
+            elif option in ("-a", "--annotate"):
+                Options.annotate = True
             else:
                 bad_usage()
         else:

Cython/Compiler/Code.py

@@ -96,6 +96,8 @@ class CCodeWriter:
         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
 
     def mark_pos(self, pos):
+#        if self.marker is not None:
+#            print "new marker"
         file, line, col = pos
         contents = self.file_contents(file)
 

Cython/Compiler/ExprNodes.py

@@ -12,6 +12,7 @@ import PyrexTypes
 from PyrexTypes import py_object_type, c_long_type, typecast, error_type
 import Symtab
 import Options
+from Annotate import AnnotationItem
 
 from Cython.Debugging import print_call_chain
 from DebugFlags import debug_disposal_code, debug_temp_alloc, \
@@ -404,6 +405,7 @@ class ExprNode(Node):
             code.put_incref(self.result_code, self.ctype())
     
     def generate_evaluation_code(self, code):
+        code.mark_pos(self.pos)
         #  Generate code to evaluate this node and
         #  its sub-expressions, and dispose of any
         #  temporary results of its sub-expressions.
@@ -456,6 +458,12 @@ class ExprNode(Node):
         #  will have been reported earlier.
         pass
     
+    # ---------------- Annotation ---------------------
+    
+    def annotate(self, code):
+        for node in self.subexpr_nodes():
+            node.annotate(code)
+    
     # ----------------- Coercion ----------------------
     
     def coerce_to(self, dst_type, env):
@@ -970,6 +978,14 @@ class NameNode(AtomicExprNode):
                 Naming.module_cname,
                 self.entry.name))
                 
+    def annotate(self, code):
+        if hasattr(self, 'is_called') and self.is_called:
+            pos = (self.pos[0], self.pos[1], self.pos[2] - len(self.name) - 1)
+            if self.type.is_pyobject:
+                code.annotate(pos, AnnotationItem('py_call', 'python function', size=len(self.name)))
+            else:
+                code.annotate(pos, AnnotationItem('c_call', 'c function', size=len(self.name)))
+            
 
 class BackquoteNode(ExprNode):
     #  `expr`
@@ -1971,6 +1987,12 @@ class AttributeNode(ExprNode):
             error(self.pos, "Cannot delete C attribute of extension type")
         self.obj.generate_disposal_code(code)
         
+    def annotate(self, code):
+        if self.is_py_attr:
+            code.annotate(self.pos, AnnotationItem('py_attr', 'python attribute', size=len(self.attribute)))
+        else:
+            code.annotate(self.pos, AnnotationItem('c_attr', 'c attribute', size=len(self.attribute)))
+
 #-------------------------------------------------------------------
 #
 #  Constructor nodes
@@ -2090,6 +2112,15 @@ class SequenceNode(ExprNode):
 
         code.putln("}")
         
+    def annotate(self, code):
+        for arg in self.args:
+            arg.annotate(code)
+        if self.unpacked_items:
+            for arg in self.unpacked_items:
+                arg.annotate(code)
+            for arg in self.coerced_unpacked_items:
+                arg.annotate(code)
+
 
 class TupleNode(SequenceNode):
     #  Tuple constructor.
@@ -2177,6 +2208,9 @@ class ListComprehensionNode(SequenceNode):
             code.error_goto_if_null(self.result_code, self.pos)))
         self.loop.generate_execution_code(code)
         
+    def annotate(self, code):
+        self.loop.annotate(code)
+
 
 class ListComprehensionAppendNode(ExprNode):
 
@@ -2250,6 +2284,10 @@ class DictNode(ExprNode):
             key.generate_disposal_code(code)
             value.generate_disposal_code(code)
             
+    def annotate(self, code):
+        for key, value in self.key_value_pairs:
+            key.annotate(code)
+            value.annotate(code)
 
 class ClassNode(ExprNode):
     #  Helper class used in the implementation of Python
@@ -3322,6 +3360,12 @@ class PrimaryCmpNode(ExprNode, CmpNode):
         self.operand1.generate_disposal_code(code)
         self.operand2.generate_disposal_code(code)
         
+    def annotate(self, code):
+        self.operand1.annotate(code)
+        self.operand2.annotate(code)
+        if self.cascade:
+            self.cascade.annotate(code)
+
 
 class CascadedCmpNode(Node, CmpNode):
     #  A CascadedCmpNode is not a complete expression node. It 
@@ -3385,6 +3429,11 @@ class CascadedCmpNode(Node, CmpNode):
         self.operand2.generate_disposal_code(code)
         code.putln("}")
 
+    def annotate(self, code):
+        self.operand2.annotate(code)
+        if self.cascade:
+            self.cascade.annotate(code)
+
 
 binop_node_classes = {
     "or":		BoolBinopNode,
@@ -3435,6 +3484,12 @@ class CoercionNode(ExprNode):
         if debug_coercion:
             print self, "Coercing", self.arg
             
+    def annotate(self, code):
+        self.arg.annotate(code)
+        if self.arg.type != self.type:
+            file, line, col = self.pos
+            code.annotate((file, line, col-1), AnnotationItem(style='coerce', tag='coerce', text='[%s] to [%s]' % (self.arg.type, self.type)))
+
 
 class CastNode(CoercionNode):
     #  Wrap a node in a C type cast.

Cython/Compiler/ModuleNode.py

@@ -6,6 +6,7 @@ import os, time
 from cStringIO import StringIO
 from PyrexTypes import CPtrType
 
+import Annotate
 import Code
 import Naming
 import Nodes
@@ -200,6 +201,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     
     def generate_c_code(self, env, result):
         modules = self.referenced_modules
+        if Options.annotate:
+            code = Annotate.AnnotationCCodeWriter(StringIO())
+        else:
             code = Code.CCodeWriter(StringIO())
         code.h = Code.CCodeWriter(StringIO())
         code.init_labels()
@@ -233,6 +237,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         f.write(code.f.getvalue())
         f.close()
         result.c_file_generated = 1
+        if Options.annotate:
+            self.annotate(code)
+            code.save_annotation(result.c_file[:-1] + "pyx") # change?
     
     def find_referenced_modules(self, env, module_list, modules_seen):
         if env not in modules_seen:
@@ -388,6 +395,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         return header, footer
     
     def generate_struct_union_definition(self, entry, code):
+        code.mark_pos(entry.pos)
         type = entry.type
         scope = type.scope
         if scope:
@@ -407,6 +415,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln(footer)
 
     def generate_enum_definition(self, entry, code):
+        code.mark_pos(entry.pos)
         type = entry.type
         name = entry.cname or entry.name or ""
         header, footer = \
@@ -450,6 +459,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             #	code.putln("staticforward PyTypeObject %s;" % name)
     
     def generate_exttype_vtable_struct(self, entry, code):
+        code.mark_pos(entry.pos)
         # Generate struct declaration for an extension type's vtable.
         type = entry.type
         scope = type.scope
@@ -470,6 +480,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 "};")
     
     def generate_exttype_vtabptr_declaration(self, entry, code):
+        code.mark_pos(entry.pos)
         # Generate declaration of pointer to an extension type's vtable.
         type = entry.type
         if type.vtabptr_cname:
@@ -478,6 +489,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 type.vtabptr_cname))
     
     def generate_obj_struct_definition(self, type, code):
+        code.mark_pos(type.pos)
         # Generate object struct definition for an
         # extension type.
         if not type.scope:

Cython/Compiler/Nodes.py

@@ -84,6 +84,13 @@ class Node:
         raise InternalError("generate_code not implemented for %s" % \
             self.__class__.__name__)
             
+    def annotate(self, code):
+        # mro does the wrong thing
+        if isinstance(self, BlockNode):
+            self.body.annotate(code)
+        else:
+            print "skipping", self
+
 
 class BlockNode:
     #  Mixin class for nodes representing a declaration block.
@@ -161,6 +168,10 @@ class StatListNode(Node):
             code.mark_pos(stat.pos)
             stat.generate_execution_code(code)
             
+    def annotate(self, code):
+        for stat in self.stats:
+            stat.annotate(code)
+    
 
 class StatNode(Node):
     #
@@ -201,6 +212,9 @@ class CDefExternNode(StatNode):
     def generate_execution_code(self, code):
         pass
 
+    def annotate(self, code):
+        self.body.annotate(code)
+        
 
 class CDeclaratorNode(Node):
     # Part of a C declaration.
@@ -303,6 +317,8 @@ class CFuncDeclaratorNode(CDeclaratorNode):
     # nogil            boolean    Can be called without gil
     # with_gil         boolean    Acquire gil around function body
     
+    overridable = 0
+
     def analyse(self, return_type, env):
         func_type_args = []
         for arg_node in self.args:
@@ -373,6 +389,10 @@ class CArgDeclNode(Node):
         base_type = self.base_type.analyse(env)
         return self.declarator.analyse(base_type, env)
 
+    def annotate(self, code):
+        if self.default:
+            self.default.annotate(code)
+
 
 class CBaseTypeNode(Node):
     # Abstract base class for C base type nodes.
@@ -599,6 +619,7 @@ class FuncDefNode(StatNode, BlockNode):
         return 0
                 
     def generate_function_definitions(self, env, code):
+        code.mark_pos(self.pos)
         # Generate C code for header and body of function
         genv = env.global_scope()
         lenv = LocalScope(name = self.entry.name, outer_scope = genv)
@@ -1579,6 +1600,10 @@ class CClassDefNode(StatNode):
         if self.body:
             self.body.generate_execution_code(code)
             
+    def annotate(self, code):
+        if self.body:
+            self.body.annotate(code)
+
 
 class PropertyNode(StatNode):
     #  Definition of a property in an extension type.
@@ -1604,6 +1629,9 @@ class PropertyNode(StatNode):
     def generate_execution_code(self, code):
         pass
 
+    def annotate(self, code):
+        self.body.annotate(code)
+
 
 class GlobalNode(StatNode):
     # Global variable declaration.
@@ -1636,6 +1664,9 @@ class ExprStatNode(StatNode):
             code.putln("%s;" % self.expr.result_code)
         self.expr.generate_disposal_code(code)
 
+    def annotate(self, code):
+        self.expr.annotate(code)
+
 
 class AssignmentNode(StatNode):
     #  Abstract base class for assignment nodes.
@@ -1705,6 +1736,10 @@ class SingleAssignmentNode(AssignmentNode):
     def generate_assignment_code(self, code):
         self.lhs.generate_assignment_code(self.rhs, code)
 
+    def annotate(self, code):
+        self.lhs.annotate(code)
+        self.rhs.annotate(code)
+
 
 class CascadedAssignmentNode(AssignmentNode):
     #  An assignment with multiple left hand sides:
@@ -1781,6 +1816,13 @@ class CascadedAssignmentNode(AssignmentNode):
             # Assignment has disposed of the cloned RHS
         self.rhs.generate_disposal_code(code)
 
+    def annotate(self, code):
+        for i in range(len(self.lhs_list)):
+            lhs = self.lhs_list[i].annotate(code)
+            rhs = self.coerced_rhs_list[i].annotate(code)
+        self.rhs.annotate(code)
+        
+
 class ParallelAssignmentNode(AssignmentNode):
     #  A combined packing/unpacking assignment:
     #
@@ -1818,6 +1860,11 @@ class ParallelAssignmentNode(AssignmentNode):
         for stat in self.stats:
             stat.generate_assignment_code(code)
 
+    def annotate(self, code):
+        for stat in self.stats:
+            stat.annotate(code)
+
+
 class InPlaceAssignmentNode(AssignmentNode):
     #  An in place arithmatic operand:
     #
@@ -1922,6 +1969,11 @@ class InPlaceAssignmentNode(AssignmentNode):
         "%":		"PyNumber_InPlaceRemainder",
     }
 
+    def annotate(self, code):
+        self.lhs.annotate(code)
+        self.rhs.annotate(code)
+        self.dup.annotate(code)
+
 
 class PrintStatNode(StatNode):
     #  print statement
@@ -1953,6 +2005,10 @@ class PrintStatNode(StatNode):
                 "if (__Pyx_PrintNewline() < 0) %s" %
                     code.error_goto(self.pos))
                     
+    def annotate(self, code):
+        for arg in self.args:
+            arg.annotate(code)
+
 
 class DelStatNode(StatNode):
     #  del statement
@@ -1976,6 +2032,10 @@ class DelStatNode(StatNode):
                 arg.generate_deletion_code(code)
             # else error reported earlier
 
+    def annotate(self, code):
+        for arg in self.args:
+            arg.annotate(code)
+
 
 class PassStatNode(StatNode):
     #  pass statement
@@ -2049,6 +2109,7 @@ class ReturnStatNode(StatNode):
                     error(self.pos, "Return value required")
     
     def generate_execution_code(self, code):
+        code.mark_pos(self.pos)
         if not self.return_type:
             # error reported earlier
             return
@@ -2075,6 +2136,10 @@ class ReturnStatNode(StatNode):
         #		code.return_label)
         code.put_goto(code.return_label)
         
+    def annotate(self, code):
+        if self.value:
+            self.value.annotate(code)
+
 
 class RaiseStatNode(StatNode):
     #  raise statement
@@ -2141,6 +2206,14 @@ class RaiseStatNode(StatNode):
         code.putln(
             code.error_goto(self.pos))
 
+    def annotate(self, code):
+        if self.exc_type:
+            self.exc_type.annotate(code)
+        if self.exc_value:
+            self.exc_value.annotate(code)
+        if self.exc_tb:
+            self.exc_tb.annotate(code)
+
 
 class ReraiseStatNode(StatNode):
 
@@ -2195,6 +2268,12 @@ class AssertStatNode(StatNode):
         self.cond.generate_disposal_code(code)
         code.putln("#endif")
 
+    def annotate(self, code):
+        self.cond.annotate(code)
+        if self.value:
+            self.value.annotate(code)
+
+
 class IfStatNode(StatNode):
     #  if statement
     #
@@ -2214,6 +2293,7 @@ class IfStatNode(StatNode):
             self.else_clause.analyse_expressions(env)
     
     def generate_execution_code(self, code):
+        code.mark_pos(self.pos)
         end_label = code.new_label()
         for if_clause in self.if_clauses:
             if_clause.generate_execution_code(code, end_label)
@@ -2223,6 +2303,12 @@ class IfStatNode(StatNode):
             code.putln("}")
         code.put_label(end_label)
         
+    def annotate(self, code):
+        for if_clause in self.if_clauses:
+            if_clause.annotate(code)
+        if self.else_clause:
+            self.else_clause.annotate(code)
+
 
 class IfClauseNode(Node):
     #  if or elif clause in an if statement
@@ -2253,6 +2339,10 @@ class IfClauseNode(Node):
         code.put_goto(end_label)
         code.putln("}")
 
+    def annotate(self, code):
+        self.condition.annotate(code)
+        self.body.annotate(code)
+        
     
 class WhileStatNode(StatNode):
     #  while statement
@@ -2294,6 +2384,12 @@ class WhileStatNode(StatNode):
             code.putln("}")
         code.put_label(break_label)
 
+    def annotate(self, code):
+        self.condition.annotate(code)
+        self.body.annotate(code)
+        if self.else_clause:
+            self.else_clause.annotate(code)
+
 
 def ForStatNode(pos, **kw):
     if kw.has_key('iterator'):
@@ -2351,6 +2447,14 @@ class ForInStatNode(StatNode):
         code.put_label(break_label)
         self.iterator.generate_disposal_code(code)
 
+    def annotate(self, code):
+        self.target.annotate(code)
+        self.iterator.annotate(code)
+        self.body.annotate(code)
+        if self.else_clause:
+            self.else_clause.annotate(code)
+        self.item.annotate(code)
+
 
 class ForFromStatNode(StatNode):
     #  for name from expr rel name rel expr
@@ -2469,6 +2573,16 @@ class ForFromStatNode(StatNode):
         '>' : ("-1", "--")
     }
     
+    def annotate(self, code):
+        self.target.annotate(code)
+        self.bound1.annotate(code)
+        self.bound2.annotate(code)
+        if self.step:
+            self.bound2.annotate(code)
+        self.body.annotate(code)
+        if self.else_clause:
+            self.else_clause.annotate(code)
+
 
 class TryExceptStatNode(StatNode):
     #  try .. except statement
@@ -2524,6 +2638,13 @@ class TryExceptStatNode(StatNode):
             code.put_goto(code.error_label)
         code.put_label(end_label)
 
+    def annotate(self, code):
+        self.body.annotate(code)
+        for except_node in self.except_clauses:
+            except_node.annotate(code)
+        if self.else_clause:
+            self.else_clause.annotate(code)
+
 
 class ExceptClauseNode(Node):
     #  Part of try ... except statement.
@@ -2597,6 +2718,12 @@ class ExceptClauseNode(Node):
         code.putln(
             "}")
 
+    def annotate(self, code):
+        self.pattern.annotate(code)
+        if self.target:
+            self.target.annotate(code)
+        self.body.annotate(code)
+
 
 class TryFinallyStatNode(StatNode):
     #  try ... finally statement
@@ -2763,6 +2890,10 @@ class TryFinallyStatNode(StatNode):
         code.putln(
             "}")
 
+    def annotate(self, code):
+        self.body.annotate(code)
+        self.finally_clause.annotate(code)
+
 
 class GILStatNode(TryFinallyStatNode):
     #  'with gil' or 'with nogil' statement

Cython/Compiler/Options.py

@@ -20,3 +20,5 @@ incref_local_binop = 0
 # Decref global variables in this module on exit for garbage collection. 
 # 0: None, 1+: interned objects, 2+: cdef globals, 3+: types objects
 generate_cleanup_code = 1
+
+annotate = 0
\ No newline at end of file

Cython/Compiler/Symtab.py

@@ -847,6 +847,7 @@ class ModuleScope(Scope):
         #
         if not entry:
             type = PyExtensionType(name, typedef_flag, base_type)
+            type.pos = pos
             if visibility == 'extern':
                 type.module_name = module_name
             else: