Line number support

C function with context support
Preliminary stepping support
Source code listing
more stuff I forgot about

Cython/Compiler/Code.py

@@ -876,6 +876,14 @@ class CCodeWriter(object):
         return self.buffer.getvalue()
 
     def write(self, s):
+        # also put invalid markers (lineno 0), to indicate that those lines 
+        # have no Cython source code correspondence
+        if self.marker is None:
+            cython_lineno = self.last_marker_line
+        else:
+            cython_lineno = self.marker[0]
+        
+        self.buffer.markers.extend([cython_lineno] * s.count('\n'))
         self.buffer.write(s)
 
     def insertion_point(self):
@@ -954,6 +962,7 @@ class CCodeWriter(object):
             self.emit_marker()
         if self.emit_linenums and self.last_marker_line != 0:
             self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))
+        
         if code:
             if safe:
                 self.put_safe(code)

Cython/Compiler/Main.py

@@ -86,6 +86,8 @@ class Context(object):
         self.include_directories = include_directories + [standard_include_path]
 
         self.set_language_level(language_level)
+        
+        self.debug_outputwriter = None
 
     def set_language_level(self, level):
         self.language_level = level
@@ -179,8 +181,10 @@ class Context(object):
             test_support.append(TreeAssertVisitor())
 
         if options.debug:
-            from ParseTreeTransforms import DebuggerTransform
-            debug_transform = [DebuggerTransform(self, options.output_dir)]
+            from Cython.Debugger import debug_output
+            from ParseTreeTransforms import DebugTransform
+            self.debug_outputwriter = debug_output.CythonDebugWriter(options)
+            debug_transform = [DebugTransform(self)]
         else:
             debug_transform = []
             

Cython/Compiler/ModuleNode.py

@@ -298,12 +298,34 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         
         f = open_new_file(result.c_file)
         rootwriter.copyto(f)
+        if options.debug:
+            self._serialize_lineno_map(env, rootwriter)
         f.close()
         result.c_file_generated = 1
         if Options.annotate or options.annotate:
             self.annotate(rootwriter)
             rootwriter.save_annotation(result.main_source_file, result.c_file)
     
+    def _serialize_lineno_map(self, env, ccodewriter):
+        tb = env.context.debug_outputwriter
+        markers = ccodewriter.buffer.allmarkers()
+        
+        d = {}
+        for c_lineno, cython_lineno in enumerate(markers): 
+            if cython_lineno > 0:
+                d.setdefault(cython_lineno, []).append(c_lineno + 1)
+        
+        tb.start('LineNumberMapping')
+        for cython_lineno, c_linenos in sorted(d.iteritems()):
+                attrs = {
+                    'c_linenos': ' '.join(map(str, c_linenos)),
+                    'cython_lineno': str(cython_lineno),
+                }
+                tb.start('LineNumber', attrs)
+                tb.end('LineNumber')
+        tb.end('LineNumberMapping')
+        tb.serialize()
+        
     def find_referenced_modules(self, env, module_list, modules_seen):
         if env not in modules_seen:
             modules_seen[env] = 1

Cython/Compiler/ParseTreeTransforms.py

@@ -7,7 +7,6 @@ from Cython.Compiler.UtilNodes import *
 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler.Errors import error, CompileError
-from Cython.Compiler import Errors
 
 try:
     set
@@ -15,31 +14,6 @@ except NameError:
     from sets import Set as set
 
 import copy
-import os
-import errno
-
-try:
-  from lxml import etree
-  have_lxml = True
-except ImportError:
-    have_lxml = False
-    try:
-        # Python 2.5
-        from xml.etree import cElementTree as etree
-    except ImportError:
-        try:
-            # Python 2.5
-            from xml.etree import ElementTree as etree
-        except ImportError:
-            try:
-                # normal cElementTree install
-                import cElementTree as etree
-            except ImportError:
-                try:
-                    # normal ElementTree install
-                    import elementtree.ElementTree as etree
-                except ImportError:
-                    etree = None
 
 
 class NameNodeCollector(TreeVisitor):
@@ -1461,56 +1435,44 @@ class TransformBuiltinMethods(EnvTransform):
         return node
 
 
-def _create_xmlnode(tb, name, attrs=None):
-    "create a xml node with name name and attrs attrs given TreeBuilder tb"
-    tb.start(name, attrs or {})
-    tb.end(name)
-
-
-class DebuggerTransform(CythonTransform):
+class DebugTransform(CythonTransform):
     """
-    Class to output debugging information for cygdb
-    
-    It writes debug information to cython_debug/cython_debug_info_<modulename>
-    in the build directory. Also sets all functions' visibility to extern to 
-    enable debugging
+    Create debug information and all functions' visibility to extern in order
+    to enable debugging.
     """
     
-    def __init__(self, context, output_dir):
-        super(DebuggerTransform, self).__init__(context)
-        self.output_dir = os.path.join(output_dir, 'cython_debug')
-        
-        if etree is None:
-            raise Errors.NoElementTreeInstalledException()
-        
-        self.tb = etree.TreeBuilder()
+    def __init__(self, context):
+        super(DebugTransform, self).__init__(context)
         self.visited = set()
+        # our treebuilder and debug output writer 
+        # (see Cython.Debugger.debug_output.CythonDebugWriter)
+        self.tb = self.context.debug_outputwriter
         
     def visit_ModuleNode(self, node):
-        self.module_name = node.full_module_name
+        self.tb.module_name = node.full_module_name
         attrs = dict(
-            module_name=self.module_name,
+            module_name=node.full_module_name,
             filename=node.pos[0].filename)
         
         self.tb.start('Module', attrs)
         
         # serialize functions
-        self.tb.start('Functions', {})
+        self.tb.start('Functions')
         self.visitchildren(node)
         self.tb.end('Functions')
         
         # 2.3 compatibility. Serialize global variables
-        self.tb.start('Globals', {})
+        self.tb.start('Globals')
         entries = {}
         for k, v in node.scope.entries.iteritems():
             if (v.qualified_name not in self.visited and 
                 not v.name.startswith('__pyx_')):
-                # if v.qualified_name == 'testcython.G': import pdb; pdb.set_trace()
                 entries[k]= v
         
         self.serialize_local_variables(entries)
         self.tb.end('Globals')
-        self.tb.end('Module')
+        # self.tb.end('Module') # end Module after the line number mapping in
+        # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
         return node
     
     def visit_FuncDefNode(self, node):
@@ -1530,14 +1492,32 @@ class DebuggerTransform(CythonTransform):
         
         self.tb.start('Function', attrs=attrs)
         
-        self.tb.start('Locals', {})
+        self.tb.start('Locals')
         self.serialize_local_variables(node.local_scope.entries)
         self.tb.end('Locals')
-        self.tb.start('Arguments', {})
+
+        self.tb.start('Arguments')
         for arg in node.local_scope.arg_entries:
-            _create_xmlnode(self.tb, arg.name)
+            self.tb.start(arg.name)
+            self.tb.end(arg.name)
         self.tb.end('Arguments')
+
+        self.tb.start('StepIntoFunctions')
+        self.visitchildren(node)
+        self.tb.end('StepIntoFunctions')
         self.tb.end('Function')
+
+        return node
+
+    def visit_NameNode(self, node):
+        if (node.type.is_cfunction and 
+            node.is_called and 
+            node.entry.in_cinclude):
+            attrs = dict(name=node.entry.func_cname)
+            self.tb.start('StepIntoFunction', attrs=attrs)
+            self.tb.end('StepIntoFunction')
+            
+        self.visitchildren(node)
         return node
     
     def serialize_local_variables(self, entries):
@@ -1557,26 +1537,6 @@ class DebuggerTransform(CythonTransform):
                 qualified_name=entry.qualified_name,
                 type=vartype)
                 
-            _create_xmlnode(self.tb, 'LocalVar', attrs)
-    
-    def __call__(self, root):
-        self.tb.start('cython_debug', attrs=dict(version='1.0'))
-        super(DebuggerTransform, self).__call__(root)
-        self.tb.end('cython_debug')
-        xml_root_element = self.tb.close()
-
-        try:
-            os.makedirs(self.output_dir)
-        except OSError, e:
-            if e.errno != errno.EEXIST:
-                raise
-
-        et = etree.ElementTree(xml_root_element)
-        kw = {}
-        if have_lxml:
-            kw['pretty_print'] = True
+            self.tb.start('LocalVar', attrs)
+            self.tb.end('LocalVar')
         
-        fn = "cython_debug_info_" + self.module_name
-        et.write(os.path.join(self.output_dir, fn), encoding="UTF-8", **kw)
-            
-        return root
\ No newline at end of file

Cython/StringIOTree.py

@@ -11,10 +11,12 @@ class StringIOTree(object):
             stream = StringIO()
         self.stream = stream
         self.write = stream.write
+        self.markers = []
 
     def getvalue(self):
         content = [x.getvalue() for x in self.prepended_children]
         content.append(self.stream.getvalue())
+        print self.linenumber_map()
         return "".join(content)
 
     def copyto(self, target):
@@ -59,6 +61,11 @@ class StringIOTree(object):
         self.prepended_children.append(other)
         return other
 
+    def allmarkers(self):
+        children = self.prepended_children
+        return [m for c in children for m in c.allmarkers()] + self.markers
+        
+
 __doc__ = r"""
 Implements a buffer with insertion points. When you know you need to
 "get back" to a place and write more later, simply call insertion_point()

setup.py

@@ -238,7 +238,7 @@ setup(
     'Cython.Runtime',
     'Cython.Distutils',
     'Cython.Plex',
-
+    'Cython.Debugger',
     'Cython.Tests',
     'Cython.Compiler.Tests',
     ],