Preliminary debug support for Cython

added the --pyrex-debug flag to Cython's build_ext
added the pyrex_debug boolean to Cython's Cython.Distutils.extension.Extension
    (for per-module debugging information)
debug output is written to the cython_debug directory
bin/cygdb is included (start this from the build directory)
    working commands: cy import, cy locals, cy break
when debugging is active, export all functions as extern

Cython/Compiler/CmdLine.py

@@ -113,6 +113,8 @@ def parse_command_line(args):
                 Options.convert_range = True
             elif option == "--line-directives":
                 options.emit_linenums = True
+            elif option == "--debug":
+                options.debug = True
             elif option == '-2':
                 options.language_level = 2
             elif option == '-3':

Cython/Compiler/Errors.py

@@ -91,6 +91,10 @@ class CompilerCrash(CompileError):
             message += u'%s: %s' % (cause.__class__.__name__, cause)
         CompileError.__init__(self, pos, message)
 
+class NoElementTreeInstalledException(PyrexError):
+    """raised when the user enabled options.debug but no ElementTree 
+    implementation was found
+    """
 
 listing_file = None
 num_errors = 0

Cython/Compiler/Main.py

@@ -13,7 +13,9 @@ except NameError:
     # Python 2.3
     from sets import Set as set
 
+import itertools
 from time import time
+
 import Code
 import Errors
 import Parsing
@@ -176,13 +178,19 @@ class Context(object):
             from Cython.TestUtils import TreeAssertVisitor
             test_support.append(TreeAssertVisitor())
 
-        return ([
-                create_parse(self),
-            ] + self.create_pipeline(pxd=False, py=py) + test_support + [
-                inject_pxd_code,
-                abort_on_errors,
-                generate_pyx_code,
-            ])
+        if options.debug:
+            import ParseTreeTransforms
+            debug_transform = [ParseTreeTransforms.DebuggerTransform(self)]
+        else:
+            debug_transform = []
+            
+        return list(itertools.chain(
+            [create_parse(self)],
+            self.create_pipeline(pxd=False, py=py),
+            test_support,
+            [inject_pxd_code, abort_on_errors],
+            debug_transform,
+            [generate_pyx_code]))
 
     def create_pxd_pipeline(self, scope, module_name):
         def parse_pxd(source_desc):
@@ -798,4 +806,5 @@ default_options = dict(
     evaluate_tree_assertions = False,
     emit_linenums = False,
     language_level = 2,
+    debug = False,
 )

Cython/Compiler/ParseTreeTransforms.py

@@ -7,11 +7,39 @@ from Cython.Compiler.UtilNodes import *
 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler.Errors import error, CompileError
+from Cython.Compiler import Errors
+
 try:
     set
 except NameError:
     from sets import Set as set
+
 import copy
+import os
+import errno
+
+try:
+  from lxml import etree
+  have_lxml = True
+except ImportError:
+    have_lxml = False
+    try:
+        # Python 2.5
+        from xml.etree import cElementTree as etree
+    except ImportError:
+        try:
+            # Python 2.5
+            from xml.etree import ElementTree as etree
+        except ImportError:
+            try:
+                # normal cElementTree install
+                import cElementTree as etree
+            except ImportError:
+                try:
+                    # normal ElementTree install
+                    import elementtree.ElementTree as etree
+                except ImportError:
+                    etree = None
 
 
 class NameNodeCollector(TreeVisitor):
@@ -1431,3 +1459,122 @@ class TransformBuiltinMethods(EnvTransform):
         
         self.visitchildren(node)
         return node
+
+
+def _create_xmlnode(tb, name, attrs=None):
+    "create a xml node with name name and attrs attrs given TreeBuilder tb"
+    tb.start(name, attrs or {})
+    tb.end(name)
+
+
+class DebuggerTransform(CythonTransform):
+    """
+    Class to output debugging information for cygdb
+    
+    It writes debug information to cython_debug/cython_debug_info_<modulename>
+    in the build directory. Also sets all functions' visibility to extern to 
+    enable debugging
+    """
+    
+    def __init__(self, context):
+        super(DebuggerTransform, self).__init__(context)
+        if etree is None:
+            raise Errors.NoElementTreeInstalledException()
+        else:
+            self.tb = etree.TreeBuilder()
+        self.visited = set()
+        
+    def visit_ModuleNode(self, node):
+        self.module_name = node.full_module_name
+        attrs = dict(
+            module_name=self.module_name,
+            filename=node.pos[0].filename)
+        
+        self.tb.start('Module', attrs)
+        
+        # serialize functions
+        self.tb.start('Functions', {})
+        self.visitchildren(node)
+        self.tb.end('Functions')
+        
+        # 2.3 compatibility. Serialize global variables
+        self.tb.start('Globals', {})
+        entries = {}
+        for k, v in node.scope.entries.iteritems():
+            if (v.qualified_name not in self.visited and 
+                not v.name.startswith('__pyx_')):
+                # if v.qualified_name == 'testcython.G': import pdb; pdb.set_trace()
+                entries[k]= v
+        
+        self.serialize_local_variables(entries)
+        self.tb.end('Globals')
+        self.tb.end('Module')
+        return node
+    
+    def visit_FuncDefNode(self, node):
+        self.visited.add(node.local_scope.qualified_name)
+        node.entry.visibility = 'extern'
+        if node.py_func is None:
+            pf_cname = ''
+        else:
+            pf_cname = node.py_func.entry.func_cname
+            
+        attrs = dict(
+            name=node.entry.name,
+            cname=node.entry.func_cname,
+            pf_cname=pf_cname,
+            qualified_name=node.local_scope.qualified_name,
+            lineno=str(node.pos[1]))
+        
+        self.tb.start('Function', attrs=attrs)
+        
+        self.tb.start('Locals', {})
+        self.serialize_local_variables(node.local_scope.entries)
+        self.tb.end('Locals')
+        self.tb.start('Arguments', {})
+        for arg in node.local_scope.arg_entries:
+            _create_xmlnode(self.tb, arg.name)
+        self.tb.end('Arguments')
+        self.tb.end('Function')
+        return node
+    
+    def serialize_local_variables(self, entries):
+        for entry in entries.values():
+            if entry.type.is_pyobject:
+                vartype = 'PyObject'
+            else:
+                vartype = 'CObject'
+            
+            cname = entry.cname
+            if entry.type.is_extension_type:
+                cname = entry.type.typeptr_cname
+
+            attrs = dict(
+                name=entry.name,
+                cname=cname,
+                qualified_name=entry.qualified_name,
+                type=vartype)
+                
+            _create_xmlnode(self.tb, 'LocalVar', attrs)
+    
+    def __call__(self, root):
+        self.tb.start('cython_debug', attrs=dict(version='1.0'))
+        super(DebuggerTransform, self).__call__(root)
+        self.tb.end('cython_debug')
+        xml_root_element = self.tb.close()
+
+        try:
+            os.mkdir('cython_debug')
+        except OSError, e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        et = etree.ElementTree(xml_root_element)
+        kw = {}
+        if have_lxml:
+            kw['pretty_print'] = True
+        et.write("cython_debug/cython_debug_info_" + self.module_name, 
+                 encoding="UTF-8", 
+                 **kw)
+            
+        return root
\ No newline at end of file

Cython/Distutils/build_ext.py

@@ -46,10 +46,13 @@ class build_ext(_build_ext.build_ext):
             "generate .pxi file for public declarations"),
         ('pyrex-directives=', None,
             "compiler directive overrides"),
+        ('pyrex-debug', None,
+         "generate debug information for cygdb"),
         ])
 
     boolean_options.extend([
-        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-line-directives', 'pyrex-c-in-temp'
+        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-line-directives', 
+        'pyrex-c-in-temp', 'pyrex-debug',
     ])
 
     def initialize_options(self):
@@ -61,6 +64,7 @@ class build_ext(_build_ext.build_ext):
         self.pyrex_directives = None
         self.pyrex_c_in_temp = 0
         self.pyrex_gen_pxi = 0
+        self.pyrex_debug = False
 
     def finalize_options (self):
         _build_ext.build_ext.finalize_options(self)
@@ -127,7 +131,7 @@ class build_ext(_build_ext.build_ext):
         cplus = self.pyrex_cplus or getattr(extension, 'pyrex_cplus', 0) or \
                 (extension.language and extension.language.lower() == 'c++')
         pyrex_gen_pxi = self.pyrex_gen_pxi or getattr(extension, 'pyrex_gen_pxi', 0)
-
+        pyrex_debug = self.pyrex_debug or getattr(extension, 'pyrex_debug', False)
         # Set up the include_path for the Cython compiler:
         #    1.    Start with the command line option.
         #    2.    Add in any (unique) paths from the extension
@@ -207,7 +211,8 @@ class build_ext(_build_ext.build_ext):
                     output_file = target,
                     cplus = cplus,
                     emit_linenums = line_directives,
-                    generate_pxi = pyrex_gen_pxi)
+                    generate_pxi = pyrex_gen_pxi,
+                    debug = pyrex_debug)
                 result = cython_compile(source, options=options,
                                         full_module_name=module_name)
             else:

Cython/Distutils/extension.py

@@ -31,6 +31,8 @@ class Extension(_Extension.Extension):
         put generated C files in temp directory.
     pyrex_gen_pxi : boolean
         generate .pxi file for public declarations
+    pyrex_debug : boolean
+        generate Cython debug information for this extension for cygdb
     """
 
     # When adding arguments to this constructor, be sure to update
@@ -56,6 +58,7 @@ class Extension(_Extension.Extension):
             pyrex_cplus = 0,
             pyrex_c_in_temp = 0,
             pyrex_gen_pxi = 0,
+            pyrex_debug = False,
             **kw):
 
         _Extension.Extension.__init__(self, name, sources,
@@ -81,6 +84,7 @@ class Extension(_Extension.Extension):
         self.pyrex_cplus = pyrex_cplus
         self.pyrex_c_in_temp = pyrex_c_in_temp
         self.pyrex_gen_pxi = pyrex_gen_pxi
+        self.pyrex_debug = pyrex_debug
 
 # class Extension
 

bin/cygdb

+#!/usr/bin/env python
+
+"""
+The Cython debugger
+
+The current directory should contain a directory named 'cython_debug', or a
+path to the cython project directory should be given (the parent directory of
+cython_debug).
+"""
+
+import os
+import sys
+import glob
+import tempfile
+import subprocess
+
+def main(import_libpython=False, path_to_debug_info=os.curdir):
+    """
+    Start the Cython debugger. This tells gdb to import the Cython and Python
+    extensions (libpython.py and libcython.py) and it enables gdb's pending 
+    breakpoints
+    """
+    debug_files = glob.glob(
+        os.path.join(os.curdir, 'cython_debug/cython_debug_info_*'))
+
+    if not debug_files:
+        sys.exit('No debug files were found in the current directory. '
+                 'Aborting.')
+        
+    fd, tempfilename = tempfile.mkstemp()
+    f = os.fdopen(fd, 'w')
+    f.write('set breakpoint pending on\n')
+    f.write('python from Cython.Debugger import libcython\n')
+    if import_libpython:
+        f.write('python import libpython')
+    else:
+        f.write('python from Cython.Debugger import libpython\n')
+    f.write('\n'.join('cy import %s\n' % fn for fn in debug_files))
+    f.close()
+    
+    p = subprocess.Popen(['gdb', '-command', tempfilename])
+    while True:
+        try:
+            p.wait()
+        except KeyboardInterrupt:
+            pass
+        else:
+            break
+    os.remove(tempfilename)
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file