dispatch based on frame

python code stepping (for libpython and libcython)
generic stepper class
fix step-into functions
have cygdb accept a '--' command line argument to disable automatic importing
replace gdb.execute() with something that actually captures all output
have 'cy break' break properly on line numbers

Cython/Compiler/Main.py

@@ -185,7 +185,7 @@ class Context(object):
             from ParseTreeTransforms import DebugTransform
             self.debug_outputwriter = DebugWriter.CythonDebugWriter(
                 options.output_dir)
-            debug_transform = [DebugTransform(self)]
+            debug_transform = [DebugTransform(self, options)]
         else:
             debug_transform = []
             

Cython/Compiler/ParseTreeTransforms.py

@@ -1441,18 +1441,23 @@ class DebugTransform(CythonTransform):
     to enable debugging.
     """
     
-    def __init__(self, context):
+    def __init__(self, context, options):
         super(DebugTransform, self).__init__(context)
         self.visited = set()
         # our treebuilder and debug output writer 
         # (see Cython.Debugger.debug_output.CythonDebugWriter)
         self.tb = self.context.debug_outputwriter
+        self.c_output_file = options.output_file
+        
+        # tells visit_NameNode whether it should register step-into functions
+        self.register_stepinto = False
         
     def visit_ModuleNode(self, node):
         self.tb.module_name = node.full_module_name
         attrs = dict(
             module_name=node.full_module_name,
-            filename=node.pos[0].filename)
+            filename=node.pos[0].filename,
+            c_filename=self.c_output_file)
         
         self.tb.start('Module', attrs)
         
@@ -1503,20 +1508,25 @@ class DebugTransform(CythonTransform):
         self.tb.end('Arguments')
 
         self.tb.start('StepIntoFunctions')
+        self.register_stepinto = True
         self.visitchildren(node)
+        self.register_stepinto = False
         self.tb.end('StepIntoFunctions')
         self.tb.end('Function')
 
         return node
 
     def visit_NameNode(self, node):
-        if (node.type.is_cfunction and 
-            node.is_called and 
-            node.entry.in_cinclude):
+        if (self.register_stepinto and node.type.is_cfunction and node.is_called):
+            # don't check node.entry.in_cinclude, as 'cdef extern: ...' 
+            # declared functions are not 'in_cinclude'. 
+            # This means we will list called 'cdef' functions as 
+            # "step into functions", but this is not an issue as they will be 
+            # recognized as Cython functions anyway.
             attrs = dict(name=node.entry.func_cname)
             self.tb.start('StepIntoFunction', attrs=attrs)
             self.tb.end('StepIntoFunction')
-            
+        
         self.visitchildren(node)
         return node
     

Cython/Debugger/Cygdb.py

@@ -20,25 +20,29 @@ import subprocess
 def usage():
     print("Usage: cygdb [PATH GDB_ARGUMENTS]")
 
-def make_command_file(path_to_debug_info):
-    debug_files = glob.glob(
-        os.path.join(path_to_debug_info, 'cython_debug/cython_debug_info_*'))
-
-    if not debug_files:
-        usage()
-        sys.exit('No debug files were found in %s. Aborting.' % (
-                 os.path.abspath(path_to_debug_info)))
+def make_command_file(path_to_debug_info, prefix_code='', no_import=False):
+    if not no_import:
+        pattern = os.path.join(path_to_debug_info, 
+                               'cython_debug/cython_debug_info_*')
+        debug_files = glob.glob(pattern)
+
+        if not debug_files:
+            usage()
+            sys.exit('No debug files were found in %s. Aborting.' % (
+                    os.path.abspath(path_to_debug_info)))
         
     fd, tempfilename = tempfile.mkstemp()
     f = os.fdopen(fd, 'w')
+    f.write(prefix_code)
     f.write('set breakpoint pending on\n')
     f.write('python from Cython.Debugger import libcython\n')
-    f.write('\n'.join('cy import %s\n' % fn for fn in debug_files))
+    if not no_import:
+        f.write('\n'.join('cy import %s\n' % fn for fn in debug_files))
     f.close()
     
     return tempfilename
 
-def main(gdb_argv=[], path_to_debug_info=os.curdir):
+def main(path_to_debug_info=os.curdir, gdb_argv=[], no_import=False):
     """
     Start the Cython debugger. This tells gdb to import the Cython and Python
     extensions (libpython.py and libcython.py) and it enables gdb's pending 
@@ -46,7 +50,7 @@ def main(gdb_argv=[], path_to_debug_info=os.curdir):
     
     path_to_debug_info is the path to the cython_debug directory
     """
-    tempfilename = make_command_file(path_to_debug_info)
+    tempfilename = make_command_file(path_to_debug_info, no_import=no_import)
     p = subprocess.Popen(['gdb', '-command', tempfilename] + gdb_argv)
     while True:
         try:

Cython/Debugger/DebugWriter.py

+import os
+import errno
+
+try:
+  from lxml import etree
+  have_lxml = True
+except ImportError:
+    have_lxml = False
+    try:
+        # Python 2.5
+        from xml.etree import cElementTree as etree
+    except ImportError:
+        try:
+            # Python 2.5
+            from xml.etree import ElementTree as etree
+        except ImportError:
+            try:
+                # normal cElementTree install
+                import cElementTree as etree
+            except ImportError:
+                try:
+                    # normal ElementTree install
+                    import elementtree.ElementTree as etree
+                except ImportError:
+                    etree = None
+
+from Cython.Compiler import Errors
+
+
+class CythonDebugWriter(object):
+    """
+    Class to output debugging information for cygdb
+    
+    It writes debug information to cython_debug/cython_debug_info_<modulename>
+    in the build directory.
+    """
+    
+    def __init__(self, output_dir):
+        if etree is None:
+            raise Errors.NoElementTreeInstalledException()
+        
+        self.output_dir = os.path.join(output_dir, 'cython_debug')
+        self.tb = etree.TreeBuilder()
+        # set by Cython.Compiler.ParseTreeTransforms.DebugTransform
+        self.module_name = None
+        self.start('cython_debug', attrs=dict(version='1.0'))
+    
+    def start(self, name, attrs=None):
+        self.tb.start(name, attrs or {})
+    
+    def end(self, name):
+        self.tb.end(name)
+
+    def serialize(self):
+        self.tb.end('Module')
+        self.tb.end('cython_debug')
+        xml_root_element = self.tb.close()
+
+        try:
+            os.makedirs(self.output_dir)
+        except OSError, e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        et = etree.ElementTree(xml_root_element)
+        kw = {}
+        if have_lxml:
+            kw['pretty_print'] = True
+        
+        fn = "cython_debug_info_" + self.module_name
+        et.write(os.path.join(self.output_dir, fn), encoding="UTF-8", **kw)
\ No newline at end of file

Cython/Debugger/libpython.py

@@ -44,8 +44,12 @@ the type names are known to the debugger
 The module also extends gdb with some python-specific commands.
 '''
 from __future__ import with_statement
-import gdb
+
+import os
 import sys
+import tempfile
+
+import gdb
 
 # Look up the gdb.Type for some standard types:
 _type_char_ptr = gdb.lookup_type('char').pointer() # char*
@@ -1448,3 +1452,165 @@ class PyLocals(gdb.Command):
                       pyop_value.get_truncated_repr(MAX_OUTPUT_LEN)))
 
 PyLocals()
+
+
+def execute(command, from_tty=False, to_string=False):
+    """
+    Replace gdb.execute() with this function and have it accept a 'to_string'
+    argument (new in 7.2). Have it properly capture stderr also.
+    
+    Unfortuntaly, this function is not reentrant.
+    """
+    if not to_string:
+        return _execute(command, from_tty)
+            
+    fd, filename = tempfile.mkstemp()
+    
+    try:
+        _execute("set logging file %s" % filename)
+        _execute("set logging redirect on")
+        _execute("set logging on")
+        _execute("set pagination off")
+        
+        _execute(command, from_tty)
+    finally:
+        data = os.fdopen(fd).read()
+        os.remove(filename)
+        _execute("set logging off")
+        _execute("set pagination on")
+        return data
+        
+_execute = gdb.execute
+gdb.execute = execute
+
+
+class GenericCodeStepper(gdb.Command):
+    """
+    Superclass for code stepping. Subclasses must implement the following 
+    methods:
+        
+        lineno(frame)               - tells the current line number (only
+                                      called for a relevant frame)
+        is_relevant_function(frame) - tells whether we care about frame 'frame'
+        get_source_line(frame)      - get the line of source code for the 
+                                      current line (only called for a relevant
+                                      frame)
+        
+    This class provides an 'invoke' method that invokes a 'step' or 'step-over'
+    depending on the 'stepper' argument.
+    """
+    
+    def __init__(self, name, stepper=False):
+        super(GenericCodeStepper, self).__init__(name, 
+                                                 gdb.COMMAND_RUNNING,
+                                                 gdb.COMPLETE_NONE)
+        self.stepper = stepper
+    
+    def _init_stepping(self):
+        self.beginframe = gdb.selected_frame()
+        self.beginline = self.lineno(self.beginframe)
+        if not self.stepper:
+            self.depth = self._stackdepth(self.beginframe)
+
+    def _next_step(self, gdb_command):
+        """
+        Teturns whether to continue stepping. This method sets the instance 
+        attributes 'result' and 'stopped_running'. 'result' hold the output
+        of the executed gdb command ('step' or 'next')
+        """
+        self.result = gdb.execute(gdb_command, to_string=True)
+        self.stopped_running = gdb.inferiors()[0].pid == 0
+        
+        if self.stopped_running:
+            # We stopped running
+            return False
+            
+        newframe = gdb.selected_frame()
+        
+        hit_breakpoint = self.result.startswith('Breakpoint')
+        is_relevant_function = self.is_relevant_function(newframe)
+        
+        if newframe != self.beginframe:
+            # new function
+            if not self.stepper:
+                is_relevant_function = (
+                    self._stackdepth(newframe) < self.depth and
+                    is_relevant_function)
+
+            new_lineno = False
+        else:
+            is_relevant_function = False
+            new_lineno = self.lineno(newframe) > self.beginline
+        
+        return not (hit_breakpoint or new_lineno or is_relevant_function)
+        
+    def _end_stepping(self):
+        # sys.stdout.write(self.result)
+        
+        if not self.stopped_running:
+            frame = gdb.selected_frame()
+            if self.is_relevant_function(frame):
+                print self.get_source_line(frame)
+    
+    def _stackdepth(self, frame):
+        depth = 0
+        while frame:
+            frame = frame.older()
+            depth += 1
+            
+        return depth
+    
+    def invoke(self, args, from_tty):
+        if args:
+            nsteps = int(args)
+        else:
+            nsteps = 1
+        
+        if self.stepper:
+            gdb_command = 'step'
+        else:
+            gdb_command= 'next'
+        
+        for nthstep in xrange(nsteps):
+            self._init_stepping()
+            while self._next_step(gdb_command):
+                pass
+            
+            self._end_stepping()
+
+
+class PythonCodeStepper(GenericCodeStepper):
+    
+    def pyframe(self, frame):
+        pyframe = Frame(frame).get_pyop()
+        if pyframe:
+            return pyframe
+        else:
+            raise gdb.GdbError(
+                "Unable to find the Python frame, run your code with a debug "
+                "build (configure with --with-pydebug or compile with -g).")
+        
+    def lineno(self, frame):
+        return self.pyframe(frame).current_line_num()
+    
+    def is_relevant_function(self, frame):
+        return Frame(frame).is_evalframeex()
+
+    def get_source_line(self, frame):
+        try:
+            return self.pyframe(frame).current_line().rstrip()
+        except IOError, e:
+            gdb.GdbError('Unable to retrieve source code: %s' % (e,))
+
+
+class PyStep(PythonCodeStepper):
+    "Step through Python code."
+
+class PyNext(PythonCodeStepper):
+    "Step-over Python code."
+
+py_step = PyStep('py-step', stepper=True)
+py_next = PyNext('py-next', stepper=False)
+
+class PyShowCCode(gdb.Parameter):
+    pass
\ No newline at end of file

bin/cygdb

@@ -6,7 +6,14 @@ from Cython.Debugger import Cygdb as cygdb
 
 if __name__ == '__main__':
     if len(sys.argv) > 1:
-        cygdb.main(path_to_debug_info=sys.argv[1], 
-                   gdb_argv=sys.argv[2:])
+        path_to_debug_info = sys.argv[1]
+        
+        no_import = False
+        if path_to_debug_info == '--':
+            no_import = True
+        
+        cygdb.main(path_to_debug_info, 
+                   gdb_argv=sys.argv[2:],
+                   no_import=no_import)
     else:
         cygdb.main()

cygdb.py

@@ -6,7 +6,14 @@ from Cython.Debugger import Cygdb as cygdb
 
 if __name__ == '__main__':
     if len(sys.argv) > 1:
-        cygdb.main(path_to_debug_info=sys.argv[1], 
-                   gdb_argv=sys.argv[2:])
+        path_to_debug_info = sys.argv[1]
+        
+        no_import = False
+        if path_to_debug_info == '--':
+            no_import = True
+        
+        cygdb.main(path_to_debug_info, 
+                   gdb_argv=sys.argv[2:],
+                   no_import=no_import)
     else:
         cygdb.main()