Tests!

(run: python runtests.py Cython.Tests.TestStringIOTree \
                         Cython.Debugger.Tests.TestLibCython \
                         Cython.Compiler.Tests.TestParseTreeTransforms)

--HG--
rename : Cython/Debugger/cygdb.py => Cython/Debugger/Cygdb.py

Cython/Compiler/Main.py

@@ -181,9 +181,10 @@ class Context(object):
             test_support.append(TreeAssertVisitor())
 
         if options.debug:
-            from Cython.Debugger import debug_output
+            from Cython.Debugger import DebugWriter
             from ParseTreeTransforms import DebugTransform
-            self.debug_outputwriter = debug_output.CythonDebugWriter(options)
+            self.debug_outputwriter = DebugWriter.CythonDebugWriter(
+                options.output_dir)
             debug_transform = [DebugTransform(self)]
         else:
             debug_transform = []

Cython/Compiler/ParseTreeTransforms.py

@@ -1523,7 +1523,7 @@ class DebugTransform(CythonTransform):
     def serialize_local_variables(self, entries):
         for entry in entries.values():
             if entry.type.is_pyobject:
-                vartype = 'PyObject'
+                vartype = 'PythonObject'
             else:
                 vartype = 'CObject'
             

Cython/Compiler/Tests/TestParseTreeTransforms.py

+import os
+
+from Cython.Debugger import DebugWriter
+from Cython.Compiler import Main
+from Cython.Compiler import CmdLine
 from Cython.TestUtils import TransformTest
 from Cython.Compiler.ParseTreeTransforms import *
 from Cython.Compiler.Nodes import *
+from Cython.Debugger.Tests import TestLibCython
 
 class TestNormalizeTree(TransformTest):
     def test_parserbehaviour_is_what_we_coded_for(self):
@@ -139,6 +145,66 @@ class TestWithTransform(object): # (TransformTest): # Disabled!
 
         """, t)
                           
+        
+class TestDebugTransform(TestLibCython.DebuggerTestCase):
+    
+    def elem_hasattrs(self, elem, attrs):
+        return all(attr in elem.attrib for attr in attrs)
+    
+    def test_debug_info(self):
+        try:
+            assert os.path.exists(self.debug_dest)
+            
+            t = DebugWriter.etree.parse(self.debug_dest)
+            # the xpath of the standard ElementTree is primitive, don't use
+            # anything fancy
+            L = list(t.find('/Module/Globals'))
+            # assertTrue is retarded, use the normal assert statement
+            assert L
+            xml_globals = dict((e.attrib['name'], e.attrib['type']) for e in L)
+            self.assertEqual(len(L), len(xml_globals))
+            
+            L = list(t.find('/Module/Functions'))
+            assert L
+            xml_funcs = dict((e.attrib['qualified_name'], e) for e in L)
+            self.assertEqual(len(L), len(xml_funcs))
+            
+            # test globals
+            self.assertEqual('CObject', xml_globals.get('c_var'))
+            self.assertEqual('PythonObject', xml_globals.get('python_var'))
+            
+            # test functions
+            funcnames = 'codefile.spam', 'codefile.ham', 'codefile.eggs'
+            required_xml_attrs = 'name', 'cname', 'qualified_name'
+            assert all(f in xml_funcs for f in funcnames)
+            spam, ham, eggs = (xml_funcs[funcname] for funcname in funcnames) 
+            
+            self.assertEqual(spam.attrib['name'], 'spam')
+            self.assertNotEqual('spam', spam.attrib['cname'])
+            assert self.elem_hasattrs(spam, required_xml_attrs)
+
+            # test locals of functions
+            spam_locals = list(spam.find('Locals'))
+            assert spam_locals
+            spam_locals.sort(key=lambda e: e.attrib['name'])
+            names = [e.attrib['name'] for e in spam_locals]
+            self.assertEqual(list('abcd'), names)
+            assert self.elem_hasattrs(spam_locals[0], required_xml_attrs)
+            
+            # test arguments of functions
+            spam_arguments = list(spam.find('Arguments'))
+            assert spam_arguments
+            self.assertEqual(1, len(list(spam_arguments)))
+            
+            # test step-into functions
+            spam_stepinto = list(spam.find('StepIntoFunctions'))
+            assert spam_stepinto
+            self.assertEqual(1, len(list(spam_stepinto)))
+            self.assertEqual('puts', list(spam_stepinto)[0].attrib['name'])
+        except:
+            print open(self.debug_dest).read()
+            raise
+            
 
 if __name__ == "__main__":
     import unittest

Cython/Debugger/cygdb.py → Cython/Debugger/Cygdb.py

@@ -20,36 +20,33 @@ import subprocess
 def usage():
     print("Usage: cygdb [PATH GDB_ARGUMENTS]")
 
-def main(gdb_argv=[], import_libpython=False, path_to_debug_info=os.curdir):
-    """
-    Start the Cython debugger. This tells gdb to import the Cython and Python
-    extensions (libpython.py and libcython.py) and it enables gdb's pending 
-    breakpoints
-    
-    import_libpython indicates whether we should just 'import libpython',
-    or import it from Cython.Debugger
-    
-    path_to_debug_info is the path to the cython_debug directory
-    """
+def make_command_file(path_to_debug_info):
     debug_files = glob.glob(
         os.path.join(path_to_debug_info, 'cython_debug/cython_debug_info_*'))
 
     if not debug_files:
         usage()
         sys.exit('No debug files were found in %s. Aborting.' % (
-                 os.path.abspath(path_to_debug_info))) 
+                 os.path.abspath(path_to_debug_info)))
         
     fd, tempfilename = tempfile.mkstemp()
     f = os.fdopen(fd, 'w')
     f.write('set breakpoint pending on\n')
     f.write('python from Cython.Debugger import libcython\n')
-    if import_libpython:
-        f.write('python import libpython')
-    else:
-        f.write('python from Cython.Debugger import libpython\n')
     f.write('\n'.join('cy import %s\n' % fn for fn in debug_files))
     f.close()
+    
+    return tempfilename
 
+def main(gdb_argv=[], path_to_debug_info=os.curdir):
+    """
+    Start the Cython debugger. This tells gdb to import the Cython and Python
+    extensions (libpython.py and libcython.py) and it enables gdb's pending 
+    breakpoints
+    
+    path_to_debug_info is the path to the cython_debug directory
+    """
+    tempfilename = make_command_file(path_to_debug_info)
     p = subprocess.Popen(['gdb', '-command', tempfilename] + gdb_argv)
     while True:
         try:

Cython/Debugger/Tests/TestLibCython.py

+import os
+import re
+import sys
+import uuid
+import shutil
+import textwrap
+import unittest
+import tempfile
+import subprocess
+import distutils.core
+from distutils import sysconfig
+
+import Cython.Distutils.extension
+from Cython.Debugger import Cygdb as cygdb
+
+
+class DebuggerTestCase(unittest.TestCase):
+    
+    def setUp(self):
+        """
+        Run gdb and have cygdb import the debug information from the code
+        defined in TestParseTreeTransforms's setUp method
+        """
+        self.tempdir = tempfile.mkdtemp()
+        self.destfile = os.path.join(self.tempdir, 'codefile.pyx')
+        self.debug_dest = os.path.join(self.tempdir, 
+                                      'cython_debug', 
+                                      'cython_debug_info_codefile')
+        
+        code = textwrap.dedent("""
+            cdef extern from "stdio.h":
+                int puts(char *s)
+                
+            cdef int c_var = 0
+            python_var = 0
+            
+            def spam(a=0):
+                cdef:
+                    int b, c, d
+                
+                b = c = d = 0
+                
+                b = 1
+                c = 2
+                d = 3
+                int(10)
+                puts("spam")
+                
+            cdef ham():
+                pass
+                
+            cpdef eggs():
+                pass
+            
+            cdef class SomeClass(object):
+                def spam(self):
+                    pass
+            
+            spam()
+        """)
+        
+        self.cwd = os.getcwd()
+        os.chdir(self.tempdir)
+        
+        open(self.destfile, 'w').write(code)
+        
+        ext = Cython.Distutils.extension.Extension(
+            'codefile',
+            ['codefile.pyx'], 
+            pyrex_debug=True)
+            
+        distutils.core.setup(
+            script_args=['build_ext', '--inplace'],
+            ext_modules=[ext], 
+            cmdclass=dict(build_ext=Cython.Distutils.build_ext)
+        )
+    
+    def tearDown(self):
+        os.chdir(self.cwd)
+        shutil.rmtree(self.tempdir)
+
+
+class GdbDebuggerTestCase(DebuggerTestCase):
+    def setUp(self):
+        super(GdbDebuggerTestCase, self).setUp()
+        
+        self.gdb_command_file = cygdb.make_command_file(self.tempdir)
+        with open(self.gdb_command_file, 'a') as f:
+            f.write('python '
+                'from Cython.Debugger.Tests import test_libcython_in_gdb;'
+                'test_libcython_in_gdb.main()\n')
+                
+        args = ['gdb', '-batch', '-x', self.gdb_command_file, '-n', '--args',
+                sys.executable, '-c', 'import codefile']
+        
+        paths = []
+        path = os.environ.get('PYTHONPATH')
+        if path:
+            paths.append(path)
+        paths.append(os.path.dirname(os.path.dirname(
+            os.path.abspath(Cython.__file__))))
+        env = dict(os.environ, PYTHONPATH=os.pathsep.join(paths))
+        self.p = subprocess.Popen(
+            args,
+            stdout=open(os.devnull, 'w'),
+            stderr=subprocess.PIPE,
+            env=env)
+        
+    def tearDown(self):
+        super(GdbDebuggerTestCase, self).tearDown()
+        self.p.stderr.close()
+        self.p.wait()
+        os.remove(self.gdb_command_file)
+        
+   
+class TestAll(GdbDebuggerTestCase):
+    
+    def test_all(self):
+        out, err = self.p.communicate()
+        border = '*' * 30
+        start = '%s   v INSIDE GDB v   %s' % (border, border)
+        end   = '%s   ^ INSIDE GDB ^   %s' % (border, border)
+        errmsg = '\n%s\n%s%s' % (start, err, end)
+        self.assertEquals(0, self.p.wait(), errmsg)
+        sys.stderr.write(err)
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

Cython/Debugger/Tests/test_libcython_in_gdb.py

+"""
+Tests that run inside GDB.
+
+Note: debug information is already imported by the file generated by
+Cython.Debugger.Cygdb.make_command_file()
+"""
+
+import sys
+
+# First, fix gdb's python. Make sure to do this before importing modules
+# that bind output streams as default parameters
+
+# for some reason sys.argv is missing in gdb
+sys.argv = ['gdb']
+
+# Allow gdb to capture output, but have errors end up on stderr
+# sys.stdout = sys.__stdout__
+sys.stderr = sys.__stderr__
+
+import os
+import warnings
+import unittest
+import traceback
+from test import test_support
+
+import gdb
+
+from Cython.Debugger import libcython
+
+class DebugTestCase(unittest.TestCase):
+    
+    def __init__(self, name):
+        super(DebugTestCase, self).__init__(name)
+        self.cy = libcython.cy
+        self.module = libcython.cy.cython_namespace['codefile']
+        self.spam_func, self.spam_meth = libcython.cy.functions_by_name['spam']
+        self.ham_func = libcython.cy.functions_by_qualified_name[
+            'codefile.ham']
+        self.eggs_func = libcython.cy.functions_by_qualified_name[
+            'codefile.eggs']
+    
+    def read_var(self, varname):
+        return gdb.parse_and_eval('$cy_cname("%s")' % varname)
+    
+    def local_info(self):
+        return gdb.execute('info locals', to_string=True)
+    
+class TestDebugInformationClasses(DebugTestCase):
+    
+    
+    def test_CythonModule(self):
+        "test that debug information was parsed properly into data structures"
+        self.assertEqual(self.module.name, 'codefile')
+        global_vars = ('c_var', 'python_var', 'SomeClass', '__name__', 
+                       '__builtins__', '__doc__', '__file__')
+        assert set(global_vars).issubset(self.module.globals)
+        
+    def test_CythonVariable(self):
+        module_globals = self.module.globals
+        c_var = module_globals['c_var']
+        python_var = module_globals['python_var']
+        self.assertEqual(c_var.type, libcython.CObject)
+        self.assertEqual(python_var.type, libcython.PythonObject)
+        self.assertEqual(c_var.qualified_name, 'codefile.c_var')
+    
+    def test_CythonFunction(self):
+        self.assertEqual(self.spam_func.qualified_name, 'codefile.spam')
+        self.assertEqual(self.spam_meth.qualified_name, 
+                         'codefile.SomeClass.spam')
+        self.assertEqual(self.spam_func.module, self.module)
+        
+        assert self.eggs_func.pf_cname
+        assert not self.ham_func.pf_cname
+        assert not self.spam_func.pf_cname
+        assert not self.spam_meth.pf_cname
+        
+        self.assertEqual(self.spam_func.type, libcython.CObject)
+        self.assertEqual(self.ham_func.type, libcython.CObject)
+        
+        self.assertEqual(self.spam_func.arguments, ['a'])
+        self.assertEqual(self.spam_func.step_into_functions, set(['puts']))
+        
+        self.assertEqual(self.spam_func.lineno, 8)
+        self.assertEqual(sorted(self.spam_func.locals), list('abcd'))
+
+
+class TestParameters(unittest.TestCase):
+    
+    def test_parameters(self):
+        assert libcython.parameters.colorize_code
+        gdb.execute('set cy_colorize_code off')
+        assert not libcython.parameters.colorize_code
+
+
+class TestBreak(DebugTestCase):
+
+    def test_break(self):
+        result = gdb.execute('cy break codefile.spam', to_string=True)
+        assert self.spam_func.cname in result
+        
+        self.assertEqual(len(gdb.breakpoints()), 1)
+        bp, = gdb.breakpoints()
+        self.assertEqual(bp.type, gdb.BP_BREAKPOINT)
+        self.assertEqual(bp.location, self.spam_func.cname)
+        assert bp.enabled
+
+
+class TestStep(DebugTestCase):
+    
+    def test_step(self):
+        # Note: breakpoint for spam is still set
+        gdb.execute('run')
+        
+        gdb.execute('cy step', to_string=True) # b = c = d = 0
+        gdb.execute('cy step', to_string=True) # b = 1
+        self.assertEqual(self.read_var('b'), 1, self.local_info())
+        self.assertRaises(RuntimeError, self.read_var('b'))
+        gdb.execute('cont')
+
+
+class TestNext(DebugTestCase):
+    
+    def test_next(self):
+        pass
+    
+def main():
+    try:
+        # unittest.main(module=__import__(__name__, fromlist=['']))
+        try:
+            gdb.lookup_type('PyModuleObject')
+        except RuntimeError:
+            msg = ("Unable to run tests, Python was not compiled with "
+                   "debugging information. Either compile python with "
+                   "-g or get a debug build (configure with --with-pydebug).")
+            warnings.warn(msg)
+        else:
+            tests = (
+                TestDebugInformationClasses,
+                TestParameters,
+                TestBreak,
+                TestStep
+            )
+            # test_support.run_unittest(tests)
+            test_loader = unittest.TestLoader()
+            suite = unittest.TestSuite(
+                [test_loader.loadTestsFromTestCase(cls) for cls in tests])
+                
+            unittest.TextTestRunner(verbosity=1).run(suite)
+    except Exception:
+        traceback.print_exc()
+        os._exit(1)
\ No newline at end of file

Cython/Debugger/libcython.py

@@ -45,30 +45,17 @@ else:
 
 from Cython.Debugger import libpython
 
-
-# Cython module namespace
-cython_namespace = {}
-
 # C or Python type
-CObject = object()
-PythonObject = object()
-
-# maps (unique) qualified function names (e.g. 
-# cythonmodule.ClassName.method_name) to the CythonFunction object
-functions_by_qualified_name = {}
-
-# unique cnames of Cython functions
-functions_by_cname = {}
-
-# map function names like method_name to a list of all such CythonFunction
-# objects
-functions_by_name = collections.defaultdict(list)
+CObject = 'CObject'
+PythonObject = 'PythonObject'
 
+_data_types = dict(CObject=CObject, PythonObject=PythonObject)
 _filesystemencoding = sys.getfilesystemencoding() or 'UTF-8'
 
 # decorators
 
 def dont_suppress_errors(function):
+    "*sigh*, readline"
     @functools.wraps(function)
     def wrapper(*args, **kwargs):
         try:
@@ -79,17 +66,31 @@ def dont_suppress_errors(function):
     
     return wrapper
 
-def default_selected_gdb_frame(function):
-    @functools.wraps(function)
-    def wrapper(self, frame=None, **kwargs):
-        frame = frame or gdb.selected_frame()
-        if frame.name() is None:
-            raise NoFunctionNameInFrameError()
-
-        return function(self, frame)
-    return wrapper
-
+def default_selected_gdb_frame(err=True):
+    def decorator(function):
+        @functools.wraps(function)
+        def wrapper(self, frame=None, **kwargs):
+            try:
+                frame = frame or gdb.selected_frame()
+            except RuntimeError:
+                raise gdb.GdbError("No frame is currently selected.")
+                
+            if err and frame.name() is None:
+                raise NoFunctionNameInFrameError()
+    
+            return function(self, frame, **kwargs)
+        return wrapper
+    return decorator
 
+def require_cython_frame(function):
+    @functools.wraps(function)
+    def wrapper(self, *args, **kwargs):
+        if not self.is_cython_function():
+            raise gdb.GdbError('Selected frame does not correspond with a '
+                               'Cython function we know about.')
+        return function(self, *args, **kwargs)
+    return wrapper 
+        
 # Classes that represent the debug information
 # Don't rename the parameters of these classes, they come directly from the XML
 
@@ -97,12 +98,14 @@ class CythonModule(object):
     def __init__(self, module_name, filename):
         self.name = module_name
         self.filename = filename
-        self.functions = {}
         self.globals = {}
         # {cython_lineno: min(c_linenos)}
         self.lineno_cy2c = {}
         # {c_lineno: cython_lineno}
         self.lineno_c2cy = {}
+    
+    def qualified_name(self, varname):
+        return '.'.join(self.name, varname)
 
 class CythonVariable(object):
 
@@ -127,15 +130,78 @@ class CythonFunction(CythonVariable):
                                              type)
         self.module = module
         self.pf_cname = pf_cname
-        self.lineno = lineno
+        self.lineno = int(lineno)
         self.locals = {}
         self.arguments = []
         self.step_into_functions = set()
 
+
+# General purpose classes
+
+class CythonBase(object):
+    
+    @default_selected_gdb_frame(err=False)
+    def is_cython_function(self, frame):
+        return frame.name() in self.cy.functions_by_cname
+
+    @default_selected_gdb_frame(err=False)
+    def is_python_function(self, frame):
+        return frame.name() == 'PyEval_EvalFrameEx'
+
+    @default_selected_gdb_frame()
+    def is_python_function(self, frame):
+        return libpython.Frame(frame).is_evalframeex()
+
+    @default_selected_gdb_frame()
+    def get_c_function_name(self, frame):
+        return frame.name()
+
+    @default_selected_gdb_frame()
+    def get_c_lineno(self, frame):
+        return frame.find_sal().line
+    
+    @default_selected_gdb_frame()
+    def get_cython_function(self, frame):
+        result = self.cy.functions_by_cname.get(frame.name())
+        if result is None:
+            raise NoCythonFunctionInFrameError()
+            
+        return result
+    
+    @default_selected_gdb_frame()
+    def get_cython_lineno(self, frame):
+        cyfunc = self.get_cython_function(frame)
+        return cyfunc.module.lineno_c2cy.get(self.get_c_lineno(frame))
+
+    @default_selected_gdb_frame()
+    def get_source_desc(self, frame):
+        filename = lineno = lexer = None
+        if self.is_cython_function():
+            filename = self.get_cython_function(frame).module.filename
+            lineno = self.get_cython_lineno(frame)
+            if pygments:
+                lexer = pygments.lexers.CythonLexer()
+        elif self.is_python_function():
+            pyframeobject = libpython.Frame(frame).get_pyop()
+
+            if not pyframeobject:
+                raise GdbError('Unable to read information on python frame')
+
+            filename = pyframeobject.filename()
+            lineno = pyframeobject.current_line_num()
+            if pygments:
+                lexer = pygments.lexers.PythonLexer()
+
+        return SourceFileDescriptor(filename, lexer), lineno
+
+    @default_selected_gdb_frame()
+    def get_source_line(self, frame):
+        source_desc, lineno = self.get_source_desc()
+        return source_desc.get_source(lineno)
+
 class SourceFileDescriptor(object):
-    def __init__(self, filename, lineno, lexer, formatter=None):
+    def __init__(self, filename, lexer, formatter=None):
         self.filename = filename
-        self.lineno = lineno
         self.lexer = lexer
         self.formatter = formatter
 
@@ -143,8 +209,8 @@ class SourceFileDescriptor(object):
         return self.filename is not None
 
     def lex(self, code):
-        if pygments and parameter.colorize_code:
-            bg = parameter.terminal_background.value
+        if pygments and self.lexer and parameters.colorize_code:
+            bg = parameters.terminal_background.value
             if self.formatter is None:
                 formatter = pygments.formatters.TerminalFormatter(bg=bg)
             else:
@@ -154,25 +220,31 @@ class SourceFileDescriptor(object):
 
         return code
 
-    def get_source(self, start=0, stop=None, lex_source=True):
-        # todo: have it detect the source file's encoding
-        if not self.filename:
-            return 'Unable to retrieve source code'
-
-        start = max(self.lineno + start, 0)
-        if stop is None:
-            stop = self.lineno + 1
-        else:
-            stop = self.lineno + stop
-            
+    def _get_source(self, start, stop, lex_source, mark_line):
         with open(self.filename) as f:
-            source = itertools.islice(f, start, stop)
-            
             if lex_source:
-                return [self.lex(line) for line in source]
+                # to provide proper colouring, the entire code needs to be 
+                # lexed
+                lines = self.lex(f.read()).splitlines()
             else:
-                return list(source)
+                lines = f
+            
+            for idx, line in enumerate(itertools.islice(lines, start - 1, stop - 1)):
+                if start + idx == mark_line:
+                    prefix = '>'
+                else:
+                    prefix = ' '
+                
+                yield '%s %4d    %s' % (prefix, start + idx, line)
 
+    def get_source(self, start, stop=None, lex_source=True, mark_line=0):
+        if not self.filename:
+            raise GdbError('Unable to retrieve source code')
+
+        if stop is None:
+            stop = start + 1
+        return '\n'.join(self._get_source(start, stop, lex_source, mark_line))
+        
 
 # Errors
 
@@ -235,32 +307,41 @@ class TerminalBackground(CythonParameter):
     Tell cygdb about the user's terminal background (light or dark)
     """
     
-class Parameter(object):
+class CythonParameters(object):
     """
     Simple container class that might get more functionality in the distant
     future (mostly to remind us that we're dealing with parameters)
     """
-    complete_unqualified = CompleteUnqualifiedFunctionNames(
-        'cy_complete_unqualified',
-        gdb.COMMAND_BREAKPOINTS,
-        gdb.PARAM_BOOLEAN,
-        True)
-    colorize_code = ColorizeSourceCode(
-        'cy_colorize_code',
-        gdb.COMMAND_FILES,
-        gdb.PARAM_BOOLEAN,
-        True)
-    terminal_background = TerminalBackground(
-        'cy_terminal_background_color',
-        gdb.COMMAND_FILES,
-        gdb.PARAM_STRING,
-        "dark")
-
-parameter = Parameter()
+    
+    def __init__(self):
+        self.complete_unqualified = CompleteUnqualifiedFunctionNames(
+            'cy_complete_unqualified',
+            gdb.COMMAND_BREAKPOINTS,
+            gdb.PARAM_BOOLEAN,
+            True)
+        self.colorize_code = ColorizeSourceCode(
+            'cy_colorize_code',
+            gdb.COMMAND_FILES,
+            gdb.PARAM_BOOLEAN,
+            True)
+        self.terminal_background = TerminalBackground(
+            'cy_terminal_background_color',
+            gdb.COMMAND_FILES,
+            gdb.PARAM_STRING,
+            "dark")
+
+parameters = CythonParameters()
+
 
 # Commands
 
-class CythonCommand(gdb.Command):
+class CythonCommand(gdb.Command, CythonBase):
+    """
+    Base class for Cython commands
+    """
+
+
+class CyCy(CythonCommand):
     """
     Invoke a Cython command. Available commands are:
         
@@ -272,53 +353,49 @@ class CythonCommand(gdb.Command):
         cy locals
         cy globals
         cy backtrace
-        cy info line
+        cy up
+        cy down
     """
     
-    def is_cython_function(self, frame=None):
-        func_name = (frame or gdb.selected_frame()).name()
-        return func_name is not None and func_name in functions_by_cname
-
-    @default_selected_gdb_frame
-    def is_python_function(self, frame):
-        return libpython.Frame(frame).is_evalframeex()
-
-    @default_selected_gdb_frame
-    def get_c_function_name(self, frame):
-        return frame.name()
-
-    @default_selected_gdb_frame
-    def get_c_lineno(self, frame):
-        return frame.find_sal().line
-    
-    @default_selected_gdb_frame
-    def get_cython_function(self, frame):
-        result = functions_by_cname.get(frame.name())
-        if result is None:
-            raise NoCythonFunctionInFrameError()
+    def __init__(self):
+        super(CythonCommand, self).__init__(
+            'cy', gdb.COMMAND_NONE, gdb.COMPLETE_COMMAND, prefix=True)
+        
+        self.import_ = CyImport(
+            'cy import', gdb.COMMAND_STATUS, gdb.COMPLETE_FILENAME)
             
-        return result
-    
-    @default_selected_gdb_frame
-    def get_cython_lineno(self, frame):
-        cyfunc = self.get_cython_function(frame)
-        return cyfunc.module.lineno_c2cy.get(self.get_c_lineno(frame))
-
-    @default_selected_gdb_frame
-    def get_source_desc(self, frame):
-        if self.is_cython_function():
-            filename = self.get_cython_function(frame).module.filename
-            lineno = self.get_cython_lineno(frame)
-            lexer = pygments.lexers.CythonLexer()
-        else:
-            filename = None
-            lineno = -1
-            lexer = None
-
-        return SourceFileDescriptor(filename, lineno, lexer)
+        self.break_ = CyBreak('cy break', gdb.COMMAND_BREAKPOINTS)
+        self.step = CyStep('cy step', gdb.COMMAND_RUNNING, gdb.COMPLETE_NONE)
+        self.next = CyNext('cy next', gdb.COMMAND_RUNNING, gdb.COMPLETE_NONE)
+        self.list = CyList('cy list', gdb.COMMAND_FILES, gdb.COMPLETE_NONE)
+        self.print_ = CyPrint('cy print', gdb.COMMAND_DATA)
+        
+        self.locals = CyLocals(
+            'cy locals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE)
+        self.globals = CyGlobals(
+            'cy globals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE)
+            
+        self.cy_cname = CyCName('cy_cname')
+        
+        objs = (self.import_, self.break_, self.step, self.list, self.print_,
+                self.locals, self.globals, self.cy_cname)
 
-   
-cy = CythonCommand('cy', gdb.COMMAND_NONE, gdb.COMPLETE_COMMAND, prefix=True)
+        for obj in objs:
+            obj.cy = self
+            
+        # Cython module namespace
+        self.cython_namespace = {}
+        
+        # maps (unique) qualified function names (e.g. 
+        # cythonmodule.ClassName.method_name) to the CythonFunction object
+        self.functions_by_qualified_name = {}
+        
+        # unique cnames of Cython functions
+        self.functions_by_cname = {}
+        
+        # map function names like method_name to a list of all such 
+        # CythonFunction objects
+        self.functions_by_name = collections.defaultdict(list)
 
 
 class CyImport(CythonCommand):
@@ -340,7 +417,7 @@ class CyImport(CythonCommand):
             
             for module in t.getroot():
                 cython_module = CythonModule(**module.attrib)
-                cython_namespace[cython_module.name] = cython_module
+                self.cy.cython_namespace[cython_module.name] = cython_module
                 
                 for variable in module.find('Globals'):
                     d = variable.attrib
@@ -349,15 +426,14 @@ class CyImport(CythonCommand):
                 for function in module.find('Functions'):
                     cython_function = CythonFunction(module=cython_module, 
                                                      **function.attrib)
-                    cython_module.functions[cython_function.name] = \
-                        cython_function
                     
                     # update the global function mappings
-                    functions_by_name[cython_function.name].append(
+                    self.cy.functions_by_name[cython_function.name].append(
                         cython_function)
-                    functions_by_qualified_name[
+                    self.cy.functions_by_qualified_name[
                         cython_function.qualified_name] = cython_function
-                    functions_by_cname[cython_function.cname] = cython_function
+                    self.cy.functions_by_cname[
+                        cython_function.cname] = cython_function
                     
                     for local in function.find('Locals'):
                         d = local.attrib
@@ -377,9 +453,6 @@ class CyImport(CythonCommand):
                     for c_lineno in c_linenos:
                         cython_module.lineno_c2cy[c_lineno] = cython_lineno
                     
-                
-cy.import_ = CyImport('cy import', gdb.COMMAND_STATUS, gdb.COMPLETE_FILENAME)
-
 
 class CyBreak(CythonCommand):
     """
@@ -399,20 +472,21 @@ class CyBreak(CythonCommand):
     def _break_pyx(self, name):
         modulename, _, lineno = name.partition(':')
         lineno = int(lineno)
-        cython_module = cython_namespace[modulename]
+        cython_module = self.cy.cython_namespace[modulename]
         if lineno in cython_module.lineno_cy2c:
             c_lineno = cython_module.lineno_cy2c[lineno]
             breakpoint = '%s:%s' % (cython_module.name, c_lineno)
             gdb.execute('break ' + breakpoint)
         else:
-            sys.stderr.write("Not a valid line number (does it contain actual code?)\n")
+            raise GdbError("Not a valid line number. "
+                           "Does it contain actual code?")
     
     def _break_funcname(self, funcname):
-        func = functions_by_qualified_name.get(funcname)
+        func = self.cy.functions_by_qualified_name.get(funcname)
         break_funcs = [func]
         
         if not func:
-            funcs = functions_by_name.get(funcname)
+            funcs = self.cy.functions_by_name.get(funcname)
             if not funcs:
                 gdb.execute('break ' + funcname)
                 return
@@ -459,13 +533,13 @@ class CyBreak(CythonCommand):
     
     @dont_suppress_errors
     def complete(self, text, word):
-        names = functions_by_qualified_name
-        if parameter.complete_unqualified:
-            names = itertools.chain(names, functions_by_name)
+        names = self.cy.functions_by_qualified_name
+        if parameters.complete_unqualified:
+            names = itertools.chain(names, self.cy.functions_by_name)
 
         words = text.strip().split()
         if words and '.' in words[-1]:
-            compl = [n for n in functions_by_qualified_name 
+            compl = [n for n in self.cy.functions_by_qualified_name 
                            if n.startswith(lastword)]
         else:
             seen = set(text[:-len(word)].split())
@@ -479,27 +553,37 @@ class CyBreak(CythonCommand):
             
         return compl
 
-cy.break_ = CyBreak('cy break', gdb.COMMAND_BREAKPOINTS)
+
+class CodeStepperMixin(object):
+    
+    def init_stepping(self):
+        self.cython_func = self.get_cython_function()
+        self.beginline = self.get_cython_lineno()
+        self.curframe = gdb.selected_frame() 
+
+    def next_step(self, command):
+        "returns whether to continue stepping"
+        result = gdb.execute(command, to_string=True)
+        newframe = gdb.selected_frame()
+        
+        c1 = result.startswith('Breakpoint')
+        c2 = (newframe == self.curframe and 
+              self.get_cython_lineno() > self.beginline)
+        return not c1 and not c2
+        
+    def end_stepping(self):
+        sys.stdout.write(self.get_source_line())
 
 
-class CyStep(CythonCommand):
+class CyStep(CythonCommand, CodeStepperMixin):
 
-    def step(self, from_tty=True, nsteps=1):
+    def step(self, nsteps=1):
         for nthstep in xrange(nsteps):
-            cython_func = self.get_cython_function()
-            beginline = self.get_cython_lineno()
-            curframe = gdb.selected_frame() 
-    
-            while True:
-                result = gdb.execute('step', False, True)
-                if result.startswith('Breakpoint'):
-                        break
+            self.init_stepping()
+            
+            while self.next_step('step'):
                 newframe = gdb.selected_frame()
-                if newframe == curframe:
-                    # still in the same function
-                    if self.get_cython_lineno() > beginline:
-                        break
-                else:
+                if newframe != self.curframe:
                     # we entered a function
                     funcname = self.get_c_function_name(newframe)
                     if (self.is_cython_function() or 
@@ -507,64 +591,73 @@ class CyStep(CythonCommand):
                         funcname in cython_function.step_into_functions):
                         break
         
-        line, = self.get_source_desc().get_source()
-        sys.stdout.write(line)
+        self.end_stepping()
 
     def invoke(self, steps, from_tty):
         if self.is_cython_function():
             if steps:
-                self.step(from_tty, int(steps))
+                self.step(int(steps))
             else:
-                self.step(from_tty)
+                self.step()
         else:
-            gdb.execute('step ' + steps)
+            gdb.execute('step ' + steps, from_tty)
 
-cy.step = CyStep('cy step', gdb.COMMAND_RUNNING, gdb.COMPLETE_NONE)
+
+class CyNext(CythonCommand, CodeStepperMixin):
+    
+    def next(self, nsteps=1):
+        for nthstep in xrange(nsteps):
+            self.init_stepping()
+            
+            while self.next_step('next'):
+                pass
+            
+            self.end_stepping()
+    
+    def invoke(self, steps, from_tty):
+        if self.is_cython_function():
+            if steps:
+                self.next(int(steps))
+            else:
+                self.next()
+        else:
+            gdb.execute('next ' + steps, from_tty)
 
 
 class CyList(CythonCommand):
     
     def invoke(self, _, from_tty):
-        sd = self.get_source_desc()
-        it = enumerate(sd.get_source(-5, +5))
-        sys.stdout.write(
-            ''.join('%4d    %s' % (sd.lineno + i, line) for i, line in it))
-
-cy.list = CyList('cy list', gdb.COMMAND_FILES, gdb.COMPLETE_NONE)
+        sd, lineno = self.get_source_desc()
+        source = sd.get_source(lineno - 5, lineno + 5, mark_line=lineno)
+        print source
 
 
 class CyPrint(CythonCommand):
     """
     Print a Cython variable using 'cy-print x' or 'cy-print module.function.x'
     """
-   
     
     def invoke(self, name, from_tty):
-        cname = None
-        if self.is_cython_function():
-            cython_function = self.get_cython_function()
-            if name in cython_function.locals:
-                cname = cython_function.locals[name].cname
-            elif name in cython_function.module.globals:
-                cname = cython_function.module.globals[name].cname
-
-        # let the pretty printers do the work
-        cname = cname or name
+        try:
+            cname = cy.cy_cname.invoke(name)
+        except gdb.GdbError:
+            cname = name
+       
         gdb.execute('print ' + cname)
+        
     
     def complete(self):
         if self.is_cython_function():
-            cf = self.get_cython_function()
-            return list(itertools.chain(cf.locals, cf.globals))
+            f = self.get_cython_function()
+            return list(itertools.chain(f.locals, f.globals))
         return []
-    
-cy.print_ = CyPrint('cy print', gdb.COMMAND_DATA)
 
 
 class CyLocals(CythonCommand):
     def ns(self):
         return self.get_cython_function().locals
-        
+    
+    @require_cython_frame
     def invoke(self, name, from_tty):
         try:
             ns = self.ns()
@@ -573,9 +666,10 @@ class CyLocals(CythonCommand):
             return
         
         if ns is None:
-            print ('Information of Cython locals could not be obtained. '
-                   'Is this an actual Cython function and did you '
-                   "'cy import' the debug information?")
+            raise gdb.GdbError(
+                'Information of Cython locals could not be obtained. '
+                'Is this an actual Cython function and did you '
+                "'cy import' the debug information?")
         
         for var in ns.itervalues():
             val = gdb.parse_and_eval(var.cname)
@@ -591,6 +685,7 @@ class CyGlobals(CythonCommand):
     def ns(self):
         return self.get_cython_function().globals
     
+    @require_cython_frame
     def invoke(self, name, from_tty):
         # include globals from the debug info XML file!
         m = gdb.parse_and_eval('__pyx_m')
@@ -610,5 +705,30 @@ class CyGlobals(CythonCommand):
         print d.get_truncated_repr(1000)
 
 
-cy.locals = CyLocals('cy locals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE)
-cy.globals = CyGlobals('cy globals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE)
+# Functions
+
+class CyCName(gdb.Function, CythonBase):
+    """
+    Get the C name of a Cython variable.
+    """
+    
+    @require_cython_frame
+    def invoke(self, cyname, frame=None):
+        frame = frame or gdb.selected_frame()
+        cname = None
+        
+        cyname = cyname.string()
+        if self.is_cython_function(frame):
+            cython_function = self.get_cython_function(frame)
+            if cyname in cython_function.locals:
+                cname = cython_function.locals[cyname].cname
+            elif cyname in cython_function.module.globals:
+                cname = cython_function.module.globals[cyname].cname
+        
+        if not cname:
+            raise gdb.GdbError('No such Cython variable: %s' % cyname)
+        
+        return cname
+        
+
+cy = CyCy()
\ No newline at end of file

Cython/Debugger/libpython.py

 #!/usr/bin/python
+
+# NOTE: this file is taken from the Python source distribution
+# It can be found under Tools/gdb/libpython.py
+
 '''
 From gdb 7 onwards, gdb's build can be configured --with-python, allowing gdb
 to be extended with Python code e.g. for library-specific data visualizations,
@@ -41,6 +45,7 @@ The module also extends gdb with some python-specific commands.
 '''
 from __future__ import with_statement
 import gdb
+import sys
 
 # Look up the gdb.Type for some standard types:
 _type_char_ptr = gdb.lookup_type('char').pointer() # char*
@@ -1009,6 +1014,18 @@ class PyTypeObjectPtr(PyObjectPtr):
     _typename = 'PyTypeObject'
 
 
+if sys.maxunicode >= 0x10000:
+    _unichr = unichr
+else:
+    # Needed for proper surrogate support if sizeof(Py_UNICODE) is 2 in gdb
+    def _unichr(x):
+        if x < 0x10000:
+            return unichr(x)
+        x -= 0x10000
+        ch1 = 0xD800 | (x >> 10)
+        ch2 = 0xDC00 | (x & 0x3FF)
+        return unichr(ch1) + unichr(ch2)
+
 class PyUnicodeObjectPtr(PyObjectPtr):
     _typename = 'PyUnicodeObject'
 
@@ -1025,37 +1042,36 @@ class PyUnicodeObjectPtr(PyObjectPtr):
 
         # Gather a list of ints from the Py_UNICODE array; these are either
         # UCS-2 or UCS-4 code points:
-        Py_UNICODEs = [int(field_str[i]) for i in safe_range(field_length)]
+        if self.char_width() > 2:
+            Py_UNICODEs = [int(field_str[i]) for i in safe_range(field_length)]
+        else:
+            # A more elaborate routine if sizeof(Py_UNICODE) is 2 in the
+            # inferior process: we must join surrogate pairs.
+            Py_UNICODEs = []
+            i = 0
+            limit = safety_limit(field_length)
+            while i < limit:
+                ucs = int(field_str[i])
+                i += 1
+                if ucs < 0xD800 or ucs >= 0xDC00 or i == field_length:
+                    Py_UNICODEs.append(ucs)
+                    continue
+                # This could be a surrogate pair.
+                ucs2 = int(field_str[i])
+                if ucs2 < 0xDC00 or ucs2 > 0xDFFF:
+                    continue
+                code = (ucs & 0x03FF) << 10
+                code |= ucs2 & 0x03FF
+                code += 0x00010000
+                Py_UNICODEs.append(code)
+                i += 1
 
         # Convert the int code points to unicode characters, and generate a
-        # local unicode instance:
-        result = u''.join([unichr(ucs) for ucs in Py_UNICODEs])
+        # local unicode instance.
+        # This splits surrogate pairs if sizeof(Py_UNICODE) is 2 here (in gdb).
+        result = u''.join([_unichr(ucs) for ucs in Py_UNICODEs])
         return result
 
-    def write_repr(self, out, visited):
-        proxy = self.proxyval(visited)
-        if self.char_width() == 2:
-            # sizeof(Py_UNICODE)==2: join surrogates
-            proxy2 = []
-            i = 0
-            while i < len(proxy):
-                ch = proxy[i]
-                i += 1
-                if (i < len(proxy)
-                and 0xD800 <= ord(ch) < 0xDC00 \
-                and 0xDC00 <= ord(proxy[i]) <= 0xDFFF):
-                    # Get code point from surrogate pair
-                    ch2 = proxy[i]
-                    code = (ord(ch) & 0x03FF) << 10
-                    code |= ord(ch2) & 0x03FF
-                    code += 0x00010000
-                    i += 1
-                    proxy2.append(unichr(code))
-                else:
-                    proxy2.append(ch)
-            proxy = u''.join(proxy2)
-        out.write(repr(proxy))
-
 
 def int_from_int(gdbval):
     return int(str(gdbval))
@@ -1431,4 +1447,4 @@ class PyLocals(gdb.Command):
                    % (pyop_name.proxyval(set()),
                       pyop_value.get_truncated_repr(MAX_OUTPUT_LEN)))
 
-PyLocals()
+PyLocals()
\ No newline at end of file

Cython/StringIOTree.py

@@ -16,7 +16,6 @@ class StringIOTree(object):
     def getvalue(self):
         content = [x.getvalue() for x in self.prepended_children]
         content.append(self.stream.getvalue())
-        print self.linenumber_map()
         return "".join(content)
 
     def copyto(self, target):
@@ -33,6 +32,8 @@ class StringIOTree(object):
         # itself is empty -- this makes it ready for insertion
         if self.stream.tell():
             self.prepended_children.append(StringIOTree(self.stream))
+            self.prepended_children[-1].markers = self.markers
+            self.markers = []
             self.stream = StringIO()
             self.write = self.stream.write
 

bin/cygdb

@@ -2,7 +2,7 @@
 
 import sys
 
-from Cython.Debugger import cygdb
+from Cython.Debugger import Cygdb as cygdb
 
 if __name__ == '__main__':
     if len(sys.argv) > 1:

cygdb.py

@@ -2,7 +2,7 @@
 
 import sys
 
-from Cython.Debugger import cygdb
+from Cython.Debugger import Cygdb as cygdb
 
 if __name__ == '__main__':
     if len(sys.argv) > 1:

runtests.py

@@ -606,8 +606,13 @@ def collect_doctests(path, module_prefix, suite, selectors):
     def package_matches(dirname):
         return dirname not in ("Mac", "Distutils", "Plex")
     def file_matches(filename):
-        return (filename.endswith(".py") and not ('~' in filename
-                or '#' in filename or filename.startswith('.')))
+        filename, ext = os.path.splitext(filename)
+        blacklist = ('libcython', 'libpython', 'test_libcython_in_gdb')
+        return (ext == '.py' and not
+                '~' in filename and not
+                '#' in filename and not
+                filename.startswith('.') and not
+                filename in blacklist)
     import doctest, types
     for dirpath, dirnames, filenames in os.walk(path):
         parentname = os.path.split(dirpath)[-1]