Commit 1f4dc2f4 authored by Mark Florisson's avatar Mark Florisson

Line number support

C function with context support
Preliminary stepping support
Source code listing
more stuff I forgot about
parent 9eaba388
......@@ -876,6 +876,14 @@ class CCodeWriter(object):
return self.buffer.getvalue()
def write(self, s):
# also put invalid markers (lineno 0), to indicate that those lines
# have no Cython source code correspondence
if self.marker is None:
cython_lineno = self.last_marker_line
else:
cython_lineno = self.marker[0]
self.buffer.markers.extend([cython_lineno] * s.count('\n'))
self.buffer.write(s)
def insertion_point(self):
......@@ -954,6 +962,7 @@ class CCodeWriter(object):
self.emit_marker()
if self.emit_linenums and self.last_marker_line != 0:
self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))
if code:
if safe:
self.put_safe(code)
......
......@@ -87,6 +87,8 @@ class Context(object):
self.set_language_level(language_level)
self.debug_outputwriter = None
def set_language_level(self, level):
self.language_level = level
if level >= 3:
......@@ -179,8 +181,10 @@ class Context(object):
test_support.append(TreeAssertVisitor())
if options.debug:
from ParseTreeTransforms import DebuggerTransform
debug_transform = [DebuggerTransform(self, options.output_dir)]
from Cython.Debugger import debug_output
from ParseTreeTransforms import DebugTransform
self.debug_outputwriter = debug_output.CythonDebugWriter(options)
debug_transform = [DebugTransform(self)]
else:
debug_transform = []
......
......@@ -298,12 +298,34 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
f = open_new_file(result.c_file)
rootwriter.copyto(f)
if options.debug:
self._serialize_lineno_map(env, rootwriter)
f.close()
result.c_file_generated = 1
if Options.annotate or options.annotate:
self.annotate(rootwriter)
rootwriter.save_annotation(result.main_source_file, result.c_file)
def _serialize_lineno_map(self, env, ccodewriter):
tb = env.context.debug_outputwriter
markers = ccodewriter.buffer.allmarkers()
d = {}
for c_lineno, cython_lineno in enumerate(markers):
if cython_lineno > 0:
d.setdefault(cython_lineno, []).append(c_lineno + 1)
tb.start('LineNumberMapping')
for cython_lineno, c_linenos in sorted(d.iteritems()):
attrs = {
'c_linenos': ' '.join(map(str, c_linenos)),
'cython_lineno': str(cython_lineno),
}
tb.start('LineNumber', attrs)
tb.end('LineNumber')
tb.end('LineNumberMapping')
tb.serialize()
def find_referenced_modules(self, env, module_list, modules_seen):
if env not in modules_seen:
modules_seen[env] = 1
......
......@@ -7,7 +7,6 @@ from Cython.Compiler.UtilNodes import *
from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
from Cython.Compiler.StringEncoding import EncodedString
from Cython.Compiler.Errors import error, CompileError
from Cython.Compiler import Errors
try:
set
......@@ -15,31 +14,6 @@ except NameError:
from sets import Set as set
import copy
import os
import errno
try:
from lxml import etree
have_lxml = True
except ImportError:
have_lxml = False
try:
# Python 2.5
from xml.etree import cElementTree as etree
except ImportError:
try:
# Python 2.5
from xml.etree import ElementTree as etree
except ImportError:
try:
# normal cElementTree install
import cElementTree as etree
except ImportError:
try:
# normal ElementTree install
import elementtree.ElementTree as etree
except ImportError:
etree = None
class NameNodeCollector(TreeVisitor):
......@@ -1461,56 +1435,44 @@ class TransformBuiltinMethods(EnvTransform):
return node
def _create_xmlnode(tb, name, attrs=None):
"create a xml node with name name and attrs attrs given TreeBuilder tb"
tb.start(name, attrs or {})
tb.end(name)
class DebuggerTransform(CythonTransform):
class DebugTransform(CythonTransform):
"""
Class to output debugging information for cygdb
It writes debug information to cython_debug/cython_debug_info_<modulename>
in the build directory. Also sets all functions' visibility to extern to
enable debugging
Create debug information and all functions' visibility to extern in order
to enable debugging.
"""
def __init__(self, context, output_dir):
super(DebuggerTransform, self).__init__(context)
self.output_dir = os.path.join(output_dir, 'cython_debug')
if etree is None:
raise Errors.NoElementTreeInstalledException()
self.tb = etree.TreeBuilder()
def __init__(self, context):
super(DebugTransform, self).__init__(context)
self.visited = set()
# our treebuilder and debug output writer
# (see Cython.Debugger.debug_output.CythonDebugWriter)
self.tb = self.context.debug_outputwriter
def visit_ModuleNode(self, node):
self.module_name = node.full_module_name
self.tb.module_name = node.full_module_name
attrs = dict(
module_name=self.module_name,
module_name=node.full_module_name,
filename=node.pos[0].filename)
self.tb.start('Module', attrs)
# serialize functions
self.tb.start('Functions', {})
self.tb.start('Functions')
self.visitchildren(node)
self.tb.end('Functions')
# 2.3 compatibility. Serialize global variables
self.tb.start('Globals', {})
self.tb.start('Globals')
entries = {}
for k, v in node.scope.entries.iteritems():
if (v.qualified_name not in self.visited and
not v.name.startswith('__pyx_')):
# if v.qualified_name == 'testcython.G': import pdb; pdb.set_trace()
entries[k]= v
self.serialize_local_variables(entries)
self.tb.end('Globals')
self.tb.end('Module')
# self.tb.end('Module') # end Module after the line number mapping in
# Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
return node
def visit_FuncDefNode(self, node):
......@@ -1530,14 +1492,32 @@ class DebuggerTransform(CythonTransform):
self.tb.start('Function', attrs=attrs)
self.tb.start('Locals', {})
self.tb.start('Locals')
self.serialize_local_variables(node.local_scope.entries)
self.tb.end('Locals')
self.tb.start('Arguments', {})
self.tb.start('Arguments')
for arg in node.local_scope.arg_entries:
_create_xmlnode(self.tb, arg.name)
self.tb.start(arg.name)
self.tb.end(arg.name)
self.tb.end('Arguments')
self.tb.start('StepIntoFunctions')
self.visitchildren(node)
self.tb.end('StepIntoFunctions')
self.tb.end('Function')
return node
def visit_NameNode(self, node):
if (node.type.is_cfunction and
node.is_called and
node.entry.in_cinclude):
attrs = dict(name=node.entry.func_cname)
self.tb.start('StepIntoFunction', attrs=attrs)
self.tb.end('StepIntoFunction')
self.visitchildren(node)
return node
def serialize_local_variables(self, entries):
......@@ -1557,26 +1537,6 @@ class DebuggerTransform(CythonTransform):
qualified_name=entry.qualified_name,
type=vartype)
_create_xmlnode(self.tb, 'LocalVar', attrs)
def __call__(self, root):
self.tb.start('cython_debug', attrs=dict(version='1.0'))
super(DebuggerTransform, self).__call__(root)
self.tb.end('cython_debug')
xml_root_element = self.tb.close()
try:
os.makedirs(self.output_dir)
except OSError, e:
if e.errno != errno.EEXIST:
raise
et = etree.ElementTree(xml_root_element)
kw = {}
if have_lxml:
kw['pretty_print'] = True
fn = "cython_debug_info_" + self.module_name
et.write(os.path.join(self.output_dir, fn), encoding="UTF-8", **kw)
self.tb.start('LocalVar', attrs)
self.tb.end('LocalVar')
return root
\ No newline at end of file
......@@ -3,6 +3,7 @@ GDB extension that adds Cython support.
"""
import sys
import textwrap
import traceback
import functools
import itertools
......@@ -30,6 +31,13 @@ except ImportError:
# normal ElementTree install
import elementtree.ElementTree as etree
try:
import pygments.lexers
import pygments.formatters
except ImportError:
pygments = None
sys.stderr.write("Install pygments for colorized source code.\n")
if hasattr(gdb, 'string_to_argv'):
from gdb import string_to_argv
else:
......@@ -58,6 +66,8 @@ functions_by_name = collections.defaultdict(list)
_filesystemencoding = sys.getfilesystemencoding() or 'UTF-8'
# decorators
def dont_suppress_errors(function):
@functools.wraps(function)
def wrapper(*args, **kwargs):
......@@ -69,14 +79,33 @@ def dont_suppress_errors(function):
return wrapper
def default_selected_gdb_frame(function):
@functools.wraps(function)
def wrapper(self, frame=None, **kwargs):
frame = frame or gdb.selected_frame()
if frame.name() is None:
raise NoFunctionNameInFrameError()
return function(self, frame)
return wrapper
# Classes that represent the debug information
# Don't rename the parameters of these classes, they come directly from the XML
class CythonModule(object):
def __init__(self, module_name, filename):
self.name = module_name
self.filename = filename
self.functions = {}
self.globals = {}
# {cython_lineno: min(c_linenos)}
self.lineno_cy2c = {}
# {c_lineno: cython_lineno}
self.lineno_c2cy = {}
class CythonVariable(object):
def __init__(self, name, cname, qualified_name, type):
self.name = name
self.cname = cname
......@@ -92,13 +121,144 @@ class CythonFunction(CythonVariable):
qualified_name,
lineno,
type=CObject):
super(CythonFunction, self).__init__(name, cname, qualified_name, type)
super(CythonFunction, self).__init__(name,
cname,
qualified_name,
type)
self.module = module
self.pf_cname = pf_cname
self.lineno = lineno
self.locals = {}
self.arguments = []
self.step_into_functions = set()
class SourceFileDescriptor(object):
def __init__(self, filename, lineno, lexer, formatter=None):
self.filename = filename
self.lineno = lineno
self.lexer = lexer
self.formatter = formatter
def valid(self):
return self.filename is not None
def lex(self, code):
if pygments and parameter.colorize_code:
bg = parameter.terminal_background.value
if self.formatter is None:
formatter = pygments.formatters.TerminalFormatter(bg=bg)
else:
formatter = self.formatter
return pygments.highlight(code, self.lexer, formatter)
return code
def get_source(self, start=0, stop=None, lex_source=True):
# todo: have it detect the source file's encoding
if not self.filename:
return 'Unable to retrieve source code'
start = max(self.lineno + start, 0)
if stop is None:
stop = self.lineno + 1
else:
stop = self.lineno + stop
with open(self.filename) as f:
source = itertools.islice(f, start, stop)
if lex_source:
return [self.lex(line) for line in source]
else:
return list(source)
# Errors
class CyGDBError(gdb.GdbError):
"""
Base class for Cython-command related erorrs
"""
def __init__(self, *args):
args = args or (self.msg,)
super(CyGDBError, self).__init__(*args)
class NoCythonFunctionInFrameError(CyGDBError):
"""
raised when the user requests the current cython function, which is
unavailable
"""
msg = "Current function is a function cygdb doesn't know about"
class NoFunctionNameInFrameError(NoCythonFunctionInFrameError):
"""
raised when the name of the C function could not be determined
in the current C stack frame
"""
msg = ('C function name could not be determined in the current C stack '
'frame')
# Parameters
class CythonParameter(gdb.Parameter):
"""
Base class for cython parameters
"""
def __init__(self, name, command_class, parameter_class, default=None):
self.show_doc = self.set_doc = self.__class__.__doc__
super(CythonParameter, self).__init__(name, command_class,
parameter_class)
if default is not None:
self.value = default
def __nonzero__(self):
return bool(self.value)
__bool__ = __nonzero__ # python 3
class CompleteUnqualifiedFunctionNames(CythonParameter):
"""
Have 'cy break' complete unqualified function or method names.
"""
class ColorizeSourceCode(CythonParameter):
"""
Tell cygdb whether to colorize source code
"""
class TerminalBackground(CythonParameter):
"""
Tell cygdb about the user's terminal background (light or dark)
"""
class Parameter(object):
"""
Simple container class that might get more functionality in the distant
future (mostly to remind us that we're dealing with parameters)
"""
complete_unqualified = CompleteUnqualifiedFunctionNames(
'cy_complete_unqualified',
gdb.COMMAND_BREAKPOINTS,
gdb.PARAM_BOOLEAN,
True)
colorize_code = ColorizeSourceCode(
'cy_colorize_code',
gdb.COMMAND_FILES,
gdb.PARAM_BOOLEAN,
True)
terminal_background = TerminalBackground(
'cy_terminal_background_color',
gdb.COMMAND_FILES,
gdb.PARAM_STRING,
"dark")
parameter = Parameter()
# Commands
class CythonCommand(gdb.Command):
"""
......@@ -106,22 +266,62 @@ class CythonCommand(gdb.Command):
cy import
cy break
cy condition
cy step
cy enable
cy disable
cy print
cy list
cy locals
cy globals
cy tb
cy cname
cy backtrace
cy info line
"""
CythonCommand('cy', gdb.COMMAND_NONE, gdb.COMPLETE_COMMAND, prefix=True)
def is_cython_function(self, frame=None):
func_name = (frame or gdb.selected_frame()).name()
return func_name is not None and func_name in functions_by_cname
@default_selected_gdb_frame
def is_python_function(self, frame):
return libpython.Frame(frame).is_evalframeex()
@default_selected_gdb_frame
def get_c_function_name(self, frame):
return frame.name()
@default_selected_gdb_frame
def get_c_lineno(self, frame):
return frame.find_sal().line
@default_selected_gdb_frame
def get_cython_function(self, frame):
result = functions_by_cname.get(frame.name())
if result is None:
raise NoCythonFunctionInFrameError()
return result
@default_selected_gdb_frame
def get_cython_lineno(self, frame):
cyfunc = self.get_cython_function(frame)
return cyfunc.module.lineno_c2cy.get(self.get_c_lineno(frame))
@default_selected_gdb_frame
def get_source_desc(self, frame):
if self.is_cython_function():
filename = self.get_cython_function(frame).module.filename
lineno = self.get_cython_lineno(frame)
lexer = pygments.lexers.CythonLexer()
else:
filename = None
lineno = -1
lexer = None
return SourceFileDescriptor(filename, lineno, lexer)
class CyImport(gdb.Command):
cy = CythonCommand('cy', gdb.COMMAND_NONE, gdb.COMPLETE_COMMAND, prefix=True)
class CyImport(CythonCommand):
"""
Import debug information outputted by the Cython compiler
Example: cy import FILE...
......@@ -163,25 +363,51 @@ class CyImport(gdb.Command):
d = local.attrib
cython_function.locals[d['name']] = CythonVariable(**d)
for step_into_func in function.find('StepIntoFunctions'):
d = step_into_func.attrib
cython_function.step_into_functions.add(d['name'])
cython_function.arguments.extend(
funcarg.tag for funcarg in function.find('Arguments'))
CyImport('cy import', gdb.COMMAND_STATUS, gdb.COMPLETE_FILENAME)
for marker in module.find('LineNumberMapping'):
cython_lineno = int(marker.attrib['cython_lineno'])
c_linenos = map(int, marker.attrib['c_linenos'].split())
cython_module.lineno_cy2c[cython_lineno] = min(c_linenos)
for c_lineno in c_linenos:
cython_module.lineno_c2cy[c_lineno] = cython_lineno
cy.import_ = CyImport('cy import', gdb.COMMAND_STATUS, gdb.COMPLETE_FILENAME)
class CyBreak(gdb.Command):
class CyBreak(CythonCommand):
"""
Set a breakpoint for Cython code using Cython qualified name notation, e.g.:
cy-break cython_modulename.ClassName.method_name...
cy break cython_modulename.ClassName.method_name...
or normal notation:
cy-break function_or_method_name...
cy break function_or_method_name...
or for a line number:
cy break cython_module:lineno...
"""
def invoke(self, function_names, from_tty):
for funcname in string_to_argv(function_names.encode('UTF-8')):
def _break_pyx(self, name):
modulename, _, lineno = name.partition(':')
lineno = int(lineno)
cython_module = cython_namespace[modulename]
if lineno in cython_module.lineno_cy2c:
c_lineno = cython_module.lineno_cy2c[lineno]
breakpoint = '%s:%s' % (cython_module.name, c_lineno)
gdb.execute('break ' + breakpoint)
else:
sys.stderr.write("Not a valid line number (does it contain actual code?)\n")
def _break_funcname(self, funcname):
func = functions_by_qualified_name.get(funcname)
break_funcs = [func]
......@@ -224,9 +450,19 @@ class CyBreak(gdb.Command):
if func.pf_cname:
gdb.execute('break %s' % func.pf_cname)
def invoke(self, function_names, from_tty):
for funcname in string_to_argv(function_names.encode('UTF-8')):
if ':' in funcname:
self._break_pyx(funcname)
else:
self._break_funcname(funcname)
@dont_suppress_errors
def complete(self, text, word):
names = itertools.chain(functions_by_qualified_name, functions_by_name)
names = functions_by_qualified_name
if parameter.complete_unqualified:
names = itertools.chain(names, functions_by_name)
words = text.strip().split()
if words and '.' in words[-1]:
compl = [n for n in functions_by_qualified_name
......@@ -243,82 +479,91 @@ class CyBreak(gdb.Command):
return compl
CyBreak('cy break', gdb.COMMAND_BREAKPOINTS)
cy.break_ = CyBreak('cy break', gdb.COMMAND_BREAKPOINTS)
# This needs GDB 7.2 or the Archer branch
# class CompleteUnqualifiedFunctionNames(gdb.Parameter):
# """
# Indicates whether 'cy break' should complete unqualified function or
# method names. e.g. whether only 'modulename.functioname' should be
# completed, or also just 'functionname'
# """
#
# cy_complete_unqualified = CompleteUnqualifiedFunctionNames(
# 'cy_complete_unqualified',
# gdb.COMMAND_BREAKPOINTS,
# gdb.PARAM_BOOLEAN)
class CyStep(CythonCommand):
class NoCythonFunctionNameInFrameError(Exception):
"""
raised when the name of the C function could not be determined
in the current C stack frame
"""
def step(self, from_tty=True, nsteps=1):
for nthstep in xrange(nsteps):
cython_func = self.get_cython_function()
beginline = self.get_cython_lineno()
curframe = gdb.selected_frame()
class CyPrint(gdb.Command):
"""
Print a Cython variable using 'cy-print x' or 'cy-print module.function.x'
"""
def _get_current_cython_function(self):
func_name = gdb.selected_frame().name()
if func_name is None:
raise NoCythonFunctionNameInFrameError()
while True:
result = gdb.execute('step', False, True)
if result.startswith('Breakpoint'):
break
newframe = gdb.selected_frame()
if newframe == curframe:
# still in the same function
if self.get_cython_lineno() > beginline:
break
else:
# we entered a function
funcname = self.get_c_function_name(newframe)
if (self.is_cython_function() or
self.is_python_function() or
funcname in cython_function.step_into_functions):
break
return functions_by_cname.get(func_name)
line, = self.get_source_desc().get_source()
sys.stdout.write(line)
def _get_locals_globals(self):
try:
cython_function = self._get_current_cython_function()
except NoCythonFunctionNameInFrameError:
return (None, None)
def invoke(self, steps, from_tty):
if self.is_cython_function():
if steps:
self.step(from_tty, int(steps))
else:
if cython_function is None:
return (None, None)
self.step(from_tty)
else:
gdb.execute('step ' + steps)
cy.step = CyStep('cy step', gdb.COMMAND_RUNNING, gdb.COMPLETE_NONE)
class CyList(CythonCommand):
def invoke(self, _, from_tty):
sd = self.get_source_desc()
it = enumerate(sd.get_source(-5, +5))
sys.stdout.write(
''.join('%4d %s' % (sd.lineno + i, line) for i, line in it))
cy.list = CyList('cy list', gdb.COMMAND_FILES, gdb.COMPLETE_NONE)
class CyPrint(CythonCommand):
"""
Print a Cython variable using 'cy-print x' or 'cy-print module.function.x'
"""
return cython_function.locals, cython_function.module.globals
def invoke(self, name, from_tty):
try:
cython_function = self._get_current_cython_function()
except NoCythonFunctionNameInFrameError:
print('Unable to determine the name of the function in the '
'current frame.')
except RuntimeError, e:
print e.args[0]
else:
# a cython_function of None means we don't know about such a Cython
# function and we fall back to GDB's print
cname = name
if cython_function is not None:
cname = None
if self.is_cython_function():
cython_function = self.get_cython_function()
if name in cython_function.locals:
cname = cython_function.locals[name].cname
elif name in cython_function.module.globals:
cname = cython_function.module.globals[name].cname
# let the pretty printers do the work
cname = cname or name
gdb.execute('print ' + cname)
def complete(self):
locals_, globals_ = self._get_locals_globals()
if locals_ is None:
if self.is_cython_function():
cf = self.get_cython_function()
return list(itertools.chain(cf.locals, cf.globals))
return []
return list(itertools.chain(locals_, globals_))
CyPrint('cy print', gdb.COMMAND_DATA)
cy.print_ = CyPrint('cy print', gdb.COMMAND_DATA)
class CyLocals(CyPrint):
class CyLocals(CythonCommand):
def ns(self):
locals_, _ = self._get_locals_globals()
return locals_
return self.get_cython_function().locals
def invoke(self, name, from_tty):
try:
......@@ -337,19 +582,33 @@ class CyLocals(CyPrint):
if var.type == PythonObject:
result = libpython.PyObjectPtr.from_pyobject_ptr(val)
else:
result = CObject
result = val
print '%s = %s' % (var.name, result)
class CyGlobals(CyLocals):
class CyGlobals(CythonCommand):
def ns(self):
_, globals_ = self._get_locals_globals()
return globals_
return self.get_cython_function().globals
def invoke(self, name, from_tty):
m = gdb.parse_and_eval('PyModule_GetDict(__pyx_m)')
m = m.cast(gdb.lookup_type('PyModuleObject').pointer())
print PyObjectPtrPrinter(libpython.PyObjectPtr.from_pyobject_ptr(m['md_dict'])).to_string()
# include globals from the debug info XML file!
m = gdb.parse_and_eval('__pyx_m')
CyLocals('cy locals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE)
CyGlobals('cy globals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE)
try:
PyModuleObject = gdb.lookup_type('PyModuleObject')
except RuntimeError:
raise gdb.GdbError(textwrap.dedent("""
Unable to lookup type PyModuleObject, did you compile python
with debugging support (-g)? If this installation is from your
package manager, install python-dbg and run the debug version
of python or compile it yourself.
"""))
m = m.cast(PyModuleObject.pointer())
d = libpython.PyObjectPtr.from_pyobject_ptr(m['md_dict'])
print d.get_truncated_repr(1000)
cy.locals = CyLocals('cy locals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE)
cy.globals = CyGlobals('cy globals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE)
......@@ -11,10 +11,12 @@ class StringIOTree(object):
stream = StringIO()
self.stream = stream
self.write = stream.write
self.markers = []
def getvalue(self):
content = [x.getvalue() for x in self.prepended_children]
content.append(self.stream.getvalue())
print self.linenumber_map()
return "".join(content)
def copyto(self, target):
......@@ -59,6 +61,11 @@ class StringIOTree(object):
self.prepended_children.append(other)
return other
def allmarkers(self):
children = self.prepended_children
return [m for c in children for m in c.allmarkers()] + self.markers
__doc__ = r"""
Implements a buffer with insertion points. When you know you need to
"get back" to a place and write more later, simply call insertion_point()
......
......@@ -238,7 +238,7 @@ setup(
'Cython.Runtime',
'Cython.Distutils',
'Cython.Plex',
'Cython.Debugger',
'Cython.Tests',
'Cython.Compiler.Tests',
],
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment