Debugger: Fix closure support + tests

Debugger: Recognition of module-level Cython code (initmodulename and PyInit_modulename)
Fix debug flag (import Parsing when needed)

Cython/Compiler/Main.py

@@ -18,7 +18,11 @@ from time import time
 
 import Code
 import Errors
-import Parsing
+# Do not import Parsing here, import it when needed, because Parsing imports
+# Nodes, which globally needs debug command line options initialized to set a
+# conditional metaclass. These options are processed by CmdLine called from 
+# main() in this file.
+# import Parsing
 import Version
 from Scanning import PyrexScanner, FileSourceDescriptor
 from Errors import PyrexError, CompileError, InternalError, AbortError, error, warning
@@ -493,6 +497,7 @@ class Context(object):
         try:
             f = Utils.open_source_file(source_filename, "rU")
             try:
+                import Parsing
                 s = PyrexScanner(f, source_desc, source_encoding = f.encoding,
                                  scope = scope, context = self)
                 tree = Parsing.p_module(s, pxd, full_module_name)

Cython/Compiler/ParseTreeTransforms.py

@@ -1591,6 +1591,10 @@ class DebugTransform(CythonTransform):
         #self.c_output_file = options.output_file 
         self.c_output_file = result.c_file
         
+        # Closure support, basically treat nested functions as if the AST were
+        # never nested
+        self.nested_funcdefs = []
+        
         # tells visit_NameNode whether it should register step-into functions
         self.register_stepinto = False
         
@@ -1605,7 +1609,17 @@ class DebugTransform(CythonTransform):
         
         # serialize functions
         self.tb.start('Functions')
+        # First, serialize functions normally...
         self.visitchildren(node)
+        
+        # ... then, serialize nested functions
+        for nested_funcdef in self.nested_funcdefs:
+            self.visit_FuncDefNode(nested_funcdef)
+        
+        self.register_stepinto = True
+        self.serialize_modulenode_as_function(node)
+        self.register_stepinto = False
+        
         self.tb.end('Functions')
         
         # 2.3 compatibility. Serialize global variables
@@ -1625,8 +1639,16 @@ class DebugTransform(CythonTransform):
         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
         return node
     
-    def visit_FuncDefNode(self, node):
+    def visit_FuncDefNode(self, node):        
         self.visited.add(node.local_scope.qualified_name)
+
+        if getattr(node, 'is_wrapper', False):
+            return node
+
+        if self.register_stepinto:
+            self.nested_funcdefs.append(node)
+            return node
+
         # node.entry.visibility = 'extern'
         if node.py_func is None:
             pf_cname = ''
@@ -1678,6 +1700,51 @@ class DebugTransform(CythonTransform):
         self.visitchildren(node)
         return node
     
+    def serialize_modulenode_as_function(self, node):
+        """
+        Serialize the module-level code as a function so the debugger will know
+        it's a "relevant frame" and it will know where to set the breakpoint
+        for 'break modulename'.
+        """
+        name = node.full_module_name.rpartition('.')[-1]
+        
+        cname_py2 = 'init' + name
+        cname_py3 = 'PyInit_' + name
+        
+        py2_attrs = dict(
+            name=name,
+            cname=cname_py2,
+            pf_cname='',
+            # Ignore the qualified_name, breakpoints should be set using 
+            # `cy break modulename:lineno` for module-level breakpoints.
+            qualified_name='',
+            lineno='1',
+            is_initmodule_function="True",
+        )
+        
+        py3_attrs = dict(py2_attrs, cname=cname_py3)
+        
+        self._serialize_modulenode_as_function(node, py2_attrs)
+        self._serialize_modulenode_as_function(node, py3_attrs)
+    
+    def _serialize_modulenode_as_function(self, node, attrs):
+        self.tb.start('Function', attrs=attrs)
+        
+        self.tb.start('Locals')
+        self.serialize_local_variables(node.scope.entries)
+        self.tb.end('Locals')
+
+        self.tb.start('Arguments')
+        self.tb.end('Arguments')
+
+        self.tb.start('StepIntoFunctions')
+        self.register_stepinto = True
+        self.visitchildren(node)
+        self.register_stepinto = False
+        self.tb.end('StepIntoFunctions')
+        
+        self.tb.end('Function')
+    
     def serialize_local_variables(self, entries):
         for entry in entries.values():
             if entry.type.is_pyobject:
@@ -1685,10 +1752,19 @@ class DebugTransform(CythonTransform):
             else:
                 vartype = 'CObject'
             
-            cname = entry.cname
-            # if entry.type.is_extension_type:
-                # cname = entry.type.typeptr_cname
-            
+            if entry.from_closure:
+                # We're dealing with a closure where a variable from an outer
+                # scope is accessed, get it from the scope object.
+                cname = '%s->%s' % (Naming.cur_scope_cname, 
+                                    entry.outer_entry.cname)
+                
+                qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
+                                      entry.scope.name, 
+                                      entry.name)
+            else:
+                cname = entry.cname
+                qname = entry.qualified_name
+                
             if not entry.pos:
                 # this happens for variables that are not in the user's code,
                 # e.g. for the global __builtins__, __doc__, etc. We can just
@@ -1696,11 +1772,11 @@ class DebugTransform(CythonTransform):
                 lineno = '0'
             else:
                 lineno = str(entry.pos[1])
-                
+            
             attrs = dict(
                 name=entry.name,
                 cname=cname,
-                qualified_name=entry.qualified_name,
+                qualified_name=qname,
                 type=vartype,
                 lineno=lineno)
             

Cython/Compiler/Tests/TestParseTreeTransforms.py

@@ -182,7 +182,8 @@ class TestDebugTransform(DebuggerTestCase):
             self.assertEqual('PythonObject', xml_globals.get('python_var'))
             
             # test functions
-            funcnames = 'codefile.spam', 'codefile.ham', 'codefile.eggs'
+            funcnames = ('codefile.spam', 'codefile.ham', 'codefile.eggs', 
+                         'codefile.closure', 'codefile.inner')
             required_xml_attrs = 'name', 'cname', 'qualified_name'
             assert all([f in xml_funcs for f in funcnames])
             spam, ham, eggs = [xml_funcs[funcname] for funcname in funcnames]

Cython/Debugger/Tests/codefile

@@ -21,16 +21,34 @@ def spam(a=0):
     puts("spam")
     os.path.join("foo", "bar")
     some_c_function()
+
+cpdef eggs():
+    pass    
     
 cdef ham():
     pass
     
-cpdef eggs():
-    pass
-
 cdef class SomeClass(object):
     def spam(self):
         pass
 
+def closure():
+    a = 1
+    def inner():
+        b = 2
+        # access closed over variables
+        print a, b
+    return inner
+
+def closure_without_closing_variables():
+    a = 1
+    def inner2():
+        b = 2
+        print b
+    return inner2
+
+closure()()
+closure_without_closing_variables()()
+
 spam()
 print "bye!"

Cython/Debugger/Tests/test_libcython_in_gdb.py

@@ -98,7 +98,7 @@ class TestDebugInformationClasses(DebugTestCase):
                          'codefile.SomeClass.spam')
         self.assertEqual(self.spam_func.module, self.module)
         
-        assert self.eggs_func.pf_cname
+        assert self.eggs_func.pf_cname, (self.eggs_func, self.eggs_func.pf_cname)
         assert not self.ham_func.pf_cname
         assert not self.spam_func.pf_cname
         assert not self.spam_meth.pf_cname
@@ -135,11 +135,21 @@ class TestBreak(DebugTestCase):
         self.assertEqual(bp.type, gdb.BP_BREAKPOINT)
         assert self.spam_func.cname in bp.location
         assert bp.enabled
-    
+
     def test_python_break(self):
         gdb.execute('cy break -p join')
         assert 'def join(' in gdb.execute('cy run', to_string=True)
 
+    def test_break_lineno(self):
+        beginline = 'import os'
+        nextline = 'cdef int c_var = 12'
+
+        self.break_and_run(beginline)
+        self.lineno_equals(beginline)
+        step_result = gdb.execute('cy step', to_string=True)
+        self.lineno_equals(nextline)
+        assert step_result.rstrip().endswith(nextline)
+
 
 class TestKilled(DebugTestCase):
     
@@ -341,6 +351,19 @@ class TestExec(DebugTestCase):
         gdb.execute('cy exec some_random_var = 14')
         self.assertEqual('14', self.eval_command('some_random_var'))
 
+class TestClosure(DebugTestCase):
+    
+    def test_cython_closure(self):
+        self.break_and_run('def inner():')
+        
+        self.assertEqual(str(self.read_var('a')), '1')
+        print_result = gdb.execute('cy print a', to_string=True).strip()
+        self.assertEqual(print_result, 'a = 1')
+    
+    def test_cython_closure_no_closing_variables(self):
+        self.break_and_run('def inner2():')
+        self.assertEqual(gdb.execute('cy locals', to_string=True), '')
+
 
 _do_debug = os.environ.get('GDB_DEBUG')
 if _do_debug:
@@ -381,7 +404,7 @@ def runtests():
     returned to the parent test process.
     """
     from Cython.Debugger.Tests import test_libpython_in_gdb
-    
+
     success_libcython = run_unittest_in_module(__name__)
     success_libpython = run_unittest_in_module(test_libpython_in_gdb.__name__)
     

Cython/Debugger/libcython.py

@@ -154,9 +154,6 @@ class CythonModule(object):
         self.lineno_c2cy = {}
         self.functions = {}
         
-    def qualified_name(self, varname):
-        return '.'.join(self.name, varname)
-
 class CythonVariable(object):
 
     def __init__(self, name, cname, qualified_name, type, lineno):
@@ -174,7 +171,8 @@ class CythonFunction(CythonVariable):
                  pf_cname,
                  qualified_name, 
                  lineno, 
-                 type=CObject):
+                 type=CObject,
+                 is_initmodule_function="False"):
         super(CythonFunction, self).__init__(name, 
                                              cname, 
                                              qualified_name, 
@@ -182,6 +180,7 @@ class CythonFunction(CythonVariable):
                                              lineno)
         self.module = module
         self.pf_cname = pf_cname
+        self.is_initmodule_function = is_initmodule_function == "True"
         self.locals = {}
         self.arguments = []
         self.step_into_functions = set()
@@ -243,7 +242,8 @@ class CythonBase(object):
             pyframeobject = libpython.Frame(frame).get_pyop()
 
             if not pyframeobject:
-                raise gdb.GdbError('Unable to read information on python frame')
+                raise gdb.GdbError(
+                            'Unable to read information on python frame')
 
             filename = pyframeobject.filename()
             lineno = pyframeobject.current_line_num()
@@ -752,14 +752,20 @@ class CyBreak(CythonCommand):
     
     def _break_funcname(self, funcname):
         func = self.cy.functions_by_qualified_name.get(funcname)
+        
+        if func and func.is_initmodule_function:
+            func = None
+        
         break_funcs = [func]
         
         if not func:
-            funcs = self.cy.functions_by_name.get(funcname)
+            funcs = self.cy.functions_by_name.get(funcname) or []
+            funcs = [f for f in funcs if not f.is_initmodule_function]
+            
             if not funcs:
                 gdb.execute('break ' + funcname)
                 return
-                
+            
             if len(funcs) > 1:
                 # multiple functions, let the user pick one
                 print 'There are multiple such functions:'
@@ -811,18 +817,28 @@ class CyBreak(CythonCommand):
     
     @dont_suppress_errors
     def complete(self, text, word):
-        names = self.cy.functions_by_qualified_name
+        # Filter init-module functions (breakpoints can be set using 
+        # modulename:linenumber).
+        names =  [n for n, L in self.cy.functions_by_name.iteritems() 
+                        if any(not f.is_initmodule_function for f in L)]
+        qnames = [n for n, f in self.cy.functions_by_qualified_name.iteritems()
+                        if not f.is_initmodule_function]
+        
         if parameters.complete_unqualified:
-            names = itertools.chain(names, self.cy.functions_by_name)
+            all_names = itertools.chain(qnames, names)
+        else:
+            all_names = qnames
 
         words = text.strip().split()
-        if words and '.' in words[-1]:
-            lastword = words[-1]
-            compl = [n for n in self.cy.functions_by_qualified_name 
-                           if n.startswith(lastword)]
-        else:
+        if not words or '.' not in words[-1]:
+            # complete unqualified
             seen = set(text[:-len(word)].split())
-            return [n for n in names if n.startswith(word) and n not in seen]
+            return [n for n in all_names 
+                          if n.startswith(word) and n not in seen]
+        
+        # complete qualified name
+        lastword = words[-1]
+        compl = [n for n in qnames if n.startswith(lastword)]
         
         if len(lastword) > len(word):
             # readline sees something (e.g. a '.') as a word boundary, so don't
@@ -862,6 +878,7 @@ class CythonInfo(CythonBase, libpython.PythonInfo):
     def runtime_break_functions(self):
         if self.is_cython_function():
             return self.get_cython_function().step_into_functions
+        return ()
     
     def static_break_functions(self):
         result = ['PyEval_EvalFrameEx']
@@ -1091,7 +1108,13 @@ class CyLocals(CythonCommand):
     
     @dispatch_on_frame(c_command='info locals', python_command='py-locals')
     def invoke(self, args, from_tty):
-        local_cython_vars = self.get_cython_function().locals
+        cython_function = self.get_cython_function()
+        
+        if cython_function.is_initmodule_function:
+            self.cy.globals.invoke(args, from_tty)
+            return
+
+        local_cython_vars = cython_function.locals
         max_name_length = len(max(local_cython_vars, key=len))
         for name, cyvar in sorted(local_cython_vars.iteritems(), key=sortkey):
             if self.is_initialized(self.get_cython_function(), cyvar.name):

Cython/Debugger/libpython.py

@@ -1906,8 +1906,6 @@ class ExecutionControlCommandBase(gdb.Command):
         for location in all_locations:
             result = gdb.execute('break %s' % location, to_string=True)
             yield re.search(r'Breakpoint (\d+)', result).group(1)
-       
-        
     
     def delete_breakpoints(self, breakpoint_list):
         for bp in breakpoint_list: