enable tracing for nogil functions/sections

CHANGES.rst

@@ -11,6 +11,8 @@ Features added
 * Support for coverage.py 4.0+ can be enabled by adding the plugin
   "Cython.Coverage" to the ".coveragerc" config file.
 
+* Tracing is supported in ``nogil`` functions/sections.
+
 Bugs fixed
 ----------
 

Cython/Compiler/Code.pxd

@@ -31,9 +31,10 @@ cdef class FunctionState:
 
     cdef public object return_from_error_cleanup_label # not used in __init__ ?
 
-    cdef public bint in_try_finally
     cdef public object exc_vars
+    cdef public bint in_try_finally
     cdef public bint can_trace
+    cdef public bint gil_owned
 
     cdef public list temps_allocated
     cdef public dict temps_free

Cython/Compiler/Code.py

@@ -518,6 +518,7 @@ class FunctionState(object):
         self.in_try_finally = 0
         self.exc_vars = None
         self.can_trace = False
+        self.gil_owned = True
 
         self.temps_allocated = [] # of (name, type, manage_ref, static)
         self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
@@ -1572,7 +1573,8 @@ class CCodeWriter(object):
         if (self.funcstate and self.funcstate.can_trace
                 and self.globalstate.directives['linetrace']):
             self.indent()
-            self.write('__Pyx_TraceLine(%d)\n' % self.marker[0])
+            self.write('__Pyx_TraceLine(%d,%d)\n' % (
+                self.marker[0], not self.funcstate.gil_owned))
         self.last_marker_line = self.marker[0]
         self.marker = None
 
@@ -2093,17 +2095,18 @@ class CCodeWriter(object):
         self.globalstate.use_utility_code(
             UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))
 
-    def put_trace_declarations(self, codeobj=None):
-        self.putln('__Pyx_TraceDeclarations(%s)' % (codeobj or 'NULL'))
+    def put_trace_declarations(self, codeobj=None, nogil=False):
+        self.putln('__Pyx_TraceDeclarations(%s, %d)' % (codeobj or 'NULL', nogil))
 
-    def put_trace_call(self, name, pos):
-        self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1]))
+    def put_trace_call(self, name, pos, nogil=False):
+        self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d);' % (
+            name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil))
 
     def put_trace_exception(self):
         self.putln("__Pyx_TraceException();")
 
-    def put_trace_return(self, retvalue_cname):
-        self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname)
+    def put_trace_return(self, retvalue_cname, nogil=False):
+        self.putln("__Pyx_TraceReturn(%s, %d);" % (retvalue_cname, nogil))
 
     def putln_openmp(self, string):
         self.putln("#ifdef _OPENMP")

Cython/Compiler/Nodes.py

@@ -1698,9 +1698,6 @@ class FuncDefNode(StatNode, BlockNode):
 
         profile = code.globalstate.directives['profile']
         linetrace = code.globalstate.directives['linetrace']
-        if (linetrace or profile) and lenv.nogil:
-            warning(self.pos, "Cannot profile nogil function.", 1)
-            profile = linetrace = False
         if profile or linetrace:
             code.globalstate.use_utility_code(
                 UtilityCode.load_cached("Profile", "Profile.c"))
@@ -1708,6 +1705,7 @@ class FuncDefNode(StatNode, BlockNode):
         # Generate C code for header and body of function
         code.enter_cfunc_scope()
         code.return_from_error_cleanup_label = code.new_label()
+        code.funcstate.gil_owned = not lenv.nogil
 
         # ----- Top-level constants used by this function
         code.mark_pos(self.pos)
@@ -1764,7 +1762,7 @@ class FuncDefNode(StatNode, BlockNode):
 
         if profile or linetrace:
             code_object = self.code_object.calculate_result_code(code) if self.code_object else None
-            code.put_trace_declarations(code_object)
+            code.put_trace_declarations(code_object, nogil=not code.funcstate.gil_owned)
 
         # ----- Extern library function declarations
         lenv.generate_library_function_declarations(code)
@@ -1775,10 +1773,9 @@ class FuncDefNode(StatNode, BlockNode):
         # See if we need to acquire the GIL for variable declarations, or for
         # refnanny only
 
-        # Profiling or closures are not currently possible for cdef nogil
-        # functions, but check them anyway
-        have_object_args = (self.needs_closure or self.needs_outer_scope or
-                            profile or linetrace)
+        # Closures are not currently possible for cdef nogil functions,
+        # but check them anyway
+        have_object_args = self.needs_closure or self.needs_outer_scope
         for arg in lenv.arg_entries:
             if arg.type.is_pyobject:
                 have_object_args = True
@@ -1796,6 +1793,7 @@ class FuncDefNode(StatNode, BlockNode):
 
         if acquire_gil or acquire_gil_for_var_decls_only:
             code.put_ensure_gil()
+            code.funcstate.gil_owned = True
         elif lenv.nogil and lenv.has_with_gil_block:
             code.declare_gilstate()
 
@@ -1855,7 +1853,7 @@ class FuncDefNode(StatNode, BlockNode):
         if profile or linetrace:
             # this looks a bit late, but if we don't get here due to a
             # fatal error before hand, it's not really worth tracing
-            code.put_trace_call(self.entry.name, self.pos)
+            code.put_trace_call(self.entry.name, self.pos, nogil=not code.funcstate.gil_owned)
             code.funcstate.can_trace = True
         # ----- Fetch arguments
         self.generate_argument_parsing_code(env, code)
@@ -1874,8 +1872,7 @@ class FuncDefNode(StatNode, BlockNode):
             #       incref our arguments
             elif (is_cdef and entry.type.is_memoryviewslice and
                   len(entry.cf_assignments) > 1):
-                code.put_incref_memoryviewslice(entry.cname,
-                                                have_gil=not lenv.nogil)
+                code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
         for entry in lenv.var_entries:
             if entry.is_arg and len(entry.cf_assignments) > 1:
                 code.put_var_incref(entry)
@@ -1894,6 +1891,7 @@ class FuncDefNode(StatNode, BlockNode):
 
         if acquire_gil_for_var_decls_only:
             code.put_release_ensured_gil()
+            code.funcstate.gil_owned = False
 
         # -------------------------
         # ----- Function body -----
@@ -2054,9 +2052,9 @@ class FuncDefNode(StatNode, BlockNode):
         if profile or linetrace:
             code.funcstate.can_trace = False
             if self.return_type.is_pyobject:
-                code.put_trace_return(Naming.retval_cname)
+                code.put_trace_return(Naming.retval_cname, nogil=not code.funcstate.gil_owned)
             else:
-                code.put_trace_return("Py_None")
+                code.put_trace_return("Py_None", nogil=not code.funcstate.gil_owned)
 
         if not lenv.nogil:
             # GIL holding function
@@ -2065,6 +2063,7 @@ class FuncDefNode(StatNode, BlockNode):
         if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
             # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
             code.put_release_ensured_gil()
+            code.funcstate.gil_owned = False
 
         if not self.return_type.is_void:
             code.putln("return %s;" % Naming.retval_cname)
@@ -7073,21 +7072,20 @@ class GILStatNode(NogilTryFinallyStatNode):
         else:
             variable = None
 
-        old_trace_config = code.funcstate.can_trace
+        old_gil_config = code.funcstate.gil_owned
         if self.state == 'gil':
             code.put_ensure_gil(variable=variable)
-            # FIXME: not that easy, tracing may not be possible at all here
-            #code.funcstate.can_trace = True
+            code.funcstate.gil_owned = True
         else:
             code.put_release_gil(variable=variable)
-            code.funcstate.can_trace = False
+            code.funcstate.gil_owned = False
 
         TryFinallyStatNode.generate_execution_code(self, code)
 
         if self.state_temp:
             self.state_temp.release(code)
 
-        code.funcstate.can_trace = old_trace_config
+        code.funcstate.gil_owned = old_gil_config
         code.end_block()
 
 

tests/run/coverage_nogil.srctree

@@ -23,7 +23,7 @@ plugins = Cython.Coverage
 
 ######## coverage_test_nogil.pyx ########
 # cython: linetrace=True
-# distutils: define_macros=CYTHON_TRACE=1
+# distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1
 
 cdef int func1(int a, int b) nogil:
     cdef int x                   #  5
@@ -85,8 +85,8 @@ def run_coverage(module):
     executed = set(exec_lines) - set(missing_lines)
     # check that everything that runs with the gil owned was executed
     assert all(line in executed for line in [13, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines)
-    # currently, we do not trace nogil code lines, but that should eventually be implemented
-    # we also don't trace 'with gil' blocks in 'nogil' functions
+    # check that everything that runs in nogil sections was executed
+    assert all(line in executed for line in [6, 7, 8, 9]), '%s / %s' % (exec_lines, missing_lines)
 
 
 if __name__ == '__main__':