propagate exceptions in call/line trace function calls

Cython/Compiler/Code.py

@@ -1415,9 +1415,8 @@ class CCodeWriter(object):
     def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None, c_line_in_traceback=True):
         if buffer is None: buffer = StringIOTree()
         self.buffer = buffer
-        self.marker = None
-        self.last_marker_line = 0
-        self.source_desc = ""
+        self.last_pos = None
+        self.last_marked_pos = None
         self.pyclass_stack = []
 
         self.funcstate = None
@@ -1434,6 +1433,8 @@ class CCodeWriter(object):
                 self.level = create_from.level
                 self.bol = create_from.bol
                 self.call_level = create_from.call_level
+            self.last_pos = create_from.last_pos
+            self.last_marked_pos = create_from.last_marked_pos
 
         if emit_linenums is None and self.globalstate:
             self.emit_linenums = self.globalstate.emit_linenums
@@ -1457,11 +1458,7 @@ class CCodeWriter(object):
     def write(self, s):
         # also put invalid markers (lineno 0), to indicate that those lines
         # have no Cython source code correspondence
-        if self.marker is None:
-            cython_lineno = self.last_marker_line
-        else:
-            cython_lineno = self.marker[0]
-
+        cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0
         self.buffer.markers.extend([cython_lineno] * s.count('\n'))
         self.buffer.write(s)
 
@@ -1553,11 +1550,11 @@ class CCodeWriter(object):
     # code generation
 
     def putln(self, code="", safe=False):
-        if self.marker and self.bol:
+        if self.last_pos and self.bol:
             self.emit_marker()
-        if self.emit_linenums and self.last_marker_line != 0:
-            self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))
-
+        if self.emit_linenums and self.last_marked_pos:
+            source_desc, line, _ = self.last_marked_pos
+            self.write('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description()))
         if code:
             if safe:
                 self.put_safe(code)
@@ -1566,17 +1563,32 @@ class CCodeWriter(object):
         self.write("\n")
         self.bol = 1
 
+    def mark_pos(self, pos):
+        if pos is None:
+            return
+        if self.last_marked_pos and self.last_marked_pos[:2] == pos[:2]:
+            return
+        self.last_pos = pos
+
     def emit_marker(self):
+        pos = self.last_marked_pos = self.last_pos
+        self.last_pos = None
         self.write("\n")
         self.indent()
-        self.write("/* %s */\n" % self.marker[1])
-        if (self.funcstate and self.funcstate.can_trace
-                and self.globalstate.directives['linetrace']):
+        self.write("/* %s */\n" % self._build_marker(pos))
+        if self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']:
             self.indent()
-            self.write('__Pyx_TraceLine(%d,%d)\n' % (
-                self.marker[0], not self.funcstate.gil_owned))
-        self.last_marker_line = self.marker[0]
-        self.marker = None
+            self.write('__Pyx_TraceLine(%d,%d,%s)\n' % (
+                pos[1], not self.funcstate.gil_owned, self.error_goto(pos)))
+
+    def _build_marker(self, pos):
+        source_desc, line, col = pos
+        assert isinstance(source_desc, SourceDescriptor)
+        contents = self.globalstate.commented_file_contents(source_desc)
+        lines = contents[max(0, line-3):line]  # line numbers start at 1
+        lines[-1] += u'             # <<<<<<<<<<<<<<'
+        lines += contents[line:line+2]
+        return u'"%s":%d\n%s\n' % (source_desc.get_escaped_description(), line, u'\n'.join(lines))
 
     def put_safe(self, code):
         # put code, but ignore {}
@@ -1653,24 +1665,6 @@ class CCodeWriter(object):
     def get_py_version_hex(self, pyversion):
         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
 
-    def mark_pos(self, pos):
-        if pos is None:
-            return
-        source_desc, line, col = pos
-        if self.last_marker_line == line:
-            return
-        assert isinstance(source_desc, SourceDescriptor)
-        contents = self.globalstate.commented_file_contents(source_desc)
-        lines = contents[max(0, line-3):line]  # line numbers start at 1
-        lines[-1] += u'             # <<<<<<<<<<<<<<'
-        lines += contents[line:line+2]
-
-        marker = u'"%s":%d\n%s\n' % (
-            source_desc.get_escaped_description(), line, u'\n'.join(lines))
-        self.marker = (line, marker)
-        if self.emit_linenums:
-            self.source_desc = source_desc.get_escaped_description()
-
     def put_label(self, lbl):
         if lbl in self.funcstate.labels_used:
             self.putln("%s:;" % lbl)
@@ -2099,8 +2093,8 @@ class CCodeWriter(object):
         self.putln('__Pyx_TraceDeclarations(%s, %d)' % (codeobj or 'NULL', nogil))
 
     def put_trace_call(self, name, pos, nogil=False):
-        self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d);' % (
-            name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil))
+        self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d, %s);' % (
+            name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil, self.error_goto(pos)))
 
     def put_trace_exception(self):
         self.putln("__Pyx_TraceException();")

Cython/Utility/Profile.c

@@ -49,7 +49,7 @@
   if (codeobj) $frame_code_cname = (PyCodeObject*) codeobj;
 
   #ifdef WITH_THREAD
-  #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil)                         \
+  #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error)             \
   if (nogil) {                                                                           \
       if (CYTHON_TRACE_NOGIL) {                                                          \
           PyThreadState *tstate;                                                         \
@@ -60,20 +60,23 @@
               __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno);  \
           }                                                                              \
           PyGILState_Release(state);                                                     \
+          if (unlikely(__Pyx_use_tracing < 0)) goto_error;                               \
       }                                                                                  \
   } else {                                                                               \
       PyThreadState* tstate = PyThreadState_GET();                                       \
       if (unlikely(tstate->use_tracing) && !tstate->tracing &&                           \
               (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {        \
           __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno);  \
+          if (unlikely(__Pyx_use_tracing < 0)) goto_error;                               \
       }                                                                                  \
   }
   #else
-  #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil)                         \
+  #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error)             \
   {   PyThreadState* tstate = PyThreadState_GET();                                       \
       if (unlikely(tstate->use_tracing) && !tstate->tracing &&                           \
               (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {        \
           __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno);  \
+          if (unlikely(__Pyx_use_tracing < 0)) goto_error;                               \
       }                                                                                  \
   }
   #endif
@@ -150,58 +153,71 @@
 #else
 
   #define __Pyx_TraceDeclarations(codeobj, nogil)
-  #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil)
+  // mark error label as used to avoid compiler warnings
+  #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error)   if (1); else goto_error;
   #define __Pyx_TraceException()
   #define __Pyx_TraceReturn(result, nogil)
 
 #endif /* CYTHON_PROFILE */
 
 #if CYTHON_TRACE
-  // FIXME: we should eventually propagate trace errors instead of just swallowing them
   // see call_trace_protected() in CPython's ceval.c
-  static void __Pyx_call_line_trace_func(PyThreadState *tstate, PyFrameObject *frame, int lineno) {
+  static int __Pyx_call_line_trace_func(PyThreadState *tstate, PyFrameObject *frame, int lineno) {
+      int ret;
       PyObject *type, *value, *traceback;
       PyErr_Fetch(&type, &value, &traceback);
       frame->f_lineno = lineno;
       tstate->tracing++;
       tstate->use_tracing = 0;
-      tstate->c_tracefunc(tstate->c_traceobj, frame, PyTrace_LINE, NULL);
+      ret = tstate->c_tracefunc(tstate->c_traceobj, frame, PyTrace_LINE, NULL);
       tstate->use_tracing = 1;
       tstate->tracing--;
-      PyErr_Restore(type, value, traceback);
+      if (likely(!ret)) {
+          PyErr_Restore(type, value, traceback);
+      } else {
+          Py_XDECREF(type);
+          Py_XDECREF(value);
+          Py_XDECREF(traceback);
+      }
+      return ret;
   }
 
   #ifdef WITH_THREAD
-  #define __Pyx_TraceLine(lineno, nogil)                                                   \
+  #define __Pyx_TraceLine(lineno, nogil, goto_error)                                       \
   if (likely(!__Pyx_use_tracing)); else {                                                  \
       if (nogil) {                                                                         \
           if (CYTHON_TRACE_NOGIL) {                                                        \
+              int ret = 0;                                                                 \
               PyThreadState *tstate;                                                       \
               PyGILState_STATE state = PyGILState_Ensure();                                \
               tstate = PyThreadState_GET();                                                \
               if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {                  \
-                  __Pyx_call_line_trace_func(tstate, $frame_cname, lineno);                \
+                  ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno);          \
               }                                                                            \
               PyGILState_Release(state);                                                   \
+              if (unlikely(ret)) goto_error;                                               \
           }                                                                                \
       } else {                                                                             \
           PyThreadState* tstate = PyThreadState_GET();                                     \
           if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {                      \
-              __Pyx_call_line_trace_func(tstate, $frame_cname, lineno);                    \
+              int ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno);          \
+              if (unlikely(ret)) goto_error;                                               \
           }                                                                                \
       }                                                                                    \
   }
   #else
-  #define __Pyx_TraceLine(lineno, nogil)                                                   \
+  #define __Pyx_TraceLine(lineno, nogil, goto_error)                                       \
   if (likely(!__Pyx_use_tracing)); else {                                                  \
       PyThreadState* tstate = PyThreadState_GET();                                         \
       if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {                          \
-          __Pyx_call_line_trace_func(tstate, $frame_cname, lineno);                        \
+          int ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno);              \
+          if (unlikely(ret)) goto_error;                                                   \
       }                                                                                    \
   }
   #endif
 #else
-  #define __Pyx_TraceLine(lineno, nogil)
+  // mark error label as used to avoid compiler warnings
+  #define __Pyx_TraceLine(lineno, nogil, goto_error)   if (1); else goto_error;
 #endif
 
 /////////////// Profile ///////////////
@@ -255,12 +271,13 @@ static int __Pyx_TraceSetupAndCall(PyCodeObject** code,
     tstate->tracing--;
     if (retval) {
         PyErr_Restore(type, value, traceback);
+        return tstate->use_tracing && retval;
     } else {
         Py_XDECREF(type);
         Py_XDECREF(value);
         Py_XDECREF(traceback);
+        return -1;
     }
-    return tstate->use_tracing && retval;
 }
 
 static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno) {

tests/run/line_trace.pyx

@@ -31,6 +31,21 @@ cdef int _trace_func(PyObject* _traceobj, PyFrameObject* _frame, int what, PyObj
     return 0
 
 
+cdef int _failing_call_trace_func(PyObject* _traceobj, PyFrameObject* _frame, int what, PyObject* arg) except -1:
+    if what == PyTrace_CALL:
+        raise ValueError("failing call trace!")
+    return _trace_func(_traceobj, _frame, what, arg)
+
+
+cdef int _failing_line_trace_func(PyObject* _traceobj, PyFrameObject* _frame, int what, PyObject* arg) except -1:
+    if what == PyTrace_LINE and _traceobj:
+        trace = <object>_traceobj
+        if len(trace) == 1:
+            # right after call with empty trace list => fail!
+            raise ValueError("failing line trace!")
+    return _trace_func(_traceobj, _frame, what, arg)
+
+
 def cy_add(a,b):
     x = a + b
     return x
@@ -63,3 +78,47 @@ def run_trace(func, *args):
     finally:
         PyEval_SetTrace(NULL, None)
     return trace
+
+
+def fail_on_call_trace(func, *args):
+    """
+    >>> def py_add(a,b):
+    ...     x = a+b
+    ...     return x
+    >>> fail_on_call_trace(py_add, 1, 2)
+    Traceback (most recent call last):
+    ValueError: failing call trace!
+    """
+    trace = []
+    PyEval_SetTrace(<Py_tracefunc>_failing_call_trace_func, trace)
+    try:
+        func(*args)
+    finally:
+        PyEval_SetTrace(NULL, None)
+    assert not trace
+
+
+def fail_on_line_trace(bint fail):
+    """
+    >>> fail_on_line_trace(False)
+    ['STARTING', ('call', 0), ('line', 1), ('line', 2), ('return', 2), ('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+    >>> fail_on_line_trace(True)
+    Traceback (most recent call last):
+    ValueError: failing line trace!
+    """
+    cdef int x = 1
+    trace = ['STARTING']
+    PyEval_SetTrace(<Py_tracefunc>_failing_line_trace_func, trace)
+    try:
+        x += 1
+        cy_add(1, 2)
+        x += 1
+        if fail:
+            del trace[:]  # trigger error
+        x += 1
+        cy_add(3, 4)
+        x += 1
+    finally:
+        PyEval_SetTrace(NULL, None)
+    assert x == 5
+    return trace