Fix #281 (Possible race condition w/ nogil exception propagation)

Cython/Compiler/Code.py

@@ -134,6 +134,12 @@ class FunctionState(object):
         self.temp_counter = 0
         self.closure_temps = None
 
+        # This is used for the error indicator, which needs to be local to the
+        # function. It used to be global, which relies on the GIL being held.
+        # However, exceptions may need to be propagated through 'nogil'
+        # sections, in which case we introduce a race condition.
+        self.should_declare_error_indicator = False
+
     # labels
 
     def new_label(self, name=None):
@@ -1435,10 +1441,12 @@ class CCodeWriter(object):
         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
 
     def set_error_info(self, pos):
+        self.funcstate.should_declare_error_indicator = True
         if self.c_line_in_traceback:
             cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
         else:
             cinfo = ""
+
         return "%s = %s[%s]; %s = %s;%s" % (
             Naming.filename_cname,
             Naming.filetable_cname,
@@ -1478,6 +1486,20 @@ class CCodeWriter(object):
     def put_finish_refcount_context(self):
         self.putln("__Pyx_RefNannyFinishContext();")
 
+    def put_add_traceback(self, qualified_name):
+        """
+        Build a Python traceback for propagating exceptions.
+
+        qualified_name should be the qualified name of the function
+        """
+        format_tuple = (
+            qualified_name,
+            Naming.clineno_cname,
+            Naming.lineno_cname,
+            Naming.filename_cname,
+        )
+        self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple)
+
     def put_trace_declarations(self):
         self.putln('__Pyx_TraceDeclarations');
 

Cython/Compiler/ModuleNode.py

@@ -1755,7 +1755,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if code.label_used(code.error_label):
             code.put_label(code.error_label)
             # This helps locate the offending name.
-            code.putln('__Pyx_AddTraceback("%s");' % self.full_module_name);
+            code.put_add_traceback(self.full_module_name)
         code.error_label = old_error_label
         code.putln("bad:")
         code.putln("return -1;")
@@ -1868,7 +1868,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         for cname, type in code.funcstate.all_managed_temps():
             code.put_xdecref(cname, type)
         code.putln('if (%s) {' % env.module_cname)
-        code.putln('__Pyx_AddTraceback("init %s");' % env.qualified_name)
+        code.put_add_traceback("init %s" % env.qualified_name)
         env.use_utility_code(Nodes.traceback_utility_code)
         code.put_decref_clear(env.module_cname, py_object_type, nanny=False)
         code.putln('} else if (!PyErr_Occurred()) {')

Cython/Compiler/Nodes.py

@@ -1474,7 +1474,7 @@ class FuncDefNode(StatNode, BlockNode):
                 # TODO: Fix exception tracing (though currently unused by cProfile).
                 # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                 # code.put_trace_exception()
-                code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
+                code.put_add_traceback(self.entry.qualified_name)
             else:
                 warning(self.entry.pos, "Unraisable exception in function '%s'." \
                             % self.entry.qualified_name, 0)
@@ -1563,6 +1563,12 @@ class FuncDefNode(StatNode, BlockNode):
 
         # ----- Go back and insert temp variable declarations
         tempvardecl_code.put_temp_declarations(code.funcstate)
+        if code.funcstate.should_declare_error_indicator:
+            tempvardecl_code.putln("int %s;" % Naming.lineno_cname)
+            tempvardecl_code.putln("const char *%s;" % Naming.filename_cname)
+            if code.c_line_in_traceback:
+                tempvardecl_code.putln("int %s;" % Naming.clineno_cname)
+
         # ----- Python version
         code.exit_cfunc_scope()
         if self.py_func:
@@ -2457,7 +2463,7 @@ class DefNode(FuncDefNode):
                         code.put_var_xdecref_clear(self.starstar_arg.entry)
                     else:
                         code.put_var_decref_clear(self.starstar_arg.entry)
-            code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
+            code.put_add_traceback(self.entry.qualified_name)
             # The arguments are put into the closure one after the
             # other, so when type errors are found, all references in
             # the closure instance must be properly ref-counted to
@@ -5272,7 +5278,7 @@ class ExceptClauseNode(Node):
         exc_vars = [code.funcstate.allocate_temp(py_object_type,
                                                  manage_ref=True)
                     for i in xrange(3)]
-        code.putln('__Pyx_AddTraceback("%s");' % self.function_name)
+        code.put_add_traceback(self.function_name)
         # We always have to fetch the exception value even if
         # there is no target, because this also normalises the
         # exception and stores it in the thread state.
@@ -7125,15 +7131,22 @@ requires=[raise_double_keywords_utility_code])
 #------------------------------------------------------------------------------------
 
 traceback_utility_code = UtilityCode(
-proto = """
-static void __Pyx_AddTraceback(const char *funcname); /*proto*/
-""",
-impl = """
+    proto = """
+static void __Pyx_AddTraceback(const char *funcname, int %(CLINENO)s,
+                               int %(LINENO)s, const char *%(FILENAME)s); /*proto*/
+""" % {
+    'FILENAME': Naming.filename_cname,
+    'LINENO':  Naming.lineno_cname,
+    'CLINENO':  Naming.clineno_cname,
+},
+
+    impl = """
 #include "compile.h"
 #include "frameobject.h"
 #include "traceback.h"
 
-static void __Pyx_AddTraceback(const char *funcname) {
+static void __Pyx_AddTraceback(const char *funcname, int %(CLINENO)s,
+                               int %(LINENO)s, const char *%(FILENAME)s) {
     PyObject *py_srcfile = 0;
     PyObject *py_funcname = 0;
     PyObject *py_globals = 0;