Pyrex merge: nogil declaration checking

Cython/Compiler/Nodes.py

@@ -107,6 +107,15 @@ class Node(object):
         self.pos = pos
         self.__dict__.update(kw)
 
+    gil_message = "Operation"
+
+    def gil_check(self, env):
+        if env.nogil:
+            self.gil_error()
+
+    def gil_error(self):
+        error(self.pos, "%s not allowed without gil" % self.gil_message)
+
     def get_child_accessors(self):
         """Returns an iterator over the children of the Node. Each member in the
         iterated list is an object with get(), set(value), and name() methods,
@@ -488,9 +497,9 @@ class CFuncDeclaratorNode(CDeclaratorNode):
             # Catch attempted C-style func(void) decl
             if type.is_void:
                 error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
-            if type.is_pyobject and self.nogil:
-                error(self.pos,
-                    "Function with Python argument cannot be declared nogil")
+#            if type.is_pyobject and self.nogil:
+#                error(self.pos,
+#                    "Function with Python argument cannot be declared nogil")
             func_type_args.append(
                 PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
             if arg_node.default:
@@ -537,9 +546,6 @@ class CFuncDeclaratorNode(CDeclaratorNode):
                         error(self.exception_value.pos,
                             "Exception value incompatible with function return type")
             exc_check = self.exception_check
-        if return_type.is_pyobject and self.nogil:
-            error(self.pos,
-                "Function with Python return type cannot be declared nogil")
         if return_type.is_array:
             error(self.pos,
                 "Function cannot return an array")
@@ -867,6 +873,9 @@ class FuncDefNode(StatNode, BlockNode):
         genv = env.global_scope()
         lenv = LocalScope(name = self.entry.name, outer_scope = genv)
         lenv.return_type = self.return_type
+        type = self.entry.type
+        if type.is_cfunction:
+            lenv.nogil = type.nogil and not type.with_gil
         code.init_labels()
         self.declare_arguments(lenv)
         transforms.run('before_analyse_function', self, env=env, lenv=lenv, genv=genv)
@@ -1113,8 +1122,12 @@ class CFuncDefNode(FuncDefNode):
             self.declare_argument(env, arg)
             
     def need_gil_acquisition(self, lenv):
+        type = self.type
         with_gil = self.type.with_gil
-        if self.type.nogil and not with_gil:
+        if type.nogil and not with_gil:
+            if type.return_type.is_pyobject:
+                error(self.pos,
+                      "Function with Python return type cannot be declared nogil")
             for entry in lenv.var_entries + lenv.temp_entries:
                 if entry.type.is_pyobject:
                     error(self.pos, "Function declared nogil has Python locals or temporaries")
@@ -2216,6 +2229,7 @@ class SingleAssignmentNode(AssignmentNode):
     def analyse_types(self, env, use_temp = 0):
         self.rhs.analyse_types(env)
         self.lhs.analyse_target_types(env)
+        self.lhs.gil_assignment_check(env)
         self.rhs = self.rhs.coerce_to(self.lhs.type, env)
         if use_temp:
             self.rhs = self.rhs.coerce_to_temp(env)
@@ -2280,6 +2294,7 @@ class CascadedAssignmentNode(AssignmentNode):
         self.coerced_rhs_list = []
         for lhs in self.lhs_list:
             lhs.analyse_target_types(env)
+            lhs.gil_assignment_check(env)
             rhs = CloneNode(self.rhs)
             rhs = rhs.coerce_to(lhs.type, env)
             self.coerced_rhs_list.append(rhs)
@@ -2522,15 +2537,9 @@ class PrintStatNode(StatNode):
         self.arg_tuple = self.arg_tuple.coerce_to_pyobject(env)
         self.arg_tuple.release_temp(env)
         env.use_utility_code(printing_utility_code)
-        return
-        for i in range(len(self.args)):
-            arg = self.args[i]
-            arg.analyse_types(env)
-            arg = arg.coerce_to_pyobject(env)
-            arg.allocate_temps(env)
-            arg.release_temp(env)
-            self.args[i] = arg
-            #env.recycle_pending_temps() # TEMPORARY
+        self.gil_check(env)
+
+    gil_message = "Python print statement"
 
     def generate_execution_code(self, code):
         self.arg_tuple.generate_evaluation_code(code)
@@ -2559,10 +2568,14 @@ class DelStatNode(StatNode):
     def analyse_expressions(self, env):
         for arg in self.args:
             arg.analyse_target_expression(env, None)
-            if not arg.type.is_pyobject:
+            if arg.type.is_pyobject:
+                self.gil_check(env)
+            else:
                 error(arg.pos, "Deletion of non-Python object")
             #arg.release_target_temp(env)
 
+    gil_message = "Deleting Python object"
+
     def generate_execution_code(self, code):
         for arg in self.args:
             if arg.type.is_pyobject:
@@ -2649,6 +2662,10 @@ class ReturnStatNode(StatNode):
                 and not return_type.is_pyobject
                 and not return_type.is_returncode):
                     error(self.pos, "Return value required")
+        if return_type.is_pyobject:
+            self.gil_check(env)
+
+    gil_message = "Returning Python object"
 
     def generate_execution_code(self, code):
         code.mark_pos(self.pos)
@@ -2711,10 +2728,10 @@ class RaiseStatNode(StatNode):
             self.exc_value.release_temp(env)
         if self.exc_tb:
             self.exc_tb.release_temp(env)
-#		if not (self.exc_type or self.exc_value or self.exc_tb):
-#			env.use_utility_code(reraise_utility_code)
-#		else:
         env.use_utility_code(raise_utility_code)
+        self.gil_check(env)
+
+    gil_message = "Raising exception"
 
     def generate_execution_code(self, code):
         if self.exc_type:
@@ -2764,8 +2781,11 @@ class ReraiseStatNode(StatNode):
     child_attrs = []
 
     def analyse_expressions(self, env):
+        self.gil_check(env)
         env.use_utility_code(raise_utility_code)
 
+    gil_message = "Raising exception"
+
     def generate_execution_code(self, code):
         vars = code.exc_vars
         if vars:
@@ -2792,8 +2812,11 @@ class AssertStatNode(StatNode):
         self.cond.release_temp(env)
         if self.value:
             self.value.release_temp(env)
+        self.gil_check(env)
         #env.recycle_pending_temps() # TEMPORARY
 
+    gil_message = "Raising exception"
+    
     def generate_execution_code(self, code):
         code.putln("#ifndef PYREX_WITHOUT_ASSERTIONS")
         self.cond.generate_evaluation_code(code)
@@ -2888,7 +2911,6 @@ class IfClauseNode(Node):
         self.condition = \
             self.condition.analyse_temp_boolean_expression(env)
         self.condition.release_temp(env)
-        #env.recycle_pending_temps() # TEMPORARY
         self.body.analyse_expressions(env)
     
     def generate_execution_code(self, code, end_label):
@@ -3250,6 +3272,7 @@ class TryExceptStatNode(StatNode):
             except_clause.analyse_declarations(env)
         if self.else_clause:
             self.else_clause.analyse_declarations(env)
+        self.gil_check(env)
     
     def analyse_expressions(self, env):
         self.body.analyse_expressions(env)
@@ -3258,6 +3281,9 @@ class TryExceptStatNode(StatNode):
             except_clause.analyse_expressions(env)
         if self.else_clause:
             self.else_clause.analyse_expressions(env)
+        self.gil_check(env)
+
+    gil_message = "Try-except statement"
 
     def generate_execution_code(self, code):
         old_error_label = code.new_error_label()
@@ -3418,15 +3444,10 @@ class TryFinallyStatNode(StatNode):
     def analyse_expressions(self, env):
         self.body.analyse_expressions(env)
         self.cleanup_list = env.free_temp_entries[:]
-        #self.exc_vars = (
-        #	env.allocate_temp(PyrexTypes.py_object_type),
-        #	env.allocate_temp(PyrexTypes.py_object_type),
-        #	env.allocate_temp(PyrexTypes.py_object_type))
-        #self.lineno_var = \
-        #	env.allocate_temp(PyrexTypes.c_int_type)
         self.finally_clause.analyse_expressions(env)
-        #for var in self.exc_vars:
-        #	env.release_temp(var)
+        self.gil_check(env)
+
+    gil_message = "Try-finally statement"
 
     def generate_execution_code(self, code):
         old_error_label = code.error_label
@@ -3572,6 +3593,15 @@ class GILStatNode(TryFinallyStatNode):
             body = body,
             finally_clause = GILExitNode(pos, state = state))
 
+    def analyse_expressions(self, env):
+        was_nogil = env.nogil
+        env.nogil = 1
+        TryFinallyStatNode.analyse_expressions(self, env)
+        env.nogil = was_nogil
+
+    def gil_check(self, env):
+        pass
+
     def generate_execution_code(self, code):
         code.putln("/*with %s:*/ {" % self.state)
         if self.state == 'gil':
@@ -3582,19 +3612,6 @@ class GILStatNode(TryFinallyStatNode):
         TryFinallyStatNode.generate_execution_code(self, code)
         code.putln("}")
 
-#class GILEntryNode(StatNode):
-#	#  state   string   'gil' or 'nogil'
-#
-#	def analyse_expressions(self, env):
-#		pass
-#
-#	def generate_execution_code(self, code):
-#		if self.state == 'gil':
-#			code.putln("PyGILState_STATE _save = PyGILState_Ensure();")
-#		else:
-#			code.putln("PyThreadState *_save;")
-#			code.putln("Py_UNBLOCK_THREADS")
-
 
 class GILExitNode(StatNode):
     #  Used as the 'finally' block in a GILStatNode

Cython/Compiler/PyrexTypes.py

@@ -695,9 +695,10 @@ class CFuncType(CType):
             return 0
         if not self.same_calling_convention_as(other_type):
             return 0
+        if self.nogil != other_type.nogil:
+            return 0
         return 1
 
-    
     def compatible_signature_with(self, other_type, as_cmethod = 0):
         return self.compatible_signature_with_resolved_type(other_type.resolve(), as_cmethod)
     
@@ -789,22 +790,24 @@ class CFuncType(CType):
         arg_decl_code = ", ".join(arg_decl_list)
         if not arg_decl_code and not pyrex:
             arg_decl_code = "void"
-        exc_clause = ""
+        trailer = ""
         if (pyrex or for_display) and not self.return_type.is_pyobject:
             if self.exception_value and self.exception_check:
-                exc_clause = " except? %s" % self.exception_value
+                trailer = " except? %s" % self.exception_value
             elif self.exception_value:
-                exc_clause = " except %s" % self.exception_value
+                trailer = " except %s" % self.exception_value
             elif self.exception_check == '+':
-                exc_clause = " except +"
+                trailer = " except +"
             else:
-                " except *"
+                " except *" # ignored
+            if self.nogil:
+                trailer += " nogil"
         cc = self.calling_convention_prefix()
         if (not entity_code and cc) or entity_code.startswith("*"):
             entity_code = "(%s%s)" % (cc, entity_code)
             cc = ""
         return self.return_type.declaration_code(
-            "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, exc_clause),
+            "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, trailer),
             for_display, dll_linkage, pyrex)
 	
     def function_header_code(self, func_name, arg_code):

Cython/Compiler/Symtab.py

@@ -150,12 +150,14 @@ class Scope:
     # pystring_entries  [Entry]            String const entries newly used as
     #                                        Python strings in this scope
     # control_flow     ControlFlow  Used for keeping track of environment state
+    # nogil             boolean            In a nogil section
 
     is_py_class_scope = 0
     is_c_class_scope = 0
     is_module_scope = 0
     scope_prefix = ""
     in_cinclude = 0
+    nogil = 0
     
     def __init__(self, name, outer_scope, parent_scope):
         # The outer_scope is the next scope in the lookup chain.
@@ -1324,6 +1326,7 @@ class CClassScope(ClassScope):
 #                    entry.type = type
                 else:
                     error(pos, "Signature not compatible with previous declaration")
+                    error(entry.pos, "Previous declaration is here")
         else:
             if self.defined:
                 error(pos,
@@ -1363,8 +1366,8 @@ class CClassScope(ClassScope):
                 entry.is_variable = 1
                 self.inherited_var_entries.append(entry)
         for base_entry in base_scope.cfunc_entries:
-            entry = self.add_cfunction(base_entry.name, base_entry.type, None,
-                adapt(base_entry.cname), base_entry.visibility)
+            entry = self.add_cfunction(base_entry.name, base_entry.type,
+                    base_entry.pos, adapt(base_entry.cname), base_entry.visibility)
             entry.is_inherited = 1
             
     def allocate_temp(self, type):

runtests.py

@@ -17,7 +17,7 @@ CFLAGS = os.getenv('CFLAGS', '').split()
 
 
 class ErrorWriter(object):
-    match_error = re.compile('(warning:)?(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
+    match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
     def __init__(self):
         self.output = []
         self.write = self.output.append
@@ -31,8 +31,9 @@ class ErrorWriter(object):
                 is_warning, line, column, message = match.groups()
                 if (is_warning and collect_warnings) or \
                         (not is_warning and collect_errors):
-                    result.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
-        return result
+                    result.append( (int(line), int(column), message.strip()) )
+        result.sort()
+        return [ "%d:%d:%s" % values for values in result ]
 
     def geterrors(self):
         return self._collect(True, False)

tests/compile/nogil.h

+void e1(void);
+void *e2(void);

tests/compile/nogil.pyx

-cdef extern void g(int x) nogil
+cdef extern object g(object x) nogil
+cdef extern void g2(object x) nogil
+
+cdef extern from "nogil.h":
+        void e1() nogil
+        int *e2() nogil
 
 cdef void f(int x) nogil:
         cdef int y
         y = 42
+
+cdef void h(object x) nogil:
+        cdef void *p
+        g2(x)
+        g2(<object>p)
+        p = <void *>x
+        e1()
+        e2()

tests/errors/nogil.pyx

+cdef object f(object x) nogil:
+    pass
+
+cdef void g(int x) nogil:
+    cdef object z
+    z = None
+
+cdef void h(int x) nogil:
+    p()
+
+cdef object p() nogil:
+    pass
+
+cdef void r() nogil:
+    q()
+    
+cdef object m():
+    cdef object x, y, obj
+    cdef int i, j, k
+    global fred
+    q()
+    with nogil:
+        r()
+        q()
+        i = 42
+        obj = None
+        17L
+        7j
+        help
+        `"Hello"`
+        import fred
+        from fred import obj
+        for x in obj:
+            pass
+        obj[i]
+        obj[i:j]
+        obj[i:j:k]
+        obj.fred
+        (x, y)
+        [x, y]
+        {x: y}
+        obj and x
+        t(obj)
+#        f(42) # Cython handles this internally
+        x + obj
+        -obj
+        x = y = obj
+        x, y = y, x
+        obj[i] = x
+        obj.fred = x
+        print obj
+        del fred
+        return obj
+        raise obj
+        if obj:
+            pass
+        while obj:
+            pass
+        for x <= obj <= y:
+            pass
+        try:
+            pass
+        except:
+            pass
+        try:
+            pass
+        finally:
+            pass
+
+cdef void q():
+    pass
+
+cdef class C:
+    pass
+
+cdef void t(C c) nogil:
+    pass
+
+
+_ERRORS = u"""
+ 1: 5: Function with Python return type cannot be declared nogil
+ 6: 6: Assignment of Python object not allowed without gil
+ 4: 5: Function declared nogil has Python locals or temporaries
+ 8: 5: Function declared nogil has Python locals or temporaries
+11: 5: Function with Python return type cannot be declared nogil
+15: 5: Calling gil-requiring function without gil
+24: 9: Calling gil-requiring function without gil
+26:12: Assignment of Python object not allowed without gil
+27: 8: Constructing Python long int not allowed without gil
+28: 8: Constructing complex number not allowed without gil
+29:12: Accessing Python global or builtin not allowed without gil
+30: 8: Backquote expression not allowed without gil
+31:15: Python import not allowed without gil
+31:15: Assignment of Python object not allowed without gil
+32:13: Python import not allowed without gil
+32:25: Constructing Python list not allowed without gil
+33:17: Iterating over Python object not allowed without gil
+35:11: Indexing Python object not allowed without gil
+36:11: Slicing Python object not allowed without gil
+37:11: Constructing Python slice object not allowed without gil
+37:11: Indexing Python object not allowed without gil
+37:13: Converting to Python object not allowed without gil
+37:15: Converting to Python object not allowed without gil
+37:17: Converting to Python object not allowed without gil
+38:11: Accessing Python attribute not allowed without gil
+39: 9: Constructing Python tuple not allowed without gil
+40: 8: Constructing Python list not allowed without gil
+41: 8: Constructing Python dict not allowed without gil
+42:12: Creating temporary Python reference not allowed without gil
+42:12: Truth-testing Python object not allowed without gil
+42:17: Creating temporary Python reference not allowed without gil
+43:13: Python type test not allowed without gil
+#44: 4: Converting to Python object not allowed without gil
+45:10: Operation not allowed without gil
+46: 8: Operation not allowed without gil
+47:10: Assignment of Python object not allowed without gil
+47:14: Assignment of Python object not allowed without gil
+48: 9: Assignment of Python object not allowed without gil
+48:13: Assignment of Python object not allowed without gil
+48:16: Creating temporary Python reference not allowed without gil
+48:19: Creating temporary Python reference not allowed without gil
+49:11: Indexing Python object not allowed without gil
+49:11: Assignment of Python object not allowed without gil
+50:11: Accessing Python attribute not allowed without gil
+50:11: Assignment of Python object not allowed without gil
+51: 8: Constructing Python tuple not allowed without gil
+51: 8: Python print statement not allowed without gil
+52: 8: Deleting Python object not allowed without gil
+53: 8: Returning Python object not allowed without gil
+54: 8: Raising exception not allowed without gil
+55:14: Truth-testing Python object not allowed without gil
+57:17: Truth-testing Python object not allowed without gil
+59: 8: Converting to Python object not allowed without gil
+61: 8: Try-except statement not allowed without gil
+65: 8: Try-finally statement not allowed without gil
+"""

tests/errors/nogilfunctype.pyx

+cdef extern from *:
+    cdef void f() nogil
+    cdef void (*fp)()
+
+fp = f
+
+_ERRORS = u"""
+5:6: Cannot assign type 'void (void) nogil' to 'void (*)(void)'
+"""

tests/run/withnogil.pyx

@@ -18,5 +18,5 @@ def g():
         h()
     return 1
 
-cdef int h() except -1:
+cdef int h() nogil except -1:
     pass