Merge -dagss and -devel

Cython/Compiler/Builtin.py

@@ -82,7 +82,7 @@ builtin_function_table = [
 builtin_types_table = [
 
     ("type",    "PyType_Type",     []),
-#    ("str",     "PyString_Type",   []),
+#    ("str",     "PyBytes_Type",   []),
     ("unicode", "PyUnicode_Type",  []),
     ("file",    "PyFile_Type",     []),
 #    ("slice",   "PySlice_Type",    []),

Cython/Compiler/CmdLine.py

@@ -17,6 +17,10 @@ Options:
   -I, --include-dir <directory>  Search for include files in named directory
                                  (multiply include directories are allowed).
   -o, --output-file <filename>   Specify name of generated C file
+  -r, --recursive                Recursively find and compile dependencies
+  -t, --timestamps               Only compile newer source files (implied with -r)
+  -f, --force                    Compile all source files (overrides implied -t)
+  -q, --quiet                    Don't print module names in recursive mode
   -p, --embed-positions          If specified, the positions in Cython files of each
                                  function definition is embedded in its docstring.
   -z, --pre-import <module>      If specified, assume undeclared names in this 
@@ -111,6 +115,12 @@ def parse_command_line(args):
                 options.working_path = pop_arg()
             elif option in ("-o", "--output-file"):
                 options.output_file = pop_arg()
+            elif option in ("-r", "--recursive"):
+                options.recursive = 1
+            elif option in ("-t", "--timestamps"):
+                options.timestamps = 1
+            elif option in ("-f", "--force"):
+                options.timestamps = 0
             elif option in ("-p", "--embed-positions"):
                 Options.embed_pos_in_docstring = 1
             elif option in ("-z", "--pre-import"):

Cython/Compiler/ExprNodes.py

@@ -300,7 +300,11 @@ class ExprNode(Node):
     
     def analyse_target_types(self, env):
         self.analyse_types(env)
-    
+
+    def gil_assignment_check(self, env):
+        if env.nogil and self.type.is_pyobject:
+            error(self.pos, "Assignment of Python object not allowed without gil")
+
     def check_const(self):
         self.not_const()
     
@@ -312,7 +316,11 @@ class ExprNode(Node):
     
     def addr_not_const(self):
         error(self.pos, "Address is not constant")
-    
+
+    def gil_check(self, env):
+        if env.nogil and self.type.is_pyobject:
+            self.gil_error()
+
     # ----------------- Result Allocation -----------------
     
     def result_in_temp(self):
@@ -758,11 +766,16 @@ class LongNode(AtomicExprNode):
     
     def compile_time_value(self, denv):
         return long(self.value)
+
+    gil_message = "Constructing Python long int"
     
     def analyse_types(self, env):
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Constructing Python long int"
+
     def generate_evaluation_code(self, code):
         code.putln(
             '%s = PyLong_FromString("%s", 0, 0); %s' % (
@@ -781,8 +794,11 @@ class ImagNode(AtomicExprNode):
     
     def analyse_types(self, env):
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Constructing complex number"
+
     def generate_evaluation_code(self, code):
         code.putln(
             "%s = PyComplex_FromDoubles(0.0, %s); %s" % (
@@ -883,7 +899,10 @@ class NameNode(AtomicExprNode):
             else:
                 self.is_temp = 1
             env.use_utility_code(get_name_interned_utility_code)
-    
+            self.gil_check(env)
+
+    gil_message = "Accessing Python global or builtin"
+
     def analyse_entry(self, env):
         #print "NameNode.analyse_entry:", self.name ###
         self.check_identifier_kind()
@@ -1060,8 +1079,11 @@ class BackquoteNode(ExprNode):
         self.arg.analyse_types(env)
         self.arg = self.arg.coerce_to_pyobject(env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Backquote expression"
+
     def generate_result_code(self, code):
         code.putln(
             "%s = PyObject_Repr(%s); %s" % (
@@ -1086,9 +1108,12 @@ class ImportNode(ExprNode):
         if self.name_list:
             self.name_list.analyse_types(env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
         env.use_utility_code(import_utility_code)
-    
+
+    gil_message = "Python import"
+
     def generate_result_code(self, code):
         if self.name_list:
             name_list_code = self.name_list.py_result()
@@ -1114,11 +1139,14 @@ class IteratorNode(ExprNode):
         self.sequence.analyse_types(env)
         self.sequence = self.sequence.coerce_to_pyobject(env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
         
         self.counter = TempNode(self.pos, PyrexTypes.c_py_ssize_t_type, env)
         self.counter.allocate_temp(env)
-        
+
+    gil_message = "Iterating over Python object"
+
     def release_temp(self, env):
         env.release_temp(self.result_code)
         self.counter.release_temp(env)
@@ -1265,6 +1293,7 @@ class IndexNode(ExprNode):
             else:
                 self.index = self.index.coerce_to_pyobject(env)
             self.type = py_object_type
+            self.gil_check(env)
             self.is_temp = 1
         else:
             if self.base.type.is_ptr or self.base.type.is_array:
@@ -1281,7 +1310,9 @@ class IndexNode(ExprNode):
                 error(self.pos,
                     "Invalid index type '%s'" %
                         self.index.type)
-    
+
+    gil_message = "Indexing Python object"
+
     def check_const_addr(self):
         self.base.check_const_addr()
         self.index.check_const()
@@ -1408,8 +1439,11 @@ class SliceIndexNode(ExprNode):
         if self.stop:
             self.stop = self.stop.coerce_to(c_int, env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Slicing Python object"
+
     def generate_result_code(self, code):
         code.putln(
             "%s = PySequence_GetSlice(%s, %s, %s); %s" % (
@@ -1482,8 +1516,11 @@ class SliceNode(ExprNode):
         self.stop = self.stop.coerce_to_pyobject(env)
         self.step = self.step.coerce_to_pyobject(env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Constructing Python slice object"
+
     def generate_result_code(self, code):
         code.putln(
             "%s = PySlice_New(%s, %s, %s); %s" % (
@@ -1493,7 +1530,15 @@ class SliceNode(ExprNode):
                 self.step.py_result(),
                 code.error_goto_if_null(self.result_code, self.pos)))
 
-class SimpleCallNode(ExprNode):
+
+class CallNode(ExprNode):
+    def gil_check(self, env):
+        # Make sure we're not in a nogil environment
+        if env.nogil:
+            error(self.pos, "Calling gil-requiring function without gil")
+
+
+class SimpleCallNode(CallNode):
     #  Function call without keyword, * or ** args.
     #
     #  function       ExprNode
@@ -1542,6 +1587,7 @@ class SimpleCallNode(ExprNode):
             self.arg_tuple.analyse_types(env)
             self.args = None
             self.type = py_object_type
+            self.gil_check(env)
             self.is_temp = 1
         else:
             for arg in self.args:
@@ -1614,6 +1660,9 @@ class SimpleCallNode(ExprNode):
         if func_type.exception_check == '+':
             if func_type.exception_value is None:
                 env.use_utility_code(cpp_exception_utility_code)
+        # Check gil
+        if not func_type.nogil:
+            self.gil_check(env)
 
     def calculate_result_code(self):
         return self.c_call_code()
@@ -1708,7 +1757,7 @@ class SimpleCallNode(ExprNode):
                         rhs,
                         code.error_goto_if(" && ".join(exc_checks), self.pos)))    
 
-class GeneralCallNode(ExprNode):
+class GeneralCallNode(CallNode):
     #  General Python function call, including keyword,
     #  * and ** arguments.
     #
@@ -1744,6 +1793,7 @@ class GeneralCallNode(ExprNode):
             self.starstar_arg = \
                 self.starstar_arg.coerce_to_pyobject(env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
         
     def generate_result_code(self, code):
@@ -1794,8 +1844,11 @@ class AsTupleNode(ExprNode):
         self.arg.analyse_types(env)
         self.arg = self.arg.coerce_to_pyobject(env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Constructing Python tuple"
+
     def generate_result_code(self, code):
         code.putln(
             "%s = PySequence_Tuple(%s); %s" % (
@@ -2000,12 +2053,15 @@ class AttributeNode(ExprNode):
             self.type = py_object_type
             self.is_py_attr = 1
             self.interned_attr_cname = env.intern_identifier(self.attribute)
+            self.gil_check(env)
         else:
             if not obj_type.is_error:
                 error(self.pos, 
                     "Object of type '%s' has no attribute '%s'" %
                     (obj_type, self.attribute))
-        
+
+    gil_message = "Accessing Python attribute"
+
     def is_simple(self):
         if self.obj:
             return self.result_in_temp() or self.obj.is_simple()
@@ -2122,8 +2178,9 @@ class SequenceNode(ExprNode):
             arg.analyse_types(env)
             self.args[i] = arg.coerce_to_pyobject(env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
     def analyse_target_types(self, env):
         self.iterator = PyTempNode(self.pos, env)
         self.unpacked_items = []
@@ -2220,7 +2277,9 @@ class SequenceNode(ExprNode):
 
 class TupleNode(SequenceNode):
     #  Tuple constructor.
-    
+
+    gil_message = "Constructing Python tuple"
+
     def analyse_types(self, env):
         if len(self.args) == 0:
             self.is_temp = 0
@@ -2271,7 +2330,9 @@ class TupleNode(SequenceNode):
 
 class ListNode(SequenceNode):
     #  List constructor.
-    
+
+    gil_message = "Constructing Python list"
+
     def analyse_types(self, env):
         SequenceNode.analyse_types(self, env)
         self.type = list_type
@@ -2368,8 +2429,11 @@ class DictNode(ExprNode):
         for item in self.key_value_pairs:
             item.analyse_types(env)
         self.type = dict_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Constructing Python dict"
+
     def allocate_temps(self, env, result = None):
         #  Custom method used here because key-value
         #  pairs are evaluated and used one at a time.
@@ -2444,9 +2508,12 @@ class ClassNode(ExprNode):
             self.doc = self.doc.coerce_to_pyobject(env)
         self.module_name = env.global_scope().qualified_name
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
         env.use_utility_code(create_class_utility_code);
 
+    gil_message = "Constructing Python class"
+
     def generate_result_code(self, code):
         if self.doc:
             code.put_error_if_neg(self.pos, 
@@ -2476,8 +2543,11 @@ class UnboundMethodNode(ExprNode):
     def analyse_types(self, env):
         self.function.analyse_types(env)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Constructing an unbound method"
+
     def generate_result_code(self, code):
         code.putln(
             "%s = PyMethod_New(%s, 0, %s); %s" % (
@@ -2496,8 +2566,11 @@ class PyCFunctionNode(AtomicExprNode):
     
     def analyse_types(self, env):
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
-    
+
+    gil_message = "Constructing Python function"
+
     def generate_result_code(self, code):
         code.putln(
             "%s = PyCFunction_New(&%s, 0); %s" % (
@@ -2549,6 +2622,7 @@ class UnopNode(ExprNode):
         if self.is_py_operation():
             self.coerce_operand_to_pyobject(env)
             self.type = py_object_type
+            self.gil_check(env)
             self.is_temp = 1
         else:
             self.analyse_c_operation(env)
@@ -2898,6 +2972,7 @@ class BinopNode(ExprNode):
         if self.is_py_operation():
             self.coerce_operands_to_pyobjects(env)
             self.type = py_object_type
+            self.gil_check(env)
             self.is_temp = 1
             if Options.incref_local_binop and self.operand1.type.is_pyobject:
                 self.operand1 = self.operand1.coerce_to_temp(env)
@@ -3129,6 +3204,7 @@ class BoolBinopNode(ExprNode):
             self.operand2 = self.operand2.coerce_to_pyobject(env)
             self.temp_bool = TempNode(self.pos, PyrexTypes.c_bint_type, env)
             self.type = py_object_type
+            self.gil_check(env)
         else:
             self.operand1 = self.operand1.coerce_to_boolean(env)
             self.operand2 = self.operand2.coerce_to_boolean(env)
@@ -3137,11 +3213,10 @@ class BoolBinopNode(ExprNode):
         # both operands be temp nodes.
         self.operand1 = self.operand1.coerce_to_temp(env) #CTT
         self.operand2 = self.operand2.coerce_to_temp(env)
-        # coerce_to_simple does not seem to be sufficient
-        #self.operand1 = self.operand1.coerce_to_simple(env)
-        #self.operand2 = self.operand2.coerce_to_simple(env)
         self.is_temp = 1
-    
+
+    gil_message = "Truth-testing Python object"
+
     def allocate_temps(self, env, result_code = None):
         #  We don't need both operands at the same time, and
         #  one of the operands will also be our result. So we
@@ -3438,7 +3513,6 @@ class PrimaryCmpNode(ExprNode, CmpNode):
         if self.has_int_operands():
             self.coerce_chars_to_ints(env)
         if self.cascade:
-            #self.operand2 = self.operand2.coerce_to_temp(env) #CTT
             self.operand2 = self.operand2.coerce_to_simple(env)
             self.cascade.coerce_cascaded_operands_to_temp(env)
         self.check_operand_types(env)
@@ -3688,8 +3762,11 @@ class PyTypeTestNode(CoercionNode):
         assert dst_type.is_extension_type or dst_type.is_builtin_type, "PyTypeTest on non extension type"
         CoercionNode.__init__(self, arg)
         self.type = dst_type
+        self.gil_check(env)
         self.result_ctype = arg.ctype()
         env.use_utility_code(type_test_utility_code)
+
+    gil_message = "Python type test"
     
     def analyse_types(self, env):
         pass
@@ -3724,11 +3801,14 @@ class CoerceToPyTypeNode(CoercionNode):
     def __init__(self, arg, env):
         CoercionNode.__init__(self, arg)
         self.type = py_object_type
+        self.gil_check(env)
         self.is_temp = 1
         if not arg.type.to_py_function:
             error(arg.pos,
                 "Cannot convert '%s' to Python object" % arg.type)
-    
+
+    gil_message = "Converting to Python object"
+
     def generate_result_code(self, code):
         function = self.arg.type.to_py_function
         code.putln('%s = %s(%s); %s' % (
@@ -3773,7 +3853,11 @@ class CoerceToBooleanNode(CoercionNode):
         CoercionNode.__init__(self, arg)
         self.type = PyrexTypes.c_bint_type
         if arg.type.is_pyobject:
+            if env.nogil:
+                self.gil_error()
             self.is_temp = 1
+
+    gil_message = "Truth-testing Python object"
     
     def check_const(self):
         if self.is_temp:
@@ -3802,8 +3886,11 @@ class CoerceToTempNode(CoercionNode):
         self.type = self.arg.type
         self.is_temp = 1
         if self.type.is_pyobject:
+            self.gil_check(env)
             self.result_ctype = py_object_type
-    
+
+    gil_message = "Creating temporary Python reference"
+
     def generate_result_code(self, code):
         #self.arg.generate_evaluation_code(code) # Already done
         # by generic generate_subexpr_evaluation_code!

Cython/Compiler/Main.py

@@ -3,8 +3,8 @@
 #
 
 import os, sys, re, codecs
-if sys.version_info[:2] < (2, 2):
-    sys.stderr.write("Sorry, Cython requires Python 2.2 or later\n")
+if sys.version_info[:2] < (2, 3):
+    sys.stderr.write("Sorry, Cython requires Python 2.3 or later\n")
     sys.exit(1)
 
 try:
@@ -14,14 +14,13 @@ except NameError:
     from sets import Set as set
 
 from time import time
+import Code
+import Errors
+import Parsing
 import Version
 from Scanning import PyrexScanner, FileSourceDescriptor
-import Errors
 from Errors import PyrexError, CompileError, error
-import Parsing
 from Symtab import BuiltinScope, ModuleScope
-import Code
-from Cython.Utils import replace_suffix
 from Cython import Utils
 
 # Note: PHASES and TransformSet should be removed soon; but that's for
@@ -116,24 +115,27 @@ class Context:
                     pass
         return scope
     
-    def find_pxd_file(self, module_name, pos):
-        # Search include directories for the .pxd file
-        # corresponding to the given (full) module name.
-        if "." in module_name:
-            pxd_filename = "%s.pxd" % os.path.join(*module_name.split('.'))
-        else:
-            pxd_filename = "%s.pxd" % module_name
-        return self.search_include_directories(pxd_filename, pos)
+    def find_pxd_file(self, qualified_name, pos):
+        # Search include path for the .pxd file corresponding to the
+        # given fully-qualified module name.
+        return self.search_include_directories(qualified_name, ".pxd", pos)
+
+    def find_pyx_file(self, qualified_name, pos):
+        # Search include path for the .pyx file corresponding to the
+        # given fully-qualified module name, as for find_pxd_file().
+        return self.search_include_directories(qualified_name, ".pyx", pos)
     
     def find_include_file(self, filename, pos):
         # Search list of include directories for filename.
         # Reports an error and returns None if not found.
-        path = self.search_include_directories(filename, pos)
+        path = self.search_include_directories(filename, "", pos,
+                                               split_package=False)
         if not path:
             error(pos, "'%s' not found" % filename)
         return path
     
-    def search_include_directories(self, filename, pos):
+    def search_include_directories(self, qualified_name, suffix, pos,
+                                   split_package=True):
         # Search the list of include directories for the given
         # file name. If a source file position is given, first
         # searches the directory containing that file. Returns
@@ -145,12 +147,88 @@ class Context:
                 raise RuntimeError("Only file sources for code supported")
             here_dir = os.path.dirname(file_desc.filename)
             dirs = [here_dir] + dirs
+
+        dotted_filename = qualified_name + suffix
+        if split_package:
+            names = qualified_name.split('.')
+            package_names = names[:-1]
+            module_name = names[-1]
+            module_filename = module_name + suffix
+            package_filename = "__init__" + suffix
+
         for dir in dirs:
-            path = os.path.join(dir, filename)
+            path = os.path.join(dir, dotted_filename)
             if os.path.exists(path):
                 return path
+            if split_package:
+                package_dir = self.check_package_dir(dir, package_names)
+                if package_dir is not None:
+                    path = os.path.join(package_dir, module_filename)
+                    if os.path.exists(path):
+                        return path
+                    path = os.path.join(dir, package_dir, module_name,
+                                        package_filename)
+                    if os.path.exists(path):
+                        return path
         return None
 
+    def check_package_dir(self, dir, package_names):
+        package_dir = os.path.join(dir, *package_names)
+        if not os.path.exists(package_dir):
+            return None
+        for dirname in package_names:
+            dir = os.path.join(dir, dirname)
+            package_init = os.path.join(dir, "__init__.py")
+            if not os.path.exists(package_init) and \
+                    not os.path.exists(package_init + "x"): # same with .pyx ?
+                return None
+        return package_dir
+
+    def c_file_out_of_date(self, source_path):
+        c_path = Utils.replace_suffix(source_path, ".c")
+        if not os.path.exists(c_path):
+            return 1
+        c_time = Utils.modification_time(c_path)
+        if Utils.file_newer_than(source_path, c_time):
+            return 1
+        pos = [source_path]
+        pxd_path = Utils.replace_suffix(source_path, ".pxd")
+        if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time):
+            return 1
+        for kind, name in self.read_dependency_file(source_path):
+            if kind == "cimport":
+                dep_path = self.find_pxd_file(name, pos)
+            elif kind == "include":
+                dep_path = self.search_include_directories(name, pos)
+            else:
+                continue
+            if dep_path and Utils.file_newer_than(dep_path, c_time):
+                return 1
+        return 0
+    
+    def find_cimported_module_names(self, source_path):
+        return [ name for kind, name in self.read_dependency_file(source_path)
+                 if kind == "cimport" ]
+
+    def is_package_dir(self, dir_path):
+        #  Return true if the given directory is a package directory.
+        for filename in ("__init__.py", "__init__.pyx"):
+            path = os.path.join(dir_path, filename)
+            if os.path.exists(path):
+                return 1
+
+    def read_dependency_file(self, source_path):
+        dep_path = replace_suffix(source_path, ".dep")
+        if os.path.exists(dep_path):
+            f = open(dep_path, "rU")
+            chunks = [ line.strip().split(" ", 1)
+                       for line in f.readlines()
+                       if " " in line.strip() ]
+            f.close()
+            return chunks
+        else:
+            return ()
+
     def lookup_submodule(self, name):
         # Look up a top-level module. Returns None if not found.
         return self.modules.get(name, None)
@@ -184,10 +262,23 @@ class Context:
         return tree
 
     def extract_module_name(self, path, options):
-        # Get the module name out of a source file pathname.
-        _, tail = os.path.split(path)
-        name, _ = os.path.splitext(tail)
-        return name
+        # Find fully_qualified module name from the full pathname
+        # of a source file.
+        dir, filename = os.path.split(path)
+        module_name, _ = os.path.splitext(filename)
+        if "." in module_name:
+            return module_name
+        if module_name == "__init__":
+            dir, module_name = os.path.split(dir)
+        names = [module_name]
+        while self.is_package_dir(dir):
+            parent, package_name = os.path.split(dir)
+            if parent == dir:
+                break
+            names.append(package_name)
+            dir = parent
+        names.reverse()
+        return ".".join(names)
 
     def compile(self, source, options = None, full_module_name = None):
         # Compile a Pyrex implementation file in this context
@@ -197,16 +288,11 @@ class Context:
         result = CompilationResult()
         cwd = os.getcwd()
 
-        if full_module_name is None:
-            full_module_name, _ = os.path.splitext(source)
-            full_module_name = re.sub(r'[\\/]', '.', full_module_name)
-            full_module_name = re.sub(r'[^\w.]', '_', full_module_name)
-
         source = os.path.join(cwd, source)
         result.main_source_file = source
 
         if options.use_listing_file:
-            result.listing_file = replace_suffix(source, ".lis")
+            result.listing_file = Utils.replace_suffix(source, ".lis")
             Errors.open_listing_file(result.listing_file,
                 echo_to_stderr = options.errors_to_stderr)
         else:
@@ -218,20 +304,21 @@ class Context:
                 c_suffix = ".cpp"
             else:
                 c_suffix = ".c"
-            result.c_file = replace_suffix(source, c_suffix)
+            result.c_file = Utils .replace_suffix(source, c_suffix)
         c_stat = None
         if result.c_file:
             try:
                 c_stat = os.stat(result.c_file)
             except EnvironmentError:
                 pass
-        module_name = full_module_name # self.extract_module_name(source, options)
+        module_name = full_module_name or self.extract_module_name(source, options)
         source = FileSourceDescriptor(source)
         initial_pos = (source, 1, 0)
         scope = self.find_module(module_name, pos = initial_pos, need_pxd = 0)
         errors_occurred = False
         try:
-            tree = self.parse(source, scope.type_names, pxd = 0, full_module_name = full_module_name)
+            tree = self.parse(source, scope.type_names, pxd = 0,
+                              full_module_name = full_module_name)
             tree.process_implementation(scope, options, result)
         except CompileError:
             errors_occurred = True
@@ -241,8 +328,7 @@ class Context:
             errors_occurred = True
         if errors_occurred and result.c_file:
             try:
-                #os.unlink(result.c_file)
-                Utils.castrate_file(result.c_file, c_stat)
+                Utils.castrate_file(result.c_file, os.stat(source))
             except EnvironmentError:
                 pass
             result.c_file = None
@@ -259,7 +345,7 @@ class Context:
 
 #------------------------------------------------------------------------
 #
-#  Main Python entry point
+#  Main Python entry points
 #
 #------------------------------------------------------------------------
 
@@ -273,6 +359,11 @@ class CompilationOptions:
     include_path      [string]  Directories to search for include files
     output_file       string    Name of generated .c file
     generate_pxi      boolean   Generate .pxi file for public declarations
+    recursive         boolean   Recursively find and compile dependencies
+    timestamps        boolean   Only compile changed source files. If None,
+                                defaults to true when recursive is true.
+    verbose           boolean   Always print source names being compiled
+    quiet             boolean   Don't print source names in recursive mode
     transforms        Transform.TransformSet Transforms to use on the parse tree
     
     Following options are experimental and only used on MacOSX:
@@ -283,7 +374,7 @@ class CompilationOptions:
     cplus             boolean   Compile as c++ code
     """
     
-    def __init__(self, defaults = None, **kw):
+    def __init__(self, defaults = None, c_compile = 0, c_link = 0, **kw):
         self.include_path = []
         self.objects = []
         if defaults:
@@ -293,6 +384,10 @@ class CompilationOptions:
             defaults = default_options
         self.__dict__.update(defaults)
         self.__dict__.update(kw)
+        if c_compile:
+            self.c_only = 0
+        if c_link:
+            self.obj_only = 0
 
 
 class CompilationResult:
@@ -320,24 +415,90 @@ class CompilationResult:
         self.main_source_file = None
 
 
-def compile(source, options = None, c_compile = 0, c_link = 0,
-            full_module_name = None):
+class CompilationResultSet(dict):
+    """
+    Results from compiling multiple Pyrex source files. A mapping
+    from source file paths to CompilationResult instances. Also
+    has the following attributes:
+    
+    num_errors   integer   Total number of compilation errors
+    """
+    
+    num_errors = 0
+
+    def add(self, source, result):
+        self[source] = result
+        self.num_errors += result.num_errors
+
+
+def compile_single(source, options, full_module_name = None):
     """
-    compile(source, options = default_options)
+    compile_single(source, options, full_module_name)
     
-    Compile the given Cython implementation file and return
-    a CompilationResult object describing what was produced.
+    Compile the given Pyrex implementation file and return a CompilationResult.
+    Always compiles a single file; does not perform timestamp checking or
+    recursion.
     """
-    if not options:
-        options = default_options
-    options = CompilationOptions(defaults = options)
-    if c_compile:
-        options.c_only = 0
-    if c_link:
-        options.obj_only = 0
     context = Context(options.include_path)
     return context.compile(source, options, full_module_name)
 
+def compile_multiple(sources, options):
+    """
+    compile_multiple(sources, options)
+    
+    Compiles the given sequence of Pyrex implementation files and returns
+    a CompilationResultSet. Performs timestamp checking and/or recursion
+    if these are specified in the options.
+    """
+    sources = [os.path.abspath(source) for source in sources]
+    processed = set()
+    results = CompilationResultSet()
+    context = Context(options.include_path)
+    recursive = options.recursive
+    timestamps = options.timestamps
+    if timestamps is None:
+        timestamps = recursive
+    verbose = options.verbose or ((recursive or timestamps) and not options.quiet)
+    for source in sources:
+        if source not in processed:
+            if not timestamps or context.c_file_out_of_date(source):
+                if verbose:
+                    sys.stderr.write("Compiling %s\n" % source)
+                result = context.compile(source, options)
+                # Compiling multiple sources in one context doesn't quite
+                # work properly yet.
+                context = Context(options.include_path) # to be removed later
+                results.add(source, result)
+            processed.add(source)
+            if recursive:
+                for module_name in context.find_cimported_module_names(source):
+                    path = context.find_pyx_file(module_name, [source])
+                    if path:
+                        sources.append(path)
+                    else:
+                        sys.stderr.write(
+                            "Cannot find .pyx file for cimported module '%s'\n" % module_name)
+    return results
+
+def compile(source, options = None, c_compile = 0, c_link = 0,
+            full_module_name = None, **kwds):
+    """
+    compile(source [, options], [, <option> = <value>]...)
+    
+    Compile one or more Pyrex implementation files, with optional timestamp
+    checking and recursing on dependecies. The source argument may be a string
+    or a sequence of strings If it is a string and no recursion or timestamp
+    checking is requested, a CompilationResult is returned, otherwise a
+    CompilationResultSet is returned.
+    """
+    options = CompilationOptions(defaults = options, c_compile = c_compile,
+        c_link = c_link, **kwds)
+    if isinstance(source, basestring) and not options.timestamps \
+            and not options.recursive:
+        return compile_single(source, options, full_module_name)
+    else:
+        return compile_multiple(source, options)
+
 #------------------------------------------------------------------------
 #
 #  Main command-line entry point
@@ -351,21 +512,19 @@ def main(command_line = 0):
         from CmdLine import parse_command_line
         options, sources = parse_command_line(args)
     else:
-        options = default_options
+        options = CompilationOptions(default_options)
         sources = args
     if options.show_version:
         sys.stderr.write("Cython version %s\n" % Version.version)
     if options.working_path!="":
         os.chdir(options.working_path)
-    context = Context(options.include_path)
-    for source in sources:
-        try:
-            result = context.compile(source, options)
-            if result.num_errors > 0:
-                any_failures = 1
-        except PyrexError, e:
-            sys.stderr.write(str(e) + '\n')
+    try:
+        result = compile(sources, options)
+        if result.num_errors > 0:
             any_failures = 1
+    except (EnvironmentError, PyrexError), e:
+        sys.stderr.write(str(e) + '\n')
+        any_failures = 1
     if any_failures:
         sys.exit(1)
 
@@ -388,8 +547,11 @@ default_options = dict(
     annotate = False,
     generate_pxi = 0,
     transforms = TransformSet(),
-    working_path = "")
-    
+    working_path = "",
+    recursive = 0,
+    timestamps = None,
+    verbose = 0,
+    quiet = 0)
 if sys.platform == "mac":
     from Cython.Mac.MacSystem import c_compile, c_link, CCompilerError
     default_options['use_listing_file'] = 1

Cython/Compiler/ModuleNode.py

@@ -54,6 +54,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if self.has_imported_c_functions():
             self.module_temp_cname = env.allocate_temp_pyobject()
             env.release_temp(self.module_temp_cname)
+        self.generate_dep_file(env, result)
         self.generate_c_code(env, options, result)
         self.generate_h_code(env, options, result)
         self.generate_api_code(env, result)
@@ -65,6 +66,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     return 1
         return 0
     
+    def generate_dep_file(self, env, result):
+        modules = self.referenced_modules
+        if len(modules) > 1 or env.included_files:
+            dep_file = replace_suffix(result.c_file, ".dep")
+            f = open(dep_file, "w")
+            try:
+                for module in modules:
+                    if module is not env:
+                        f.write("cimport %s\n" % module.qualified_name)
+                    for path in module.included_files:
+                        f.write("include %s\n" % path)
+            finally:
+                f.close()
+
     def generate_h_code(self, env, options, result):
         def h_entries(entries, pxd = 0):
             return [entry for entry in entries
@@ -348,12 +363,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
     def generate_declarations_for_modules(self, env, modules, code):
         code.putln("")
-        code.putln("/* Declarations */")
+        code.putln("/* Type declarations */")
         vtab_list, vtabslot_list = self.sort_type_hierarchy(modules, env)
         self.generate_type_definitions(
             env, modules, vtab_list, vtabslot_list, code)
         for module in modules:
             defined_here = module is env
+            code.putln("/* Module declarations from %s */" %
+                       module.qualified_name.encode("ASCII", "ignore"))
             self.generate_global_declarations(module, code, defined_here)
             self.generate_cfunction_predeclarations(module, code, defined_here)
 
@@ -440,6 +457,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyInt_AsUnsignedLongMask     PyLong_AsUnsignedLongMask")
         code.putln("  #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask")
         code.putln("  #define PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
+        code.putln("#else")
+        code.putln("  #define PyBytes_Type                 PyString_Type")
         code.putln("#endif")
 
         code.putln("#if PY_MAJOR_VERSION >= 3")
@@ -686,8 +705,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     entry.type.typeptr_cname)
         code.put_var_declarations(env.var_entries, static = 1, 
             dll_linkage = "DL_EXPORT", definition = definition)
-        code.put_var_declarations(env.default_entries, static = 1,
-                                  definition = definition)
+        if definition:
+            code.put_var_declarations(env.default_entries, static = 1,
+                                      definition = definition)
     
     def generate_cfunction_predeclarations(self, env, code, definition):
         for entry in env.cfunc_entries:
@@ -2124,8 +2144,13 @@ __Pyx_import_all_from(PyObject *locals, PyObject *v)
 			break;
 		}
 		if (skip_leading_underscores &&
+#if PY_MAJOR_VERSION < 3
 		    PyString_Check(name) &&
 		    PyString_AS_STRING(name)[0] == '_')
+#else
+		    PyUnicode_Check(name) &&
+		    PyUnicode_AS_UNICODE(name)[0] == '_')
+#endif
 		{
 			Py_DECREF(name);
 			continue;
@@ -2147,10 +2172,11 @@ __Pyx_import_all_from(PyObject *locals, PyObject *v)
 }
 
 
-static int %s(PyObject* m) {
+static int %(IMPORT_STAR)s(PyObject* m) {
 
     int i;
     int ret = -1;
+    char* s;
     PyObject *locals = 0;
     PyObject *list = 0;
     PyObject *name;
@@ -2163,7 +2189,13 @@ static int %s(PyObject* m) {
     for(i=0; i<PyList_GET_SIZE(list); i++) {
         name = PyTuple_GET_ITEM(PyList_GET_ITEM(list, i), 0);
         item = PyTuple_GET_ITEM(PyList_GET_ITEM(list, i), 1);
-        if (%s(item, name, PyString_AsString(name)) < 0) goto bad;
+#if PY_MAJOR_VERSION < 3
+        s = PyString_AsString(name);
+#else
+        s = PyUnicode_AsString(name);
+#endif
+        if (!s) goto bad;
+        if (%(IMPORT_STAR_SET)s(item, name, s) < 0) goto bad;
     }
     ret = 0;
     
@@ -2172,4 +2204,5 @@ bad:
     Py_XDECREF(list);
     return ret;
 }
-""" % ( Naming.import_star, Naming.import_star_set )
+""" % {'IMPORT_STAR'     : Naming.import_star,
+       'IMPORT_STAR_SET' : Naming.import_star_set }

Cython/Compiler/Nodes.py

@@ -83,6 +83,15 @@ class Node(object):
         self.pos = pos
         self.__dict__.update(kw)
     
+    gil_message = "Operation"
+
+    def gil_check(self, env):
+        if env.nogil:
+            self.gil_error()
+
+    def gil_error(self):
+        error(self.pos, "%s not allowed without gil" % self.gil_message)
+
     def clone_node(self):
         """Clone the node. This is defined as a shallow copy, except for member lists
            amongst the child attributes (from get_child_accessors) which are also
@@ -431,9 +440,9 @@ class CFuncDeclaratorNode(CDeclaratorNode):
             # Catch attempted C-style func(void) decl
             if type.is_void:
                 error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
-            if type.is_pyobject and self.nogil:
-                error(self.pos,
-                    "Function with Python argument cannot be declared nogil")
+#            if type.is_pyobject and self.nogil:
+#                error(self.pos,
+#                    "Function with Python argument cannot be declared nogil")
             func_type_args.append(
                 PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
             if arg_node.default:
@@ -480,9 +489,6 @@ class CFuncDeclaratorNode(CDeclaratorNode):
                         error(self.exception_value.pos,
                             "Exception value incompatible with function return type")
             exc_check = self.exception_check
-        if return_type.is_pyobject and self.nogil:
-            error(self.pos,
-                "Function with Python return type cannot be declared nogil")
         if return_type.is_array:
             error(self.pos,
                 "Function cannot return an array")
@@ -810,6 +816,9 @@ class FuncDefNode(StatNode, BlockNode):
         genv = env.global_scope()
         lenv = LocalScope(name = self.entry.name, outer_scope = genv)
         lenv.return_type = self.return_type
+        type = self.entry.type
+        if type.is_cfunction:
+            lenv.nogil = type.nogil and not type.with_gil
         code.init_labels()
         self.declare_arguments(lenv)
         transforms.run('before_analyse_function', self, env=env, lenv=lenv, genv=genv)
@@ -890,24 +899,19 @@ class FuncDefNode(StatNode, BlockNode):
             exc_check = self.caller_will_check_exceptions()
             if err_val is not None or exc_check:
                 code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
-                if err_val is not None:
-                    code.putln(
-                        "%s = %s;" % (
-                            Naming.retval_cname, 
-                            err_val))
             else:
                 code.putln(
                     '__Pyx_WriteUnraisable("%s");' % 
                         self.entry.qualified_name)
                 env.use_utility_code(unraisable_exception_utility_code)
-                #if not self.return_type.is_void:
-                default_retval = self.return_type.default_value
-                if default_retval:
-                    code.putln(
-                        "%s = %s;" % (
-                            Naming.retval_cname,
-                            default_retval))
-                            #self.return_type.default_value))
+            default_retval = self.return_type.default_value
+            if err_val is None and default_retval:
+                err_val = default_retval
+            if err_val is not None:
+                code.putln(
+                    "%s = %s;" % (
+                        Naming.retval_cname, 
+                        err_val))
         # ----- Return cleanup
         code.put_label(code.return_label)
         if not Options.init_local_none:
@@ -1056,8 +1060,12 @@ class CFuncDefNode(FuncDefNode):
             self.declare_argument(env, arg)
             
     def need_gil_acquisition(self, lenv):
+        type = self.type
         with_gil = self.type.with_gil
-        if self.type.nogil and not with_gil:
+        if type.nogil and not with_gil:
+            if type.return_type.is_pyobject:
+                error(self.pos,
+                      "Function with Python return type cannot be declared nogil")
             for entry in lenv.var_entries + lenv.temp_entries:
                 if entry.type.is_pyobject:
                     error(self.pos, "Function declared nogil has Python locals or temporaries")
@@ -2159,6 +2167,7 @@ class SingleAssignmentNode(AssignmentNode):
     def analyse_types(self, env, use_temp = 0):
         self.rhs.analyse_types(env)
         self.lhs.analyse_target_types(env)
+        self.lhs.gil_assignment_check(env)
         self.rhs = self.rhs.coerce_to(self.lhs.type, env)
         if use_temp:
             self.rhs = self.rhs.coerce_to_temp(env)
@@ -2223,6 +2232,7 @@ class CascadedAssignmentNode(AssignmentNode):
         self.coerced_rhs_list = []
         for lhs in self.lhs_list:
             lhs.analyse_target_types(env)
+            lhs.gil_assignment_check(env)
             rhs = CloneNode(self.rhs)
             rhs = rhs.coerce_to(lhs.type, env)
             self.coerced_rhs_list.append(rhs)
@@ -2465,15 +2475,9 @@ class PrintStatNode(StatNode):
         self.arg_tuple = self.arg_tuple.coerce_to_pyobject(env)
         self.arg_tuple.release_temp(env)
         env.use_utility_code(printing_utility_code)
-        return
-        for i in range(len(self.args)):
-            arg = self.args[i]
-            arg.analyse_types(env)
-            arg = arg.coerce_to_pyobject(env)
-            arg.allocate_temps(env)
-            arg.release_temp(env)
-            self.args[i] = arg
-            #env.recycle_pending_temps() # TEMPORARY
+        self.gil_check(env)
+
+    gil_message = "Python print statement"
 
     def generate_execution_code(self, code):
         self.arg_tuple.generate_evaluation_code(code)
@@ -2502,10 +2506,14 @@ class DelStatNode(StatNode):
     def analyse_expressions(self, env):
         for arg in self.args:
             arg.analyse_target_expression(env, None)
-            if not arg.type.is_pyobject:
+            if arg.type.is_pyobject:
+                self.gil_check(env)
+            else:
                 error(arg.pos, "Deletion of non-Python object")
             #arg.release_target_temp(env)
-    
+
+    gil_message = "Deleting Python object"
+
     def generate_execution_code(self, code):
         for arg in self.args:
             if arg.type.is_pyobject:
@@ -2592,7 +2600,11 @@ class ReturnStatNode(StatNode):
                 and not return_type.is_pyobject
                 and not return_type.is_returncode):
                     error(self.pos, "Return value required")
-    
+        if return_type.is_pyobject:
+            self.gil_check(env)
+
+    gil_message = "Returning Python object"
+
     def generate_execution_code(self, code):
         code.mark_pos(self.pos)
         if not self.return_type:
@@ -2654,11 +2666,11 @@ class RaiseStatNode(StatNode):
             self.exc_value.release_temp(env)
         if self.exc_tb:
             self.exc_tb.release_temp(env)
-#		if not (self.exc_type or self.exc_value or self.exc_tb):
-#			env.use_utility_code(reraise_utility_code)
-#		else:
         env.use_utility_code(raise_utility_code)
-    
+        self.gil_check(env)
+
+    gil_message = "Raising exception"
+
     def generate_execution_code(self, code):
         if self.exc_type:
             self.exc_type.generate_evaluation_code(code)
@@ -2707,8 +2719,11 @@ class ReraiseStatNode(StatNode):
     child_attrs = []
 
     def analyse_expressions(self, env):
+        self.gil_check(env)
         env.use_utility_code(raise_utility_code)
 
+    gil_message = "Raising exception"
+
     def generate_execution_code(self, code):
         vars = code.exc_vars
         if vars:
@@ -2735,7 +2750,10 @@ class AssertStatNode(StatNode):
         self.cond.release_temp(env)
         if self.value:
             self.value.release_temp(env)
+        self.gil_check(env)
         #env.recycle_pending_temps() # TEMPORARY
+
+    gil_message = "Raising exception"
     
     def generate_execution_code(self, code):
         code.putln("#ifndef PYREX_WITHOUT_ASSERTIONS")
@@ -2831,7 +2849,6 @@ class IfClauseNode(Node):
         self.condition = \
             self.condition.analyse_temp_boolean_expression(env)
         self.condition.release_temp(env)
-        #env.recycle_pending_temps() # TEMPORARY
         self.body.analyse_expressions(env)
     
     def generate_execution_code(self, code, end_label):
@@ -3236,6 +3253,7 @@ class TryExceptStatNode(StatNode):
             except_clause.analyse_declarations(env)
         if self.else_clause:
             self.else_clause.analyse_declarations(env)
+        self.gil_check(env)
     
     def analyse_expressions(self, env):
         self.body.analyse_expressions(env)
@@ -3244,7 +3262,10 @@ class TryExceptStatNode(StatNode):
             except_clause.analyse_expressions(env)
         if self.else_clause:
             self.else_clause.analyse_expressions(env)
-    
+        self.gil_check(env)
+
+    gil_message = "Try-except statement"
+
     def generate_execution_code(self, code):
         old_error_label = code.new_error_label()
         our_error_label = code.error_label
@@ -3404,16 +3425,11 @@ class TryFinallyStatNode(StatNode):
     def analyse_expressions(self, env):
         self.body.analyse_expressions(env)
         self.cleanup_list = env.free_temp_entries[:]
-        #self.exc_vars = (
-        #	env.allocate_temp(PyrexTypes.py_object_type),
-        #	env.allocate_temp(PyrexTypes.py_object_type),
-        #	env.allocate_temp(PyrexTypes.py_object_type))
-        #self.lineno_var = \
-        #	env.allocate_temp(PyrexTypes.c_int_type)
         self.finally_clause.analyse_expressions(env)
-        #for var in self.exc_vars:
-        #	env.release_temp(var)
-    
+        self.gil_check(env)
+
+    gil_message = "Try-finally statement"
+
     def generate_execution_code(self, code):
         old_error_label = code.error_label
         old_labels = code.all_new_labels()
@@ -3558,6 +3574,15 @@ class GILStatNode(TryFinallyStatNode):
             body = body,
             finally_clause = GILExitNode(pos, state = state))
 
+    def analyse_expressions(self, env):
+        was_nogil = env.nogil
+        env.nogil = 1
+        TryFinallyStatNode.analyse_expressions(self, env)
+        env.nogil = was_nogil
+
+    def gil_check(self, env):
+        pass
+
     def generate_execution_code(self, code):
         code.putln("/*with %s:*/ {" % self.state)
         if self.state == 'gil':
@@ -3568,19 +3593,6 @@ class GILStatNode(TryFinallyStatNode):
         TryFinallyStatNode.generate_execution_code(self, code)
         code.putln("}")
 
-#class GILEntryNode(StatNode):
-#	#  state   string   'gil' or 'nogil'
-#
-#	def analyse_expressions(self, env):
-#		pass
-#
-#	def generate_execution_code(self, code):
-#		if self.state == 'gil':
-#			code.putln("PyGILState_STATE _save = PyGILState_Ensure();")
-#		else:
-#			code.putln("PyThreadState *_save;")
-#			code.putln("Py_UNBLOCK_THREADS")
-
 
 class GILExitNode(StatNode):
     #  Used as the 'finally' block in a GILStatNode

Cython/Compiler/Parsing.py

@@ -3,7 +3,6 @@
 #
 
 import os, re
-from string import join, replace
 from types import ListType, TupleType
 from Scanning import PyrexScanner, FileSourceDescriptor
 import Nodes
@@ -13,6 +12,26 @@ from Errors import error, InternalError
 from Cython import Utils
 import Future
 
+class Ctx(object):
+    #  Parsing context
+    level = 'other'
+    visibility = 'private'
+    cdef_flag = 0
+    typedef_flag = 0
+    api = 0
+    overridable = 0
+    nogil = 0
+
+    def __init__(self, **kwds):
+        self.__dict__.update(kwds)
+
+    def __call__(self, **kwds):
+        ctx = Ctx()
+        d = ctx.__dict__
+        d.update(self.__dict__)
+        d.update(kwds)
+        return ctx
+
 def p_ident(s, message = "Expected an identifier"):
     if s.sy == 'IDENT':
         name = s.systring
@@ -38,9 +57,7 @@ def p_ident_list(s):
 #------------------------------------------
 
 def p_binop_expr(s, ops, p_sub_expr):
-    #print "p_binop_expr:", ops, p_sub_expr ###
     n1 = p_sub_expr(s)
-    #print "p_binop_expr(%s):" % p_sub_expr, s.sy ###
     while s.sy in ops:
         op = s.sy
         pos = s.position()
@@ -74,7 +91,6 @@ def p_test(s):
 #or_test: and_test ('or' and_test)*
 
 def p_or_test(s):
-    #return p_binop_expr(s, ('or',), p_and_test)
     return p_rassoc_binop_expr(s, ('or',), p_and_test)
 
 def p_rassoc_binop_expr(s, ops, p_subexpr):
@@ -982,7 +998,7 @@ def p_from_import_statement(s, first_statement = 0):
             items.append(
                 (name,
                  ExprNodes.NameNode(name_pos, 
-                 	name = as_name or name)))
+                                    name = as_name or name)))
         import_list = ExprNodes.ListNode(
             imported_names[0][0], args = imported_name_strings)
         dotted_name = Utils.EncodedString(dotted_name)
@@ -1008,7 +1024,7 @@ def p_dotted_name(s, as_allowed):
         names.append(p_ident(s))
     if as_allowed:
         as_name = p_as_name(s)
-    return (pos, target_name, join(names, "."), as_name)
+    return (pos, target_name, u'.'.join(names), as_name)
 
 def p_as_name(s):
     if s.sy == 'IDENT' and s.systring == 'as':
@@ -1196,7 +1212,7 @@ def p_except_clause(s):
     return Nodes.ExceptClauseNode(pos,
         pattern = exc_type, target = exc_value, body = body)
 
-def p_include_statement(s, level):
+def p_include_statement(s, ctx):
     pos = s.position()
     s.next() # 'include'
     _, include_file_name = p_string_literal(s)
@@ -1204,11 +1220,12 @@ def p_include_statement(s, level):
     if s.compile_time_eval:
         include_file_path = s.context.find_include_file(include_file_name, pos)
         if include_file_path:
+            s.included_files.append(include_file_name)
             f = Utils.open_source_file(include_file_path, mode="rU")
             source_desc = FileSourceDescriptor(include_file_path)
             s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding)
             try:
-                tree = p_statement_list(s2, level)
+                tree = p_statement_list(s2, ctx)
             finally:
                 f.close()
             return tree
@@ -1258,7 +1275,7 @@ def p_simple_statement(s, first_statement = 0):
         node = p_expression_or_assignment(s)
     return node
 
-def p_simple_statement_list(s, first_statement = 0):
+def p_simple_statement_list(s, ctx, first_statement = 0):
     # Parse a series of simple statements on one line
     # separated by semicolons.
     stat = p_simple_statement(s, first_statement = first_statement)
@@ -1294,7 +1311,7 @@ def p_DEF_statement(s):
     s.expect_newline()
     return Nodes.PassStatNode(pos)
 
-def p_IF_statement(s, level, cdef_flag, visibility, api):
+def p_IF_statement(s, ctx):
     pos = s.position()
     saved_eval = s.compile_time_eval
     current_eval = saved_eval
@@ -1304,7 +1321,7 @@ def p_IF_statement(s, level, cdef_flag, visibility, api):
         s.next() # 'IF' or 'ELIF'
         expr = p_compile_time_expr(s)
         s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
-        body = p_suite(s, level, cdef_flag, visibility, api = api)
+        body = p_suite(s, ctx)
         if s.compile_time_eval:
             result = body
             current_eval = 0
@@ -1313,7 +1330,7 @@ def p_IF_statement(s, level, cdef_flag, visibility, api):
     if s.sy == 'ELSE':
         s.next()
         s.compile_time_eval = current_eval
-        body = p_suite(s, level, cdef_flag, visibility, api = api)
+        body = p_suite(s, ctx)
         if current_eval:
             result = body
     if not result:
@@ -1321,18 +1338,18 @@ def p_IF_statement(s, level, cdef_flag, visibility, api):
     s.compile_time_eval = saved_eval
     return result
 
-def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0,
-                first_statement = 0):
+def p_statement(s, ctx, first_statement = 0):
+    cdef_flag = ctx.cdef_flag
     if s.sy == 'ctypedef':
-        if level not in ('module', 'module_pxd'):
+        if ctx.level not in ('module', 'module_pxd'):
             s.error("ctypedef statement not allowed here")
-        if api:
+        if ctx.api:
             error(s.position(), "'api' not allowed with 'ctypedef'")
-        return p_ctypedef_statement(s, level, visibility, api)
+        return p_ctypedef_statement(s, ctx)
     elif s.sy == 'DEF':
         return p_DEF_statement(s)
     elif s.sy == 'IF':
-        return p_IF_statement(s, level, cdef_flag, visibility, api)
+        return p_IF_statement(s, ctx)
     else:
         overridable = 0
         if s.sy == 'cdef':
@@ -1343,36 +1360,32 @@ def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0,
             overridable = 1
             s.next()
         if cdef_flag:
-            if level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
+            if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
                 s.error('cdef statement not allowed here')
-            s.level = level
-            return p_cdef_statement(s, level, visibility = visibility,
-                                    api = api, overridable = overridable)
-    #    elif s.sy == 'cpdef':
-    #        s.next()
-    #        return p_c_func_or_var_declaration(s, level, s.position(), visibility = visibility, api = api, overridable = True)
+            s.level = ctx.level
+            return p_cdef_statement(s, ctx(overridable = overridable))
         else:
-            if api:
+            if ctx.api:
                 error(s.pos, "'api' not allowed with this statement")
             elif s.sy == 'def':
-                if level not in ('module', 'class', 'c_class', 'property'):
+                if ctx.level not in ('module', 'class', 'c_class', 'property'):
                     s.error('def statement not allowed here')
-                s.level = level
+                s.level = ctx.level
                 return p_def_statement(s)
             elif s.sy == 'class':
-                if level != 'module':
+                if ctx.level != 'module':
                     s.error("class definition not allowed here")
                 return p_class_statement(s)
             elif s.sy == 'include':
-                if level not in ('module', 'module_pxd'):
+                if ctx.level not in ('module', 'module_pxd'):
                     s.error("include statement not allowed here")
-                return p_include_statement(s, level)
-            elif level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
+                return p_include_statement(s, ctx)
+            elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
                 return p_property_decl(s)
-            elif s.sy == 'pass' and level != 'property':
+            elif s.sy == 'pass' and ctx.level != 'property':
                 return p_pass_statement(s, with_newline = 1)
             else:
-                if level in ('c_class_pxd', 'property'):
+                if ctx.level in ('c_class_pxd', 'property'):
                     s.error("Executable statement not allowed here")
                 if s.sy == 'if':
                     return p_if_statement(s)
@@ -1385,25 +1398,22 @@ def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0,
                 elif s.sy == 'with':
                     return p_with_statement(s)
                 else:
-                    return p_simple_statement_list(s, first_statement = first_statement)
+                    return p_simple_statement_list(
+                        s, ctx, first_statement = first_statement)
 
-def p_statement_list(s, level,
-        cdef_flag = 0, visibility = 'private', api = 0, first_statement = 0):
+def p_statement_list(s, ctx, first_statement = 0):
     # Parse a series of statements separated by newlines.
     pos = s.position()
     stats = []
     while s.sy not in ('DEDENT', 'EOF'):
-        stats.append(p_statement(s, level,
-            cdef_flag = cdef_flag, visibility = visibility, api = api,
-            first_statement = first_statement))
+        stats.append(p_statement(s, ctx, first_statement = first_statement))
         first_statement = 0
     if len(stats) == 1:
         return stats[0]
     else:
         return Nodes.StatListNode(pos, stats = stats)
 
-def p_suite(s, level = 'other', cdef_flag = 0,
-        visibility = 'private', with_doc = 0, with_pseudo_doc = 0, api = 0):
+def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
     pos = s.position()
     s.expect(':')
     doc = None
@@ -1413,17 +1423,13 @@ def p_suite(s, level = 'other', cdef_flag = 0,
         s.expect_indent()
         if with_doc or with_pseudo_doc:
             doc = p_doc_string(s)
-        body = p_statement_list(s, 
-            level = level,
-            cdef_flag = cdef_flag, 
-            visibility = visibility,
-            api = api)
+        body = p_statement_list(s, ctx)
         s.expect_dedent()
     else:
-        if api:
+        if ctx.api:
             error(s.pos, "'api' not allowed with this statement")
-        if level in ('module', 'class', 'function', 'other'):
-            body = p_simple_statement_list(s)
+        if ctx.level in ('module', 'class', 'function', 'other'):
+            body = p_simple_statement_list(s, ctx)
         else:
             body = p_pass_statement(s)
             s.expect_newline("Syntax error in declarations")
@@ -1556,8 +1562,9 @@ def p_opt_cname(s):
         cname = None
     return cname
 
-def p_c_declarator(s, empty = 0, is_type = 0, cmethod_flag = 0, assignable = 0,
-        nonempty = 0, calling_convention_allowed = 0):
+def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
+                   assignable = 0, nonempty = 0,
+                   calling_convention_allowed = 0):
     # If empty is true, the declarator must be empty. If nonempty is true,
     # the declarator must be nonempty. Otherwise we don't care.
     # If cmethod_flag is true, then if this declarator declares
@@ -1567,13 +1574,16 @@ def p_c_declarator(s, empty = 0, is_type = 0, cmethod_flag = 0, assignable = 0,
         s.next()
         if s.sy == ')' or looking_at_type(s):
             base = Nodes.CNameDeclaratorNode(pos, name = "", cname = None)
-            result = p_c_func_declarator(s, pos, base, cmethod_flag)
+            result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
         else:
-            result = p_c_declarator(s, empty, is_type, cmethod_flag, nonempty = nonempty,
-                calling_convention_allowed = 1)
+            result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
+                                    cmethod_flag = cmethod_flag,
+                                    nonempty = nonempty,
+                                    calling_convention_allowed = 1)
             s.expect(')')
     else:
-        result = p_c_simple_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty)
+        result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
+                                       assignable, nonempty)
     if not calling_convention_allowed and result.calling_convention and s.sy != '(':
         error(s.position(), "%s on something that is not a function"
             % result.calling_convention)
@@ -1583,7 +1593,7 @@ def p_c_declarator(s, empty = 0, is_type = 0, cmethod_flag = 0, assignable = 0,
             result = p_c_array_declarator(s, result)
         else: # sy == '('
             s.next()
-            result = p_c_func_declarator(s, pos, result, cmethod_flag)
+            result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
         cmethod_flag = 0
     return result
 
@@ -1597,10 +1607,10 @@ def p_c_array_declarator(s, base):
     s.expect(']')
     return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)
 
-def p_c_func_declarator(s, pos, base, cmethod_flag):
+def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
     #  Opening paren has already been skipped
-    args = p_c_arg_list(s, in_pyfunc = 0, cmethod_flag = cmethod_flag,
-        nonempty_declarators = 0)
+    args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
+                        nonempty_declarators = 0)
     ellipsis = p_optional_ellipsis(s)
     s.expect(')')
     nogil = p_nogil(s)
@@ -1609,19 +1619,24 @@ def p_c_func_declarator(s, pos, base, cmethod_flag):
     return Nodes.CFuncDeclaratorNode(pos, 
         base = base, args = args, has_varargs = ellipsis,
         exception_value = exc_val, exception_check = exc_check,
-        nogil = nogil or with_gil, with_gil = with_gil)
+        nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)
 
-def p_c_simple_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty):
+def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
+                          assignable, nonempty):
     pos = s.position()
     calling_convention = p_calling_convention(s)
     if s.sy == '*':
         s.next()
-        base = p_c_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty)
+        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
+                              cmethod_flag = cmethod_flag,
+                              assignable = assignable, nonempty = nonempty)
         result = Nodes.CPtrDeclaratorNode(pos, 
             base = base)
     elif s.sy == '**': # scanner returns this as a single token
         s.next()
-        base = p_c_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty)
+        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
+                              cmethod_flag = cmethod_flag,
+                              assignable = assignable, nonempty = nonempty)
         result = Nodes.CPtrDeclaratorNode(pos,
             base = Nodes.CPtrDeclaratorNode(pos,
                 base = base))
@@ -1687,28 +1702,14 @@ def p_exception_value_clause(s):
 
 c_arg_list_terminators = ('*', '**', '.', ')')
 
-#def p_c_arg_list(s, in_pyfunc, cmethod_flag = 0, nonempty_declarators = 0,
-#		kw_only = 0):
-#	args = []
-#	if s.sy not in c_arg_list_terminators:
-#		args.append(p_c_arg_decl(s, in_pyfunc, cmethod_flag,
-#			nonempty = nonempty_declarators, kw_only = kw_only))
-#		while s.sy == ',':
-#			s.next()
-#			if s.sy in c_arg_list_terminators:
-#				break
-#			args.append(p_c_arg_decl(s, in_pyfunc), nonempty = nonempty_declarators,
-#				kw_only = kw_only)
-#	return args
-
-def p_c_arg_list(s, in_pyfunc, cmethod_flag = 0, nonempty_declarators = 0,
-        kw_only = 0):
+def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
+                 nonempty_declarators = 0, kw_only = 0):
     #  Comma-separated list of C argument declarations, possibly empty.
     #  May have a trailing comma.
     args = []
     is_self_arg = cmethod_flag
     while s.sy not in c_arg_list_terminators:
-        args.append(p_c_arg_decl(s, in_pyfunc, is_self_arg,
+        args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
             nonempty = nonempty_declarators, kw_only = kw_only))
         if s.sy != ',':
             break
@@ -1723,12 +1724,12 @@ def p_optional_ellipsis(s):
     else:
         return 0
 
-def p_c_arg_decl(s, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):
+def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):
     pos = s.position()
     not_none = 0
     default = None
     base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
-    declarator = p_c_declarator(s, nonempty = nonempty)
+    declarator = p_c_declarator(s, ctx, nonempty = nonempty)
     if s.sy == 'not':
         s.next()
         if s.sy == 'IDENT' and s.systring == 'None':
@@ -1761,57 +1762,60 @@ def p_api(s):
     else:
         return 0
 
-def p_cdef_statement(s, level, visibility = 'private', api = 0,
-                     overridable = False):
+def p_cdef_statement(s, ctx):
     pos = s.position()
-    visibility = p_visibility(s, visibility)
-    api = api or p_api(s)
-    if api:
-        if visibility not in ('private', 'public'):
-            error(pos, "Cannot combine 'api' with '%s'" % visibility)
-    if (visibility == 'extern') and s.sy == 'from':
-            return p_cdef_extern_block(s, level, pos)
+    ctx.visibility = p_visibility(s, ctx.visibility)
+    ctx.api = ctx.api or p_api(s)
+    if ctx.api:
+        if ctx.visibility not in ('private', 'public'):
+            error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)
+    if (ctx.visibility == 'extern') and s.sy == 'from':
+        return p_cdef_extern_block(s, pos, ctx)
     elif s.sy == 'import':
         s.next()
-        return p_cdef_extern_block(s, level, pos)
-    elif s.sy == ':':
-        return p_cdef_block(s, level, visibility, api)
+        return p_cdef_extern_block(s, pos, ctx)
+    if p_nogil(s):
+        ctx.nogil = 1
+    if s.sy == ':':
+        return p_cdef_block(s, ctx)
     elif s.sy == 'class':
-        if level not in ('module', 'module_pxd'):
+        if ctx.level not in ('module', 'module_pxd'):
             error(pos, "Extension type definition not allowed here")
-        #if api:
+        #if ctx.api:
         #    error(pos, "'api' not allowed with extension class")
-        return p_c_class_definition(s, level, pos, visibility = visibility, api = api)
+        return p_c_class_definition(s, pos, ctx)
     elif s.sy == 'IDENT' and s.systring in struct_union_or_enum:
-        if level not in ('module', 'module_pxd'):
+        if ctx.level not in ('module', 'module_pxd'):
             error(pos, "C struct/union/enum definition not allowed here")
-        #if visibility == 'public':
+        #if ctx.visibility == 'public':
         #    error(pos, "Public struct/union/enum definition not implemented")
-        #if api:
+        #if ctx.api:
         #    error(pos, "'api' not allowed with '%s'" % s.systring)
         if s.systring == "enum":
-            return p_c_enum_definition(s, pos, level, visibility)
+            return p_c_enum_definition(s, pos, ctx)
         else:
-            return p_c_struct_or_union_definition(s, pos, level, visibility)
+            return p_c_struct_or_union_definition(s, pos, ctx)
     elif s.sy == 'pass':
         node = p_pass_statement(s)
         s.expect_newline('Expected a newline')
         return node
     else:
-        return p_c_func_or_var_declaration(s, level, pos, visibility, api,
-                                           overridable)
+        return p_c_func_or_var_declaration(s, pos, ctx)
 
-def p_cdef_block(s, level, visibility, api):
-    return p_suite(s, level, cdef_flag = 1, visibility = visibility, api = api)
+def p_cdef_block(s, ctx):
+    return p_suite(s, ctx(cdef_flag = 1))
 
-def p_cdef_extern_block(s, level, pos):
+def p_cdef_extern_block(s, pos, ctx):
     include_file = None
     s.expect('from')
     if s.sy == '*':
         s.next()
     else:
         _, include_file = p_string_literal(s)
-    body = p_suite(s, level, cdef_flag = 1, visibility = 'extern')
+    ctx = ctx(cdef_flag = 1, visibility = 'extern')
+    if p_nogil(s):
+        ctx.nogil = 1
+    body = p_suite(s, ctx)
     return Nodes.CDefExternNode(pos,
         include_file = include_file,
         body = body)
@@ -1820,7 +1824,7 @@ struct_union_or_enum = (
     "struct", "union", "enum"
 )
 
-def p_c_enum_definition(s, pos, level, visibility, typedef_flag = 0):
+def p_c_enum_definition(s, pos, ctx):
     # s.sy == ident 'enum'
     s.next()
     if s.sy == 'IDENT':
@@ -1842,9 +1846,10 @@ def p_c_enum_definition(s, pos, level, visibility, typedef_flag = 0):
         while s.sy not in ('DEDENT', 'EOF'):
             p_c_enum_line(s, items)
         s.expect_dedent()
-    return Nodes.CEnumDefNode(pos, name = name, cname = cname,
-        items = items, typedef_flag = typedef_flag, visibility = visibility,
-        in_pxd = level == 'module_pxd')
+    return Nodes.CEnumDefNode(
+        pos, name = name, cname = cname, items = items,
+        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
+        in_pxd = ctx.level == 'module_pxd')
 
 def p_c_enum_line(s, items):
     if s.sy != 'pass':
@@ -1869,7 +1874,7 @@ def p_c_enum_item(s, items):
     items.append(Nodes.CEnumDefItemNode(pos, 
         name = name, cname = cname, value = value))
 
-def p_c_struct_or_union_definition(s, pos, level, visibility, typedef_flag = 0):
+def p_c_struct_or_union_definition(s, pos, ctx):
     # s.sy == ident 'struct' or 'union'
     kind = s.systring
     s.next()
@@ -1882,10 +1887,11 @@ def p_c_struct_or_union_definition(s, pos, level, visibility, typedef_flag = 0):
         s.expect('NEWLINE')
         s.expect_indent()
         attributes = []
+        body_ctx = Ctx()
         while s.sy != 'DEDENT':
             if s.sy != 'pass':
                 attributes.append(
-                    p_c_func_or_var_declaration(s, level = 'other', pos = s.position()))
+                    p_c_func_or_var_declaration(s, s.position(), body_ctx))
             else:
                 s.next()
                 s.expect_newline("Expected a newline")
@@ -1894,8 +1900,8 @@ def p_c_struct_or_union_definition(s, pos, level, visibility, typedef_flag = 0):
         s.expect_newline("Syntax error in struct or union definition")
     return Nodes.CStructOrUnionDefNode(pos, 
         name = name, cname = cname, kind = kind, attributes = attributes,
-        typedef_flag = typedef_flag, visibility = visibility,
-        in_pxd = level == 'module_pxd')
+        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
+        in_pxd = ctx.level == 'module_pxd')
 
 def p_visibility(s, prev_visibility):
     pos = s.position()
@@ -1915,26 +1921,26 @@ def p_c_modifiers(s):
         return [modifier] + p_c_modifiers(s)
     return []
 
-def p_c_func_or_var_declaration(s, level, pos, visibility = 'private', api = 0,
-                                overridable = False):
-    cmethod_flag = level in ('c_class', 'c_class_pxd')
+def p_c_func_or_var_declaration(s, pos, ctx):
+    cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
     modifiers = p_c_modifiers(s)
     base_type = p_c_base_type(s, nonempty = 1)
-    declarator = p_c_declarator(s, cmethod_flag = cmethod_flag, assignable = 1, nonempty = 1)
-    declarator.overridable = overridable
+    declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
+                                assignable = 1, nonempty = 1)
+    declarator.overridable = ctx.overridable
     if s.sy == ':':
-        if level not in ('module', 'c_class'):
+        if ctx.level not in ('module', 'c_class'):
             s.error("C function definition not allowed here")
-        doc, suite = p_suite(s, 'function', with_doc = 1)
+        doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
         result = Nodes.CFuncDefNode(pos,
-            visibility = visibility,
+            visibility = ctx.visibility,
             base_type = base_type,
             declarator = declarator, 
             body = suite,
             doc = doc,
             modifiers = modifiers,
-            api = api,
-            overridable = overridable)
+            api = ctx.api,
+            overridable = ctx.overridable)
     else:
         #if api:
         #    error(s.pos, "'api' not allowed with variable declaration")
@@ -1943,39 +1949,40 @@ def p_c_func_or_var_declaration(s, level, pos, visibility = 'private', api = 0,
             s.next()
             if s.sy == 'NEWLINE':
                 break
-            declarator = p_c_declarator(s, cmethod_flag = cmethod_flag, assignable = 1, nonempty = 1)
+            declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
+                                        assignable = 1, nonempty = 1)
             declarators.append(declarator)
         s.expect_newline("Syntax error in C variable declaration")
         result = Nodes.CVarDefNode(pos, 
-            visibility = visibility,
-            base_type = base_type, 
+            visibility = ctx.visibility,
+            base_type = base_type,
             declarators = declarators,
-            in_pxd = level == 'module_pxd',
-            api = api,
-            overridable = overridable)
+            in_pxd = ctx.level == 'module_pxd',
+            api = ctx.api,
+            overridable = ctx.overridable)
     return result
 
-def p_ctypedef_statement(s, level, visibility = 'private', api = 0):
+def p_ctypedef_statement(s, ctx):
     # s.sy == 'ctypedef'
     pos = s.position()
     s.next()
-    visibility = p_visibility(s, visibility)
+    visibility = p_visibility(s, ctx.visibility)
+    ctx = ctx(typedef_flag = 1, visibility = visibility)
     if s.sy == 'class':
-        return p_c_class_definition(s, level, pos,
-            visibility = visibility, typedef_flag = 1, api = api)
+        return p_c_class_definition(s, pos, ctx)
     elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'):
         if s.systring == 'enum':
-            return p_c_enum_definition(s, pos, level, visibility, typedef_flag = 1)
+            return p_c_enum_definition(s, pos, ctx)
         else:
-            return p_c_struct_or_union_definition(s, pos, level, visibility,
-                typedef_flag = 1)
+            return p_c_struct_or_union_definition(s, pos, ctx)
     else:
         base_type = p_c_base_type(s, nonempty = 1)
-        declarator = p_c_declarator(s, is_type = 1, nonempty = 1)
+        declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
         s.expect_newline("Syntax error in ctypedef statement")
-        return Nodes.CTypeDefNode(pos,
-            base_type = base_type, declarator = declarator, visibility = visibility,
-            in_pxd = level == 'module_pxd')
+        return Nodes.CTypeDefNode(
+            pos, base_type = base_type,
+            declarator = declarator, visibility = visibility,
+            in_pxd = ctx.level == 'module_pxd')
 
 def p_def_statement(s):
     # s.sy == 'def'
@@ -2003,7 +2010,7 @@ def p_def_statement(s):
     s.expect(')')
     if p_nogil(s):
         error(s.pos, "Python function cannot be declared nogil")
-    doc, body = p_suite(s, 'function', with_doc = 1)
+    doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
     return Nodes.DefNode(pos, name = name, args = args, 
         star_arg = star_arg, starstar_arg = starstar_arg,
         doc = doc, body = body)
@@ -2025,14 +2032,13 @@ def p_class_statement(s):
         s.expect(')')
     else:
         base_list = []
-    doc, body = p_suite(s, 'class', with_doc = 1)
+    doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)
     return Nodes.PyClassDefNode(pos,
         name = class_name,
         bases = ExprNodes.TupleNode(pos, args = base_list),
         doc = doc, body = body)
 
-def p_c_class_definition(s, level, pos, 
-        visibility = 'private', typedef_flag = 0, api = 0):
+def p_c_class_definition(s, pos,  ctx):
     # s.sy == 'class'
     s.next()
     module_path = []
@@ -2041,7 +2047,7 @@ def p_c_class_definition(s, level, pos,
         s.next()
         module_path.append(class_name)
         class_name = p_ident(s)
-    if module_path and visibility != 'extern':
+    if module_path and ctx.visibility != 'extern':
         error(pos, "Qualified class name only allowed for 'extern' C class")
     if module_path and s.sy == 'IDENT' and s.systring == 'as':
         s.next()
@@ -2065,38 +2071,38 @@ def p_c_class_definition(s, level, pos,
         base_class_module = ".".join(base_class_path[:-1])
         base_class_name = base_class_path[-1]
     if s.sy == '[':
-        if visibility not in ('public', 'extern'):
+        if ctx.visibility not in ('public', 'extern'):
             error(s.position(), "Name options only allowed for 'public' or 'extern' C class")
         objstruct_name, typeobj_name = p_c_class_options(s)
     if s.sy == ':':
-        if level == 'module_pxd':
+        if ctx.level == 'module_pxd':
             body_level = 'c_class_pxd'
         else:
             body_level = 'c_class'
-        doc, body = p_suite(s, body_level, with_doc = 1)
+        doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)
     else:
         s.expect_newline("Syntax error in C class definition")
         doc = None
         body = None
-    if visibility == 'extern':
+    if ctx.visibility == 'extern':
         if not module_path:
             error(pos, "Module name required for 'extern' C class")
         if typeobj_name:
             error(pos, "Type object name specification not allowed for 'extern' C class")
-    elif visibility == 'public':
+    elif ctx.visibility == 'public':
         if not objstruct_name:
             error(pos, "Object struct name specification required for 'public' C class")
         if not typeobj_name:
             error(pos, "Type object name specification required for 'public' C class")
-    elif visibility == 'private':
-        if api:
+    elif ctx.visibility == 'private':
+        if ctx.api:
             error(pos, "Only 'public' C class can be declared 'api'")
     else:
-        error(pos, "Invalid class visibility '%s'" % visibility)
+        error(pos, "Invalid class visibility '%s'" % ctx.visibility)
     return Nodes.CClassDefNode(pos,
-        visibility = visibility,
-        typedef_flag = typedef_flag,
-        api = api,
+        visibility = ctx.visibility,
+        typedef_flag = ctx.typedef_flag,
+        api = ctx.api,
         module_name = ".".join(module_path),
         class_name = class_name,
         as_name = as_name,
@@ -2104,7 +2110,7 @@ def p_c_class_definition(s, level, pos,
         base_class_name = base_class_name,
         objstruct_name = objstruct_name,
         typeobj_name = typeobj_name,
-        in_pxd = level == 'module_pxd',
+        in_pxd = ctx.level == 'module_pxd',
         doc = doc,
         body = body)
 
@@ -2131,7 +2137,7 @@ def p_property_decl(s):
     pos = s.position()
     s.next() # 'property'
     name = p_ident(s)
-    doc, body = p_suite(s, 'property', with_doc = 1)
+    doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)
     return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)
 
 def p_doc_string(s):
@@ -2152,7 +2158,7 @@ def p_module(s, pxd, full_module_name):
         level = 'module_pxd'
     else:
         level = 'module'
-    body = p_statement_list(s, level, first_statement = 1)
+    body = p_statement_list(s, Ctx(level = level), first_statement = 1)
     if s.sy != 'EOF':
         s.error("Syntax error in statement [%s,%s]" % (
             repr(s.sy), repr(s.systring)))
@@ -2164,7 +2170,7 @@ def p_module(s, pxd, full_module_name):
 #
 #----------------------------------------------
 
-def print_parse_tree(f, node, level, key = None):	
+def print_parse_tree(f, node, level, key = None):
     from Nodes import Node
     ind = "  " * level
     if node:

Cython/Compiler/PyrexTypes.py

@@ -695,9 +695,10 @@ class CFuncType(CType):
             return 0
         if not self.same_calling_convention_as(other_type):
             return 0
+        if self.nogil and not other_type.nogil:
+            return 0
         return 1
-        
-    
+
     def compatible_signature_with(self, other_type, as_cmethod = 0):
         return self.compatible_signature_with_resolved_type(other_type.resolve(), as_cmethod)
     
@@ -789,22 +790,24 @@ class CFuncType(CType):
         arg_decl_code = ", ".join(arg_decl_list)
         if not arg_decl_code and not pyrex:
             arg_decl_code = "void"
-        exc_clause = ""
+        trailer = ""
         if (pyrex or for_display) and not self.return_type.is_pyobject:
             if self.exception_value and self.exception_check:
-                exc_clause = " except? %s" % self.exception_value
+                trailer = " except? %s" % self.exception_value
             elif self.exception_value:
-                exc_clause = " except %s" % self.exception_value
+                trailer = " except %s" % self.exception_value
             elif self.exception_check == '+':
-                exc_clause = " except +"
+                trailer = " except +"
             else:
-                " except *"
+                " except *" # ignored
+            if self.nogil:
+                trailer += " nogil"
         cc = self.calling_convention_prefix()
         if (not entity_code and cc) or entity_code.startswith("*"):
             entity_code = "(%s%s)" % (cc, entity_code)
             cc = ""
         return self.return_type.declaration_code(
-            "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, exc_clause),
+            "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, trailer),
             for_display, dll_linkage, pyrex)
 	
     def function_header_code(self, func_name, arg_code):

Cython/Compiler/Scanning.py

@@ -284,11 +284,10 @@ class PyrexScanner(Scanner):
     #  compile_time_env   dict     Environment for conditional compilation
     #  compile_time_eval  boolean  In a true conditional compilation context
     #  compile_time_expr  boolean  In a compile-time expression context
-    
     resword_dict = build_resword_dict()
 
     def __init__(self, file, filename, parent_scanner = None, 
-            type_names = None, context = None, source_encoding=None):
+                 type_names = None, context = None, source_encoding=None):
         Scanner.__init__(self, get_lexicon(), file, filename)
         if parent_scanner:
             self.context = parent_scanner.context

Cython/Compiler/Symtab.py

@@ -150,12 +150,14 @@ class Scope:
     # pystring_entries  [Entry]            String const entries newly used as
     #                                        Python strings in this scope
     # control_flow     ControlFlow  Used for keeping track of environment state
+    # nogil             boolean            In a nogil section
 
     is_py_class_scope = 0
     is_c_class_scope = 0
     is_module_scope = 0
     scope_prefix = ""
     in_cinclude = 0
+    nogil = 0
     
     def __init__(self, name, outer_scope, parent_scope):
         # The outer_scope is the next scope in the lookup chain.
@@ -554,10 +556,6 @@ class Scope:
         return [entry for entry in self.temp_entries
             if entry not in self.free_temp_entries]
     
-    #def recycle_pending_temps(self):
-    #	# Obsolete
-    #	pass
-
     def use_utility_code(self, new_code):
         self.global_scope().use_utility_code(new_code)
     
@@ -652,7 +650,7 @@ class BuiltinScope(Scope):
         "long":   ["((PyObject*)&PyLong_Type)", py_object_type],
         "float":  ["((PyObject*)&PyFloat_Type)", py_object_type],
         
-        "str":    ["((PyObject*)&PyString_Type)", py_object_type],
+        "str":    ["((PyObject*)&PyBytes_Type)", py_object_type],
         "unicode":["((PyObject*)&PyUnicode_Type)", py_object_type],
         "tuple":  ["((PyObject*)&PyTuple_Type)", py_object_type],
         "list":   ["((PyObject*)&PyList_Type)", py_object_type],
@@ -687,6 +685,7 @@ class ModuleScope(Scope):
     # parent_module        Scope              Parent in the import namespace
     # module_entries       {string : Entry}   For cimport statements
     # type_names           {string : 1}       Set of type names (used during parsing)
+    # included_files       [string]           Cython sources included with 'include'
     # pxd_file_loaded      boolean            Corresponding .pxd file has been processed
     # cimported_modules    [ModuleScope]      Modules imported with cimport
     # new_interned_string_entries [Entry]     New interned strings waiting to be declared
@@ -723,6 +722,7 @@ class ModuleScope(Scope):
         self.interned_objs = []
         self.all_pystring_entries = []
         self.types_imported = {}
+        self.included_files = []
         self.pynum_entries = []
         self.has_extern_class = 0
         self.cached_builtins = []
@@ -1326,6 +1326,7 @@ class CClassScope(ClassScope):
 #                    entry.type = type
                 else:
                     error(pos, "Signature not compatible with previous declaration")
+                    error(entry.pos, "Previous declaration is here")
         else:
             if self.defined:
                 error(pos,
@@ -1365,8 +1366,8 @@ class CClassScope(ClassScope):
                 entry.is_variable = 1
                 self.inherited_var_entries.append(entry)
         for base_entry in base_scope.cfunc_entries:
-            entry = self.add_cfunction(base_entry.name, base_entry.type, None,
-                adapt(base_entry.cname), base_entry.visibility)
+            entry = self.add_cfunction(base_entry.name, base_entry.type,
+                    base_entry.pos, adapt(base_entry.cname), base_entry.visibility)
             entry.is_inherited = 1
             
     def allocate_temp(self, type):

Cython/Compiler/TypeSlots.py

@@ -177,8 +177,8 @@ class FixedSlot(SlotDescriptor):
     #
     #  value        string
     
-    def __init__(self, slot_name, value):
-        SlotDescriptor.__init__(self, slot_name)
+    def __init__(self, slot_name, value, py3k=True):
+        SlotDescriptor.__init__(self, slot_name, py3k=py3k)
         self.value = value
     
     def slot_code(self, scope):
@@ -188,8 +188,8 @@ class FixedSlot(SlotDescriptor):
 class EmptySlot(FixedSlot):
     #  Descriptor for a type slot whose value is always 0.
     
-    def __init__(self, slot_name):
-        FixedSlot.__init__(self, slot_name, "0")
+    def __init__(self, slot_name, py3k=True):
+        FixedSlot.__init__(self, slot_name, "0", py3k=py3k)
 
 
 class MethodSlot(SlotDescriptor):
@@ -553,12 +553,12 @@ PyNumberMethods = (
     MethodSlot(binaryfunc, "nb_and", "__and__"),
     MethodSlot(binaryfunc, "nb_xor", "__xor__"),
     MethodSlot(binaryfunc, "nb_or", "__or__"),
-    EmptySlot("nb_coerce"),
+    EmptySlot("nb_coerce", py3k = False),
     MethodSlot(unaryfunc, "nb_int", "__int__"),
     MethodSlot(unaryfunc, "nb_long", "__long__"),
     MethodSlot(unaryfunc, "nb_float", "__float__"),
-    MethodSlot(unaryfunc, "nb_oct", "__oct__"),
-    MethodSlot(unaryfunc, "nb_hex", "__hex__"),
+    MethodSlot(unaryfunc, "nb_oct", "__oct__", py3k = False),
+    MethodSlot(unaryfunc, "nb_hex", "__hex__", py3k = False),
     
     # Added in release 2.0
     MethodSlot(ibinaryfunc, "nb_inplace_add", "__iadd__"),

Cython/Utils.py

@@ -24,7 +24,6 @@ def castrate_file(path, st):
     except EnvironmentError:
         pass
     else:
-        #st = os.stat(path)
         f.seek(0, 0)
         f.truncate()
         f.write(
@@ -33,6 +32,14 @@ def castrate_file(path, st):
         if st:
             os.utime(path, (st.st_atime, st.st_mtime))
 
+def modification_time(path):
+    st = os.stat(path)
+    return st.st_mtime
+
+def file_newer_than(path, time):
+    ftime = modification_time(path)
+    return ftime > time
+
 # support for source file encoding detection and unicode decoding
 
 def encode_filename(filename):

runtests.py

@@ -17,7 +17,7 @@ CFLAGS = os.getenv('CFLAGS', '').split()
 
 
 class ErrorWriter(object):
-    match_error = re.compile('(warning:)?(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
+    match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
     def __init__(self):
         self.output = []
         self.write = self.output.append
@@ -31,8 +31,9 @@ class ErrorWriter(object):
                 is_warning, line, column, message = match.groups()
                 if (is_warning and collect_warnings) or \
                         (not is_warning and collect_errors):
-                    result.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
-        return result
+                    result.append( (int(line), int(column), message.strip()) )
+        result.sort()
+        return [ "%d:%d: %s" % values for values in result ]
 
     def geterrors(self):
         return self._collect(True, False)

tests/compile/extcoerce.pyx

@@ -7,7 +7,7 @@ cdef class Swallow:
     pass
     
 def f(Grail g):
-    cdef int i
+    cdef int i = 0
     cdef Swallow s
     g = x
     x = g

tests/compile/nogil.h

+void e1(void);
+void *e2(void);

tests/compile/nogil.pyx

-cdef extern void g(int x) nogil
+cdef extern object g(object x) nogil
+cdef extern void g2(object x) nogil
+
+cdef extern from "nogil.h":
+        void e1() nogil
+        int *e2() nogil
 
 cdef void f(int x) nogil:
-    cdef int y
-    y = 42
+        cdef int y
+        y = 42
+
+cdef void h(object x) nogil:
+        cdef void *p
+        g2(x)
+        g2(<object>p)
+        p = <void *>x
+        e1()
+        e2()

tests/compile/while.pyx

 def f(a, b):
-    cdef int i
+    cdef int i = 5
     
     while a:
         x = 1
@@ -23,4 +23,4 @@ def f(a, b):
         x = 1
         continue
         x = 2
-    
\ No newline at end of file
+    

tests/compile/withgil.pyx

@@ -3,3 +3,6 @@ cdef void f() with gil:
 
 cdef int g(void* x) with gil:
     pass
+
+f()
+g("test")

tests/errors/cmethbasematch.pyx

+cdef class C:
+    cdef void f(self):
+        pass
+
+cdef class D(C):
+    cdef void f(self, int x):
+        pass
+
+_ERRORS = u"""
+6: 9: Signature not compatible with previous declaration
+2: 9: Previous declaration is here
+"""

tests/errors/nogil.pyx

+cdef object f(object x) nogil:
+    pass
+
+cdef void g(int x) nogil:
+    cdef object z
+    z = None
+
+cdef void h(int x) nogil:
+    p()
+
+cdef object p() nogil:
+    pass
+
+cdef void r() nogil:
+    q()
+    
+cdef object m():
+    cdef object x, y, obj
+    cdef int i, j, k
+    global fred
+    q()
+    with nogil:
+        r()
+        q()
+        i = 42
+        obj = None
+        17L
+        7j
+        help
+        `"Hello"`
+        import fred
+        from fred import obj
+        for x in obj:
+            pass
+        obj[i]
+        obj[i:j]
+        obj[i:j:k]
+        obj.fred
+        (x, y)
+        [x, y]
+        {x: y}
+        obj and x
+        t(obj)
+#        f(42) # Cython handles this internally
+        x + obj
+        -obj
+        x = y = obj
+        x, y = y, x
+        obj[i] = x
+        obj.fred = x
+        print obj
+        del fred
+        return obj
+        raise obj
+        if obj:
+            pass
+        while obj:
+            pass
+        for x <= obj <= y:
+            pass
+        try:
+            pass
+        except:
+            pass
+        try:
+            pass
+        finally:
+            pass
+
+cdef void q():
+    pass
+
+cdef class C:
+    pass
+
+cdef void t(C c) nogil:
+    pass
+
+
+_ERRORS = u"""
+ 1: 5: Function with Python return type cannot be declared nogil
+ 6: 6: Assignment of Python object not allowed without gil
+ 4: 5: Function declared nogil has Python locals or temporaries
+ 8: 5: Function declared nogil has Python locals or temporaries
+11: 5: Function with Python return type cannot be declared nogil
+15: 5: Calling gil-requiring function without gil
+24: 9: Calling gil-requiring function without gil
+26:12: Assignment of Python object not allowed without gil
+27: 8: Constructing Python long int not allowed without gil
+28: 8: Constructing complex number not allowed without gil
+29:12: Accessing Python global or builtin not allowed without gil
+30: 8: Backquote expression not allowed without gil
+31:15: Python import not allowed without gil
+31:15: Assignment of Python object not allowed without gil
+32:13: Python import not allowed without gil
+32:25: Constructing Python list not allowed without gil
+33:17: Iterating over Python object not allowed without gil
+35:11: Indexing Python object not allowed without gil
+36:11: Slicing Python object not allowed without gil
+37:11: Constructing Python slice object not allowed without gil
+37:11: Indexing Python object not allowed without gil
+37:13: Converting to Python object not allowed without gil
+37:15: Converting to Python object not allowed without gil
+37:17: Converting to Python object not allowed without gil
+38:11: Accessing Python attribute not allowed without gil
+39: 9: Constructing Python tuple not allowed without gil
+40: 8: Constructing Python list not allowed without gil
+41: 8: Constructing Python dict not allowed without gil
+42:12: Creating temporary Python reference not allowed without gil
+42:12: Truth-testing Python object not allowed without gil
+42:17: Creating temporary Python reference not allowed without gil
+43:13: Python type test not allowed without gil
+#44: 4: Converting to Python object not allowed without gil
+45:10: Operation not allowed without gil
+46: 8: Operation not allowed without gil
+47:10: Assignment of Python object not allowed without gil
+47:14: Assignment of Python object not allowed without gil
+48: 9: Assignment of Python object not allowed without gil
+48:13: Assignment of Python object not allowed without gil
+48:16: Creating temporary Python reference not allowed without gil
+48:19: Creating temporary Python reference not allowed without gil
+49:11: Indexing Python object not allowed without gil
+49:11: Assignment of Python object not allowed without gil
+50:11: Accessing Python attribute not allowed without gil
+50:11: Assignment of Python object not allowed without gil
+51: 8: Constructing Python tuple not allowed without gil
+51: 8: Python print statement not allowed without gil
+52: 8: Deleting Python object not allowed without gil
+53: 8: Returning Python object not allowed without gil
+54: 8: Raising exception not allowed without gil
+55:14: Truth-testing Python object not allowed without gil
+57:17: Truth-testing Python object not allowed without gil
+59: 8: Converting to Python object not allowed without gil
+61: 8: Try-except statement not allowed without gil
+65: 8: Try-finally statement not allowed without gil
+"""

tests/errors/nogilfunctype.pyx

+cdef extern from *:
+    cdef void f()
+    cdef void (*fp)() nogil
+
+    cdef void g() nogil
+    cdef void (*gp)()
+
+gp = g
+
+fp = f
+
+_ERRORS = u"""
+10:6: Cannot assign type 'void (void)' to 'void (*)(void) nogil'
+"""

tests/run/big_indices.pyx

 __doc__ = u"""
     >>> test()
-    neg -1
-    pos 4294967294
+    neg False
+    pos True
     neg
     pos
     neg
@@ -12,8 +12,8 @@ def test():
     cdef long neg = -1
     cdef unsigned long pos = -2 # will be a large positive number
 
-    print "neg", neg
-    print "pos", pos
+    print "neg", neg > 0
+    print "pos", pos > -
 
     D = { neg: 'neg', pos: 'pos' }
 

tests/run/exceptionpropagation.pyx

+__doc__ = u"""
+>>> foo(0)
+>>> foo(1)
+Traceback (most recent call last):
+RuntimeError
+"""
+
+cdef int CHKERR(int ierr) except -1:
+    if ierr==0: return 0
+    raise RuntimeError
+
+cdef int obj2int(object ob) except *:
+    return ob
+
+def foo(a):
+    cdef int i = obj2int(a)
+    CHKERR(i)

tests/run/inplace.pyx

 __doc__ = u"""
     >>> f(5, 7)
-    29509034655744
+    29509034655744L
 
     >>> g(13, 4)
     32
@@ -9,6 +9,10 @@ __doc__ = u"""
     105.0
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"L", u"")
+
 def f(a,b):
     a += b
     a *= b

tests/run/modop.pyx

@@ -6,6 +6,9 @@ __doc__ = u"""
 
     >>> modint(9,2)
     1
+
+    >>> print modptr()
+    spameggs
 """
 
 def modobj(obj2, obj3):
@@ -17,11 +20,10 @@ def modint(int int2, int int3):
     int1 = int2 % int3
     return int1
 
-cdef modptr():
-    # FIXME!!!
+def modptr():
     cdef char *str2, *str3
-    str2 = "spam"
+    str2 = "spam%s"
     str3 = "eggs"
 
     obj1 = str2 % str3
-    return obj1
+    return obj1.decode(u"ASCII")

tests/compile/varargdecl.pyx → tests/run/varargdecl.pyx

+__doc__ = u"""
+>>> test()
+"""
+
 cdef grail(char *blarg, ...):
     pass
+
+def test():
+    grail("test")
+    grail("test", "toast")

tests/run/withnogil.pyx

@@ -18,5 +18,5 @@ def g():
         h()
     return 1
 
-cdef int h() except -1:
+cdef int h() nogil except -1:
     pass