merged (and partially rewrote) dependency tracking and package resolution changes from Pyrex 0.9.8

Cython/Compiler/CmdLine.py

@@ -18,6 +18,10 @@ Options:
   -I, --include-dir <directory>  Search for include files in named directory
                                  (multiply include directories are allowed).
   -o, --output-file <filename>   Specify name of generated C file
+  -r, --recursive                Recursively find and compile dependencies
+  -t, --timestamps               Only compile newer source files (implied with -r)
+  -f, --force                    Compile all source files (overrides implied -t)
+  -q, --quiet                    Don't print module names in recursive mode
   -p, --embed-positions          If specified, the positions in Cython files of each
                                  function definition is embedded in its docstring.
   -z, --pre-import <module>      If specified, assume undeclared names in this 
@@ -111,6 +115,12 @@ def parse_command_line(args):
                 options.working_path = pop_arg()
             elif option in ("-o", "--output-file"):
                 options.output_file = pop_arg()
+            elif option in ("-r", "--recursive"):
+                options.recursive = 1
+            elif option in ("-t", "--timestamps"):
+                options.timestamps = 1
+            elif option in ("-f", "--force"):
+                options.timestamps = 0
             elif option in ("-p", "--embed-positions"):
                 Options.embed_pos_in_docstring = 1
             elif option in ("-z", "--pre-import"):

Cython/Compiler/ModuleNode.py

@@ -54,6 +54,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if self.has_imported_c_functions():
             self.module_temp_cname = env.allocate_temp_pyobject()
             env.release_temp(self.module_temp_cname)
+        self.generate_dep_file(env, result)
         self.generate_c_code(env, options, result)
         self.generate_h_code(env, options, result)
         self.generate_api_code(env, result)
@@ -65,6 +66,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     return 1
         return 0
     
+    def generate_dep_file(self, env, result):
+        modules = self.referenced_modules
+        if len(modules) > 1 or env.included_files:
+            dep_file = replace_suffix(result.c_file, ".dep")
+            f = open(dep_file, "w")
+            try:
+                for module in modules:
+                    if module is not env:
+                        f.write("cimport %s\n" % module.qualified_name)
+                    for path in module.included_files:
+                        f.write("include %s\n" % path)
+            finally:
+                f.close()
+
     def generate_h_code(self, env, options, result):
         def h_entries(entries, pxd = 0):
             return [entry for entry in entries

Cython/Compiler/Parsing.py

@@ -1203,8 +1203,10 @@ def p_include_statement(s, level):
     if s.compile_time_eval:
         include_file_path = s.context.find_include_file(include_file_name, pos)
         if include_file_path:
+            s.included_files.append(include_file_name)
             f = Utils.open_source_file(include_file_path, mode="rU")
-            s2 = PyrexScanner(f, include_file_path, s, source_encoding=f.encoding)
+            s2 = PyrexScanner(f, include_file_path, parent_scanner = s,
+                              source_encoding=f.encoding)
             try:
                 tree = p_statement_list(s2, level)
             finally:

Cython/Compiler/Scanning.py

@@ -206,24 +206,26 @@ def initial_compile_time_env():
 class PyrexScanner(Scanner):
     #  context            Context  Compilation context
     #  type_names         set      Identifiers to be treated as type names
+    #  included_files     [string] Files included with 'include' statement
     #  compile_time_env   dict     Environment for conditional compilation
     #  compile_time_eval  boolean  In a true conditional compilation context
     #  compile_time_expr  boolean  In a compile-time expression context
-    
     resword_dict = build_resword_dict()
 
     def __init__(self, file, filename, parent_scanner = None, 
-            type_names = None, context = None, source_encoding=None):
+                 scope = None, context = None, source_encoding=None):
         Scanner.__init__(self, get_lexicon(), file, filename)
         if parent_scanner:
             self.context = parent_scanner.context
             self.type_names = parent_scanner.type_names
+            self.included_files = parent_scanner.included_files
             self.compile_time_env = parent_scanner.compile_time_env
             self.compile_time_eval = parent_scanner.compile_time_eval
             self.compile_time_expr = parent_scanner.compile_time_expr
         else:
             self.context = context
-            self.type_names = type_names
+            self.type_names = scope.type_names
+            self.included_files = scope.included_files
             self.compile_time_env = initial_compile_time_env()
             self.compile_time_eval = 1
             self.compile_time_expr = 0

Cython/Compiler/Symtab.py

@@ -554,10 +554,6 @@ class Scope:
         return [entry for entry in self.temp_entries
             if entry not in self.free_temp_entries]
     
-    #def recycle_pending_temps(self):
-    #	# Obsolete
-    #	pass
-
     def use_utility_code(self, new_code):
         self.global_scope().use_utility_code(new_code)
     
@@ -687,6 +683,7 @@ class ModuleScope(Scope):
     # parent_module        Scope              Parent in the import namespace
     # module_entries       {string : Entry}   For cimport statements
     # type_names           {string : 1}       Set of type names (used during parsing)
+    # included_files       [string]           Cython sources included with 'include'
     # pxd_file_loaded      boolean            Corresponding .pxd file has been processed
     # cimported_modules    [ModuleScope]      Modules imported with cimport
     # new_interned_string_entries [Entry]     New interned strings waiting to be declared
@@ -723,6 +720,7 @@ class ModuleScope(Scope):
         self.interned_objs = []
         self.all_pystring_entries = []
         self.types_imported = {}
+        self.included_files = []
         self.pynum_entries = []
         self.has_extern_class = 0
         self.cached_builtins = []

Cython/Utils.py

@@ -24,7 +24,6 @@ def castrate_file(path, st):
     except EnvironmentError:
         pass
     else:
-        #st = os.stat(path)
         f.seek(0, 0)
         f.truncate()
         f.write(
@@ -33,6 +32,14 @@ def castrate_file(path, st):
         if st:
             os.utime(path, (st.st_atime, st.st_mtime))
 
+def modification_time(path):
+    st = os.stat(path)
+    return st.st_mtime
+
+def file_newer_than(path, time):
+    ftime = modification_time(path)
+    return ftime > time
+
 # support for source file encoding detection and unicode decoding
 
 def encode_filename(filename):