merged (and partially rewrote) dependency tracking and package resolution changes from Pyrex 0.9.8

Cython/Compiler/CmdLine.py

@@ -18,6 +18,10 @@ Options:
   -I, --include-dir <directory>  Search for include files in named directory
                                  (multiply include directories are allowed).
   -o, --output-file <filename>   Specify name of generated C file
+  -r, --recursive                Recursively find and compile dependencies
+  -t, --timestamps               Only compile newer source files (implied with -r)
+  -f, --force                    Compile all source files (overrides implied -t)
+  -q, --quiet                    Don't print module names in recursive mode
   -p, --embed-positions          If specified, the positions in Cython files of each
                                  function definition is embedded in its docstring.
   -z, --pre-import <module>      If specified, assume undeclared names in this 
@@ -111,6 +115,12 @@ def parse_command_line(args):
                 options.working_path = pop_arg()
             elif option in ("-o", "--output-file"):
                 options.output_file = pop_arg()
+            elif option in ("-r", "--recursive"):
+                options.recursive = 1
+            elif option in ("-t", "--timestamps"):
+                options.timestamps = 1
+            elif option in ("-f", "--force"):
+                options.timestamps = 0
             elif option in ("-p", "--embed-positions"):
                 Options.embed_pos_in_docstring = 1
             elif option in ("-z", "--pre-import"):

Cython/Compiler/Main.py

@@ -3,8 +3,8 @@
 #
 
 import os, sys, re, codecs
-if sys.version_info[:2] < (2, 2):
-    sys.stderr.write("Sorry, Cython requires Python 2.2 or later\n")
+if sys.version_info[:2] < (2, 3):
+    sys.stderr.write("Sorry, Cython requires Python 2.3 or later\n")
     sys.exit(1)
 
 try:
@@ -14,14 +14,13 @@ except NameError:
     from sets import Set as set
 
 from time import time
-import Version
-from Scanning import PyrexScanner
+import Code
 import Errors
-from Errors import PyrexError, CompileError, error
 import Parsing
+import Version
+from Errors import PyrexError, CompileError, error
+from Scanning import PyrexScanner
 from Symtab import BuiltinScope, ModuleScope
-import Code
-from Cython.Utils import replace_suffix
 from Cython import Utils
 import Transform
 
@@ -93,31 +92,34 @@ class Context:
                 try:
                     if debug_find_module:
                         print("Context.find_module: Parsing %s" % pxd_pathname)
-                    pxd_tree = self.parse(pxd_pathname, scope.type_names, pxd = 1,
+                    pxd_tree = self.parse(pxd_pathname, scope, pxd = 1,
                                           full_module_name = module_name)
                     pxd_tree.analyse_declarations(scope)
                 except CompileError:
                     pass
         return scope
     
-    def find_pxd_file(self, module_name, pos):
-        # Search include directories for the .pxd file
-        # corresponding to the given (full) module name.
-        if "." in module_name:
-            pxd_filename = "%s.pxd" % os.path.join(*module_name.split('.'))
-        else:
-            pxd_filename = "%s.pxd" % module_name
-        return self.search_include_directories(pxd_filename, pos)
+    def find_pxd_file(self, qualified_name, pos):
+        # Search include path for the .pxd file corresponding to the
+        # given fully-qualified module name.
+        return self.search_include_directories(qualified_name, ".pxd", pos)
+
+    def find_pyx_file(self, qualified_name, pos):
+        # Search include path for the .pyx file corresponding to the
+        # given fully-qualified module name, as for find_pxd_file().
+        return self.search_include_directories(qualified_name, ".pyx", pos)
     
     def find_include_file(self, filename, pos):
         # Search list of include directories for filename.
         # Reports an error and returns None if not found.
-        path = self.search_include_directories(filename, pos)
+        path = self.search_include_directories(filename, "", pos,
+                                               split_package=False)
         if not path:
             error(pos, "'%s' not found" % filename)
         return path
     
-    def search_include_directories(self, filename, pos):
+    def search_include_directories(self, qualified_name, suffix, pos,
+                                   split_package=True):
         # Search the list of include directories for the given
         # file name. If a source file position is given, first
         # searches the directory containing that file. Returns
@@ -126,12 +128,81 @@ class Context:
         if pos:
             here_dir = os.path.dirname(pos[0])
             dirs = [here_dir] + dirs
+
+        dotted_filename = qualified_name + suffix
+        if split_package:
+            names = qualified_name.split('.')
+            package_names = names[:-1]
+            module_name = names[-1]
+            module_filename = module_name + suffix
+            package_filename = "__init__" + suffix
+
         for dir in dirs:
-            path = os.path.join(dir, filename)
+            path = os.path.join(dir, dotted_filename)
             if os.path.exists(path):
                 return path
+            if split_package:
+                package_dir = self.check_package_dir(dir, package_names)
+                if package_dir is not None:
+                    path = os.path.join(package_dir, module_filename)
+                    if os.path.exists(path):
+                        return path
+                    path = os.path.join(dir, package_dir, module_name,
+                                        package_filename)
+                    if os.path.exists(path):
+                        return path
         return None
 
+    def check_package_dir(self, dir, package_names):
+        package_dir = os.path.join(dir, *package_names)
+        if not os.path.exists(package_dir):
+            return None
+        for dirname in package_names:
+            dir = os.path.join(dir, dirname)
+            package_init = os.path.join(dir, "__init__.py")
+            if not os.path.exists(package_init) and \
+                    not os.path.exists(package_init + "x"): # same with .pyx ?
+                return None
+        return package_dir
+
+    def c_file_out_of_date(self, source_path):
+        c_path = Utils.replace_suffix(source_path, ".c")
+        if not os.path.exists(c_path):
+            return 1
+        c_time = Utils.modification_time(c_path)
+        if Utils.file_newer_than(source_path, c_time):
+            return 1
+        pos = [source_path]
+        pxd_path = Utils.replace_suffix(source_path, ".pxd")
+        if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time):
+            return 1
+        for kind, name in self.read_dependency_file(source_path):
+            if kind == "cimport":
+                dep_path = self.find_pxd_file(name, pos)
+            elif kind == "include":
+                dep_path = self.search_include_directories(name, pos)
+            else:
+                continue
+            if dep_path and Utils.file_newer_than(dep_path, c_time):
+                return 1
+        return 0
+    
+    def find_cimported_module_names(self, source_path):
+        return [ name for kind, name in self.read_dependency_file(source_path)
+                 if kind == "cimport" ]
+    
+    def read_dependency_file(self, source_path):
+        dep_path = replace_suffix(source_path, ".dep")
+        if os.path.exists(dep_path):
+            f = open(dep_path, "rU")
+            chunks = [ line.strip().split(" ", 1)
+                       for line in f.readlines()
+                       if " " in line.strip() ]
+            f.close()
+            return chunks
+        else:
+            return ()
+
     def lookup_submodule(self, name):
         # Look up a top-level module. Returns None if not found.
         return self.modules.get(name, None)
@@ -145,14 +216,14 @@ class Context:
             self.modules[name] = scope
         return scope
 
-    def parse(self, source_filename, type_names, pxd, full_module_name):
+    def parse(self, source_filename, scope, pxd, full_module_name):
         name = Utils.encode_filename(source_filename)
         # Parse the given source file and return a parse tree.
         try:
             f = Utils.open_source_file(source_filename, "rU")
             try:
                 s = PyrexScanner(f, name, source_encoding = f.encoding,
-                                 type_names = type_names, context = self)
+                                 scope = scope, context = self)
                 tree = Parsing.p_module(s, pxd, full_module_name)
             finally:
                 f.close()
@@ -185,7 +256,7 @@ class Context:
         result.main_source_file = source
 
         if options.use_listing_file:
-            result.listing_file = replace_suffix(source, ".lis")
+            result.listing_file = Utils.replace_suffix(source, ".lis")
             Errors.open_listing_file(result.listing_file,
                 echo_to_stderr = options.errors_to_stderr)
         else:
@@ -197,19 +268,14 @@ class Context:
                 c_suffix = ".cpp"
             else:
                 c_suffix = ".c"
-            result.c_file = replace_suffix(source, c_suffix)
-        c_stat = None
-        if result.c_file:
-            try:
-                c_stat = os.stat(result.c_file)
-            except EnvironmentError:
-                pass
+            result.c_file = Utils.replace_suffix(source, c_suffix)
         module_name = full_module_name # self.extract_module_name(source, options)
         initial_pos = (source, 1, 0)
         scope = self.find_module(module_name, pos = initial_pos, need_pxd = 0)
         errors_occurred = False
         try:
-            tree = self.parse(source, scope.type_names, pxd = 0, full_module_name = full_module_name)
+            tree = self.parse(source, scope, pxd = 0,
+                              full_module_name = full_module_name)
             tree.process_implementation(scope, options, result)
         except CompileError:
             errors_occurred = True
@@ -219,8 +285,7 @@ class Context:
             errors_occurred = True
         if errors_occurred and result.c_file:
             try:
-                #os.unlink(result.c_file)
-                Utils.castrate_file(result.c_file, c_stat)
+                Utils.castrate_file(result.c_file, os.stat(source))
             except EnvironmentError:
                 pass
             result.c_file = None
@@ -237,7 +302,7 @@ class Context:
 
 #------------------------------------------------------------------------
 #
-#  Main Python entry point
+#  Main Python entry points
 #
 #------------------------------------------------------------------------
 
@@ -251,6 +316,10 @@ class CompilationOptions:
     include_path      [string]  Directories to search for include files
     output_file       string    Name of generated .c file
     generate_pxi      boolean   Generate .pxi file for public declarations
+    recursive         boolean   Recursively find and compile dependencies
+    timestamps        boolean   Only compile changed source files. If None,
+                                defaults to true when recursive is true.
+    quiet             boolean   Don't print source names in recursive mode
     transforms        Transform.TransformSet Transforms to use on the parse tree
     
     Following options are experimental and only used on MacOSX:
@@ -261,7 +330,7 @@ class CompilationOptions:
     cplus             boolean   Compile as c++ code
     """
     
-    def __init__(self, defaults = None, **kw):
+    def __init__(self, defaults = None, c_compile = 0, c_link = 0, **kw):
         self.include_path = []
         self.objects = []
         if defaults:
@@ -271,6 +340,10 @@ class CompilationOptions:
             defaults = default_options
         self.__dict__.update(defaults)
         self.__dict__.update(kw)
+        if c_compile:
+            self.c_only = 0
+        if c_link:
+            self.obj_only = 0
 
 
 class CompilationResult:
@@ -298,24 +371,87 @@ class CompilationResult:
         self.main_source_file = None
 
 
-def compile(source, options = None, c_compile = 0, c_link = 0,
-            full_module_name = None):
+class CompilationResultSet(dict):
     """
-    compile(source, options = default_options)
+    Results from compiling multiple Pyrex source files. A mapping
+    from source file paths to CompilationResult instances. Also
+    has the following attributes:
     
-    Compile the given Cython implementation file and return
-    a CompilationResult object describing what was produced.
+    num_errors   integer   Total number of compilation errors
+    """
+    
+    num_errors = 0
+
+    def add(self, source, result):
+        self[source] = result
+        self.num_errors += result.num_errors
+
+
+def compile_single(source, options, full_module_name = None):
+    """
+    compile_single(source, options, full_module_name)
+    
+    Compile the given Pyrex implementation file and return a CompilationResult.
+    Always compiles a single file; does not perform timestamp checking or
+    recursion.
     """
-    if not options:
-        options = default_options
-    options = CompilationOptions(defaults = options)
-    if c_compile:
-        options.c_only = 0
-    if c_link:
-        options.obj_only = 0
     context = Context(options.include_path)
     return context.compile(source, options, full_module_name)
 
+def compile_multiple(sources, options):
+    """
+    compile_multiple(sources, options)
+    
+    Compiles the given sequence of Pyrex implementation files and returns
+    a CompilationResultSet. Performs timestamp checking and/or recursion
+    if these are specified in the options.
+    """
+    sources = [os.path.abspath(source) for source in sources]
+    processed = set()
+    results = CompilationResultSet()
+    context = Context(options.include_path)
+    recursive = options.recursive
+    timestamps = options.timestamps
+    if timestamps is None:
+        timestamps = recursive
+    verbose = recursive and not options.quiet
+    for source in sources:
+        if source not in processed:
+            if not timestamps or context.c_file_out_of_date(source):
+                if verbose:
+                    sys.stderr.write("Compiling %s\n" % source)
+                result = context.compile(source, options)
+                results.add(source, result)
+            processed.add(source)
+            if recursive:
+                for module_name in context.find_cimported_module_names(source):
+                    path = context.find_pyx_file(module_name, [source])
+                    if path:
+                        sources.append(path)
+                    else:
+                        sys.stderr.write(
+                            "Cannot find .pyx file for cimported module '%s'\n" % module_name)
+    return results
+
+def compile(source, options = None, c_compile = 0, c_link = 0,
+            full_module_name = None, **kwds):
+    """
+    compile(source [, options], [, <option> = <value>]...)
+    
+    Compile one or more Pyrex implementation files, with optional timestamp
+    checking and recursing on dependecies. The source argument may be a string
+    or a sequence of strings If it is a string and no recursion or timestamp
+    checking is requested, a CompilationResult is returned, otherwise a
+    CompilationResultSet is returned.
+    """
+    options = CompilationOptions(defaults = options, c_compile = c_compile,
+        c_link = c_link, **kwds)
+    if isinstance(source, basestring) and not options.timestamps \
+            and not options.recursive:
+        return compile_single(source, options, full_module_name)
+    else:
+        return compile_multiple(source, options)
+
 #------------------------------------------------------------------------
 #
 #  Main command-line entry point
@@ -329,21 +465,19 @@ def main(command_line = 0):
         from CmdLine import parse_command_line
         options, sources = parse_command_line(args)
     else:
-        options = default_options
+        options = CompilationOptions(default_options)
         sources = args
     if options.show_version:
         sys.stderr.write("Cython version %s\n" % Version.version)
     if options.working_path!="":
         os.chdir(options.working_path)
-    context = Context(options.include_path)
-    for source in sources:
-        try:
-            result = context.compile(source, options)
-            if result.num_errors > 0:
-                any_failures = 1
-        except PyrexError, e:
-            sys.stderr.write(str(e) + '\n')
+    try:
+        result = compile(sources, options)
+        if result.num_errors > 0:
             any_failures = 1
+    except (EnvironmentError, PyrexError), e:
+        sys.stderr.write(str(e) + '\n')
+        any_failures = 1
     if any_failures:
         sys.exit(1)
 
@@ -363,6 +497,9 @@ default_options = dict(
     output_file = None,
     annotate = False,
     generate_pxi = 0,
+    recursive = 0,
+    timestamps = None,
+    quiet = 0,
     transforms = Transform.TransformSet(),
     working_path = "")
     

Cython/Compiler/ModuleNode.py

@@ -54,6 +54,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if self.has_imported_c_functions():
             self.module_temp_cname = env.allocate_temp_pyobject()
             env.release_temp(self.module_temp_cname)
+        self.generate_dep_file(env, result)
         self.generate_c_code(env, options, result)
         self.generate_h_code(env, options, result)
         self.generate_api_code(env, result)
@@ -65,6 +66,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     return 1
         return 0
     
+    def generate_dep_file(self, env, result):
+        modules = self.referenced_modules
+        if len(modules) > 1 or env.included_files:
+            dep_file = replace_suffix(result.c_file, ".dep")
+            f = open(dep_file, "w")
+            try:
+                for module in modules:
+                    if module is not env:
+                        f.write("cimport %s\n" % module.qualified_name)
+                    for path in module.included_files:
+                        f.write("include %s\n" % path)
+            finally:
+                f.close()
+
     def generate_h_code(self, env, options, result):
         def h_entries(entries, pxd = 0):
             return [entry for entry in entries

Cython/Compiler/Parsing.py

@@ -1203,8 +1203,10 @@ def p_include_statement(s, level):
     if s.compile_time_eval:
         include_file_path = s.context.find_include_file(include_file_name, pos)
         if include_file_path:
+            s.included_files.append(include_file_name)
             f = Utils.open_source_file(include_file_path, mode="rU")
-            s2 = PyrexScanner(f, include_file_path, s, source_encoding=f.encoding)
+            s2 = PyrexScanner(f, include_file_path, parent_scanner = s,
+                              source_encoding=f.encoding)
             try:
                 tree = p_statement_list(s2, level)
             finally:

Cython/Compiler/Scanning.py

@@ -206,24 +206,26 @@ def initial_compile_time_env():
 class PyrexScanner(Scanner):
     #  context            Context  Compilation context
     #  type_names         set      Identifiers to be treated as type names
+    #  included_files     [string] Files included with 'include' statement
     #  compile_time_env   dict     Environment for conditional compilation
     #  compile_time_eval  boolean  In a true conditional compilation context
     #  compile_time_expr  boolean  In a compile-time expression context
-    
     resword_dict = build_resword_dict()
 
     def __init__(self, file, filename, parent_scanner = None, 
-            type_names = None, context = None, source_encoding=None):
+                 scope = None, context = None, source_encoding=None):
         Scanner.__init__(self, get_lexicon(), file, filename)
         if parent_scanner:
             self.context = parent_scanner.context
             self.type_names = parent_scanner.type_names
+            self.included_files = parent_scanner.included_files
             self.compile_time_env = parent_scanner.compile_time_env
             self.compile_time_eval = parent_scanner.compile_time_eval
             self.compile_time_expr = parent_scanner.compile_time_expr
         else:
             self.context = context
-            self.type_names = type_names
+            self.type_names = scope.type_names
+            self.included_files = scope.included_files
             self.compile_time_env = initial_compile_time_env()
             self.compile_time_eval = 1
             self.compile_time_expr = 0

Cython/Compiler/Symtab.py

@@ -554,10 +554,6 @@ class Scope:
         return [entry for entry in self.temp_entries
             if entry not in self.free_temp_entries]
     
-    #def recycle_pending_temps(self):
-    #	# Obsolete
-    #	pass
-
     def use_utility_code(self, new_code):
         self.global_scope().use_utility_code(new_code)
     
@@ -687,6 +683,7 @@ class ModuleScope(Scope):
     # parent_module        Scope              Parent in the import namespace
     # module_entries       {string : Entry}   For cimport statements
     # type_names           {string : 1}       Set of type names (used during parsing)
+    # included_files       [string]           Cython sources included with 'include'
     # pxd_file_loaded      boolean            Corresponding .pxd file has been processed
     # cimported_modules    [ModuleScope]      Modules imported with cimport
     # new_interned_string_entries [Entry]     New interned strings waiting to be declared
@@ -723,6 +720,7 @@ class ModuleScope(Scope):
         self.interned_objs = []
         self.all_pystring_entries = []
         self.types_imported = {}
+        self.included_files = []
         self.pynum_entries = []
         self.has_extern_class = 0
         self.cached_builtins = []

Cython/Utils.py

@@ -24,7 +24,6 @@ def castrate_file(path, st):
     except EnvironmentError:
         pass
     else:
-        #st = os.stat(path)
         f.seek(0, 0)
         f.truncate()
         f.write(
@@ -33,6 +32,14 @@ def castrate_file(path, st):
         if st:
             os.utime(path, (st.st_atime, st.st_mtime))
 
+def modification_time(path):
+    st = os.stat(path)
+    return st.st_mtime
+
+def file_newer_than(path, time):
+    ftime = modification_time(path)
+    return ftime > time
+
 # support for source file encoding detection and unicode decoding
 
 def encode_filename(filename):