Merge pull request #461 from jdemeyer/optimize_dependencies

Optimize dependency checking

Cython/Build/Dependencies.py

@@ -45,10 +45,14 @@ except ImportError:
 from distutils.extension import Extension
 
 from .. import Utils
-from ..Utils import cached_function, cached_method, path_exists, find_root_package_dir, is_package_dir
+from ..Utils import (cached_function, cached_method, path_exists,
+    safe_makedirs, copy_file_to_dir_if_changed, find_root_package_dir,
+    is_package_dir)
 from ..Compiler.Main import Context, CompilationOptions, default_options
 
 join_path = cached_function(os.path.join)
+copy_once = cached_function(copy_file_to_dir_if_changed)
+safe_makedirs_once = cached_function(safe_makedirs)
 
 if sys.version_info[0] < 3:
     # stupid Py2 distutils enforces str type in list of sources
@@ -766,8 +770,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
     if 'common_utility_include_dir' in options:
         if options.get('cache'):
             raise NotImplementedError("common_utility_include_dir does not yet work with caching")
-        if not os.path.exists(options['common_utility_include_dir']):
-            os.makedirs(options['common_utility_include_dir'])
+        safe_makedirs(options['common_utility_include_dir'])
     c_options = CompilationOptions(**options)
     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
     ctx = c_options.create_context()
@@ -787,17 +790,15 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
     to_compile = []
     for m in module_list:
         if build_dir:
-            root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
+            root = os.path.abspath(find_root_package_dir(m.sources[0]))
             def copy_to_build_dir(filepath, root=root):
-                filepath_abs = os.path.realpath(os.path.abspath(filepath))
+                filepath_abs = os.path.abspath(filepath)
                 if os.path.isabs(filepath):
                     filepath = filepath_abs
                 if filepath_abs.startswith(root):
-                    mod_dir = os.path.join(build_dir,
+                    mod_dir = join_path(build_dir,
                             os.path.dirname(_relpath(filepath, root)))
-                    if not os.path.isdir(mod_dir):
-                        os.makedirs(mod_dir)
-                    shutil.copy(filepath, mod_dir)
+                    copy_once(filepath_abs, mod_dir)
             for dep in m.depends:
                 copy_to_build_dir(dep)
 
@@ -816,8 +817,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
                 if build_dir:
                     c_file = os.path.join(build_dir, c_file)
                     dir = os.path.dirname(c_file)
-                    if not os.path.isdir(dir):
-                        os.makedirs(dir)
+                    safe_makedirs_once(dir)
 
                 if os.path.exists(c_file):
                     c_timestamp = os.path.getmtime(c_file)

Cython/Utils.py

@@ -15,6 +15,7 @@ import sys
 import re
 import io
 import codecs
+import shutil
 from contextlib import contextmanager
 
 modification_time = os.path.getmtime
@@ -84,6 +85,34 @@ def file_newer_than(path, time):
     ftime = modification_time(path)
     return ftime > time
 
+
+def safe_makedirs(path):
+    try:
+        os.makedirs(path)
+    except OSError:
+        if not os.path.isdir(path):
+            raise
+
+
+def copy_file_to_dir_if_changed(sourcefile, destdir):
+    """
+    Copy file sourcefile to directory destdir (creating it if needed),
+    preserving metadata. If the destination file exists and is not
+    older than the source file, the copying is skipped.
+    """
+    destfile = os.path.join(destdir, os.path.basename(sourcefile))
+    try:
+        desttime = modification_time(destfile)
+    except OSError:
+        # New file does not exist, destdir may or may not exist
+        safe_makedirs(destdir)
+    else:
+        # New file already exists
+        if not file_newer_than(sourcefile, desttime):
+            return
+    shutil.copy2(sourcefile, destfile)
+
+
 @cached_function
 def search_include_directories(dirs, qualified_name, suffix, pos,
                                include=False, sys_path=False):