[0.29] implement the --depfile command-line option for the "cython" tool (GH-4949)

Backports https://github.com/cython/cython/pull/4916

Cython/Build/Dependencies.py

@@ -43,7 +43,7 @@ except:
     pythran = None
 
 from .. import Utils
-from ..Utils import (cached_function, cached_method, path_exists,
+from ..Utils import (cached_function, cached_method, path_exists, write_depfile,
     safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix)
 from ..Compiler.Main import Context, CompilationOptions, default_options
 
@@ -1030,22 +1030,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
                 # write out the depfile, if requested
                 if depfile:
                     dependencies = deps.all_dependencies(source)
-                    src_base_dir, _ = os.path.split(source)
-                    if not src_base_dir.endswith(os.sep):
-                        src_base_dir += os.sep
-                    # paths below the base_dir are relative, otherwise absolute
-                    paths = []
-                    for fname in dependencies:
-                        if (fname.startswith(src_base_dir) or
-                            fname.startswith('.' + os.path.sep)):
-                            paths.append(os.path.relpath(fname, src_base_dir))
-                        else:
-                            paths.append(os.path.abspath(fname))
-
-                    depline = os.path.split(c_file)[1] + ": \\\n  "
-                    depline += " \\\n  ".join(paths) + "\n"
-                    with open(c_file+'.dep', 'w') as outfile:
-                        outfile.write(depline)
+                    write_depfile(c_file, source, dependencies)
 
                 if os.path.exists(c_file):
                     c_timestamp = os.path.getmtime(c_file)

Cython/Compiler/CmdLine.py

@@ -53,6 +53,7 @@ Options:
   --module-name                  Fully qualified module name. If not given, it is deduced from the
                                  import path if source file is in a package, or equals the
                                  filename otherwise.
+  -M, --depfile                  Produce depfiles for the sources
 """
 
 
@@ -66,7 +67,6 @@ def bad_usage():
     sys.stderr.write(usage)
     sys.exit(1)
 
-
 def parse_command_line(args):
     from .Main import CompilationOptions, default_options
 
@@ -195,6 +195,8 @@ def parse_command_line(args):
                     sys.exit(1)
             elif option == "--module-name":
                 options.module_name = pop_value()
+            elif option in ('-M', '--depfile'):
+                options.depfile = True
             elif option.startswith('--debug'):
                 option = option[2:].replace('-', '_')
                 from . import DebugFlags
@@ -236,4 +238,3 @@ def parse_command_line(args):
                 "cython: Only one source file allowed when using --module-name\n")
             sys.exit(1)
     return options, sources
-

Cython/Compiler/Main.py

@@ -514,6 +514,10 @@ def run_pipeline(source, options, full_module_name=None, context=None):
     context.setup_errors(options, result)
     err, enddata = Pipeline.run_pipeline(pipeline, source)
     context.teardown_errors(err, options, result)
+    if options.depfile:
+        from ..Build.Dependencies import create_dependency_tree
+        dependencies = create_dependency_tree(context).all_dependencies(result.main_source_file)
+        Utils.write_depfile(result.c_file, result.main_source_file, dependencies)
     return result
 
 
@@ -881,6 +885,7 @@ default_options = dict(
     errors_to_stderr = 1,
     cplus = 0,
     output_file = None,
+    depfile = None,
     annotate = None,
     annotate_coverage_xml = None,
     generate_pxi = 0,

Cython/Utils.py

@@ -447,3 +447,24 @@ def build_hex_version(version_string):
         hexversion = (hexversion << 8) + digit
 
     return '0x%08X' % hexversion
+
+
+def write_depfile(target, source, dependencies):
+    src_base_dir = os.path.dirname(source)
+    cwd = os.getcwd()
+    if not src_base_dir.endswith(os.sep):
+        src_base_dir += os.sep
+    # paths below the base_dir are relative, otherwise absolute
+    paths = []
+    for fname in dependencies:
+        fname = os.path.abspath(fname)
+        if fname.startswith(src_base_dir):
+            paths.append(os.path.relpath(fname, cwd))
+        else:
+            paths.append(fname)
+
+    depline = os.path.relpath(target, cwd) + ": \\\n  "
+    depline += " \\\n  ".join(paths) + "\n"
+
+    with open(target+'.dep', 'w') as outfile:
+        outfile.write(depline)

tests/build/depfile_package_cython.srctree

+"""
+PYTHON -c 'import os; os.makedirs("builddir/pkg/sub")'
+CYTHON -M pkg/test.pyx -o builddir/pkg/test.c
+CYTHON --depfile pkg/sub/test.pyx -o builddir/pkg/sub/test.c
+PYTHON check.py
+"""
+
+######## check.py ########
+
+import os.path
+
+def pkgpath(*args):
+    return os.path.join('pkg', *args)
+
+with open(os.path.join("builddir", "pkg", "test.c.dep"), "r") as f:
+    contents = f.read().replace("\\\n", " ").replace("\n", " ")
+
+assert sorted(contents.split()) == sorted([os.path.join('builddir', 'pkg', 'test.c:'), pkgpath('sub', 'incl.pxi'), pkgpath('test.pxd'), pkgpath('test.pyx')]), contents
+
+
+with open(os.path.join("builddir", "pkg", "sub", "test.c.dep"), "r") as f:
+    contents = f.read().replace("\\\n", " ").replace("\n", " ")
+
+contents = [os.path.relpath(entry, '.')
+            if os.path.isabs(entry) else entry for entry in contents.split()]
+assert sorted(contents) == sorted([os.path.join('builddir', 'pkg', 'sub', 'test.c:'), pkgpath('sub', 'incl.pxi'), pkgpath('sub', 'test.pyx')]), contents  # last is really one level up
+
+######## pkg/__init__.py ########
+
+
+######## pkg/test.pyx ########
+
+TEST = "pkg.test"
+
+include "sub/incl.pxi"
+
+cdef object get_str():
+   return TEST
+
+
+######## pkg/test.pxd ########
+
+cdef object get_str()
+
+
+######## pkg/sub/__init__.py ########
+
+
+######## pkg/sub/test.pyx ########
+# cython: language_level=3
+
+from ..test cimport get_str
+
+include 'incl.pxi'
+
+TEST = 'pkg.sub.test'
+
+
+######## pkg/sub/incl.pxi ########
+
+pass

tests/build/depfile_package.srctree → tests/build/depfile_package_cythonize.srctree

@@ -7,10 +7,13 @@ PYTHON package_test.py
 
 import os.path
 
+def pkgpath(*args):
+    return os.path.join('pkg', *args)
+
 with open(os.path.join("pkg", "test.c.dep"), "r") as f:
     contents = f.read().replace("\\\n", " ").replace("\n", " ")
 
-assert sorted(contents.split()) == sorted(['test.c:', os.path.join('sub', 'incl.pxi'), 'test.pxd', 'test.pyx']), contents
+assert sorted(contents.split()) == sorted([pkgpath('test.c:'), pkgpath('sub', 'incl.pxi'), pkgpath('test.pxd'), pkgpath('test.pyx')]), contents
 
 
 with open(os.path.join("pkg", "sub", "test.c.dep"), "r") as f:
@@ -18,7 +21,7 @@ with open(os.path.join("pkg", "sub", "test.c.dep"), "r") as f:
 
 contents = [os.path.relpath(entry, '.')
             if os.path.isabs(entry) else entry for entry in contents.split()]
-assert sorted(contents) == sorted(['test.c:', 'incl.pxi', 'test.pyx', os.path.join('..', 'test.pxd')]), contents
+assert sorted(contents) == sorted([pkgpath('sub', 'test.c:'), pkgpath('sub', 'incl.pxi'), pkgpath('sub', 'test.pyx'), pkgpath('test.pxd')]), contents  # last is really one level up
 
 
 ######## pkg/__init__.py ########