coverage: support reporting in a separate run (when we haven't traced any files ourselves)

Cython/Coverage.py

@@ -38,13 +38,6 @@ class Plugin(CoveragePlugin):
         if not c_file:
             return None
 
-        try:
-            with open(c_file, 'rb') as f:
-                if b'/* Generated by Cython ' not in f.read(30):
-                    return None  # not a Cython file
-        except (IOError, OSError):
-            return None
-
         # parse all source file paths and lines from C file
         # to learn about all relevant source files right away (pyx/pxi/pxd)
         # FIXME: this might already be too late if the first executed line
@@ -59,11 +52,13 @@ class Plugin(CoveragePlugin):
             return None  # let coverage.py handle it (e.g. .py files)
 
         filename = os.path.abspath(filename)
-        if not self._c_files_map or filename not in self._c_files_map:
-            return None  # unknown file
-        c_file, rel_file_path, code, excluded = self._c_files_map[filename]
-        if code is None:
-            return None  # unknown file
+        if self._c_files_map and filename in self._c_files_map:
+            c_file, rel_file_path, code, excluded = self._c_files_map[filename]
+        else:
+            c_file, _ = self._find_source_files(filename)
+            if not c_file:
+                return None  # unknown file
+            rel_file_path, code, excluded = self._parse_lines(c_file, filename)
         return CythonModuleReporter(c_file, filename, rel_file_path, code, excluded)
 
     def _find_source_files(self, filename):
@@ -94,6 +89,13 @@ class Plugin(CoveragePlugin):
             if not os.path.exists(py_source_file):
                 py_source_file = None
 
+            try:
+                with open(c_file, 'rb') as f:
+                    if b'/* Generated by Cython ' not in f.read(30):
+                        return None  # not a Cython file
+            except (IOError, OSError):
+                c_file = None
+
         return c_file, py_source_file
 
     def _parse_lines(self, c_file, sourcefile):
@@ -189,3 +191,22 @@ class CythonModuleReporter(FileReporter):
 
     def excluded_statements(self):
         return self._excluded
+
+    def _iter_source_lines(self):
+        current_line = 1
+        for line_no, code_line in sorted(self._code.iteritems()):
+            while line_no > current_line:
+                yield ''
+                current_line += 1
+            yield code_line
+
+    def source(self):
+        if os.path.exists(self.source_file):
+            with open(self.source_file) as f:
+                return f.read()
+        else:
+            return '\n'.join(self._iter_source_lines())
+
+    def source_token_lines(self):
+        for line in self._iter_source_lines():
+            yield [('txt', line)]

tests/run/coverage_cmd.srctree

+# mode: run
+# tag: coverage,trace
+
+"""
+PYTHON -c 'import shutil; shutil.copy("pkg/coverage_test_pyx.pyx", "pkg/coverage_test_pyx.pxi")'
+PYTHON setup.py build_ext -i
+PYTHON -m coverage run coverage_test.py
+PYTHON collect_coverage.py
+"""
+
+######## setup.py ########
+
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(ext_modules = cythonize([
+    'coverage_test_*.py*',
+    'pkg/coverage_test_*.py*'
+]))
+
+
+######## .coveragerc ########
+[run]
+plugins = Cython.Coverage
+
+
+######## pkg/__init__.py ########
+
+######## pkg/coverage_test_py.py ########
+# cython: linetrace=True
+# distutils: define_macros=CYTHON_TRACE=1
+
+def func1(a, b):
+    x = 1               #  5
+    c = func2(a) + b    #  6
+    return x + c        #  7
+
+
+def func2(a):
+    return a * 2        # 11
+
+
+######## pkg/coverage_test_pyx.pyx ########
+# cython: linetrace=True
+# distutils: define_macros=CYTHON_TRACE=1
+
+def func1(int a, int b):
+    cdef int x = 1      #  5
+    c = func2(a) + b    #  6
+    return x + c        #  7
+
+
+def func2(int a):
+    return a * 2        # 11
+
+
+######## coverage_test_include_pyx.pyx ########
+# cython: linetrace=True
+# distutils: define_macros=CYTHON_TRACE=1
+
+cdef int x = 5                                   #  4
+
+cdef int cfunc1(int x):                          #  6
+    return x * 3                                 #  7
+
+include "pkg/coverage_test_pyx.pxi"              #  9
+
+def main_func(int x):                            # 11
+    return cfunc1(x) + func1(x, 4) + func2(x)    # 12
+
+
+######## coverage_test.py ########
+
+import os.path
+try:
+    # io.StringIO in Py2.x cannot handle str ...
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+from pkg import coverage_test_py
+from pkg import coverage_test_pyx
+import coverage_test_include_pyx
+
+
+for module in [coverage_test_py, coverage_test_pyx, coverage_test_include_pyx]:
+    assert not any(module.__file__.endswith(ext) for ext in '.py .pyc .pyo .pyw .pyx .pxi'.split()), \
+        module.__file__
+
+
+def run_coverage(module):
+    module_name = module.__name__
+    module_path = module_name.replace('.', os.path.sep) + '.' + module_name.rsplit('_', 1)[-1]
+
+    assert module.func1(1, 2) == (1 * 2) + 2 + 1
+    assert module.func2(2) == 2 * 2
+    if '_include_' in module_name:
+        assert module.main_func(2) == (2 * 3) + ((2 * 2) + 4 + 1) + (2 * 2)
+
+
+if __name__ == '__main__':
+    run_coverage(coverage_test_py)
+    run_coverage(coverage_test_pyx)
+    run_coverage(coverage_test_include_pyx)
+
+
+######## collect_coverage.py ########
+
+import sys
+import os
+import os.path
+import subprocess
+
+def run_report():
+    env = dict(os.environ, LANG='', LC_ALL='C')
+    process = subprocess.Popen(
+        [sys.executable, '-m', 'coverage', 'report', '--show-missing'],
+        stdout=subprocess.PIPE, env=env)
+    stdout, _ = process.communicate()
+    stdout = stdout.decode('iso8859-1')  # 'safe' decoding
+    lines = stdout.splitlines()
+    print(stdout)
+
+    # FIXME:  'coverage_test_pyx.pxi' may not be found if coverage.py requests it before the .pyx file
+    for module_path in ('coverage_test_py.py', 'coverage_test_pyx.pyx', 'coverage_test_include_pyx.pyx'):
+        assert any(module_path in line for line in lines), "'%s' not found in coverage report:\n\n%s" % (
+            module_path, stdout)
+
+
+if __name__ == '__main__':
+    run_report()