Add a coverage test job with a compiled Cython package to compare plain Python...

Add a coverage test job with a compiled Cython package to compare plain Python and compiled Cython both for completeness and speed.

.github/workflows/ci.yml

@@ -71,17 +71,18 @@ jobs:
           - os: ubuntu-18.04
             python-version: pypy-2.7
             backend: c
-            env: {}
+            env: { NO_CYTHON_COMPILE: 1 }
           - os: ubuntu-18.04
             python-version: pypy-3.7
             backend: c
-            env: {}
+            env: { NO_CYTHON_COMPILE: 1 }
           # Pypy [allowed-failures] - These specifically test known bugs
           - os: ubuntu-18.04
             python-version: pypy-2.7
             backend: c
             env:
               {
+                NO_CYTHON_COMPILE: 1,
                 EXCLUDE: "--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs",
               }
             allowed_failure: true
@@ -89,7 +90,7 @@ jobs:
           - os: ubuntu-18.04
             python-version: pypy-3.7
             backend: c
-            env: { EXCLUDE: "--listfile=tests/pypy_bugs.txt bugs" }
+            env: { NO_CYTHON_COMPILE: 1, EXCLUDE: "--listfile=tests/pypy_bugs.txt bugs" }
             allowed_failure: true
             extra_hash: "-allowed_failures"
           # Coverage - Disabled due to taking too long to run
@@ -149,10 +150,7 @@ jobs:
         run: bash ./Tools/ci-run.sh
         
 
-  code_quality:
-    strategy:
-      # Allows for matrix sub-jobs to fail without canceling the rest
-      fail-fast: false
+  pycoverage:
     runs-on: ubuntu-18.04
 
     env:
@@ -172,11 +170,40 @@ jobs:
           python-version: 3.9
 
       - name: Run Coverage
-        env: { COVERAGE: 1 }
+        env: { COVERAGE: 1, NO_CYTHON_COMPILE: 1 }
         run: bash ./Tools/ci-run.sh
         
       - name: Upload Coverage Report
         uses: actions/upload-artifact@v2
         with:
-          path: coverage-report-html  
+          name: pycoverage_html
+          path: coverage-report-html
         
+  cycoverage:
+    runs-on: ubuntu-18.04
+
+    env:
+      BACKEND: c,cpp
+      OS_NAME: ubuntu-18.04
+      PYTHON_VERSION: 3.9
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Run Coverage
+        env: { COVERAGE: 1 }
+        run: bash ./Tools/ci-run.sh
+
+      - name: Upload Coverage Report
+        uses: actions/upload-artifact@v2
+        with:
+          name: cycoverage_html
+          path: coverage-report-html

Tools/ci-run.sh

@@ -84,9 +84,12 @@ fi
 ccache -s 2>/dev/null || true
 export PATH="/usr/lib/ccache:$PATH"
 
-if [ "$COVERAGE" != "1" -a -n "${PYTHON_VERSION##pypy*}" ]; then
+if [ "$NO_CYTHON_COMPILE" != "1" -a -n "${PYTHON_VERSION##pypy*}" ]; then
   CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" \
-      python setup.py build_ext -i $(python -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")') || exit 1
+  python setup.py build_ext -i \
+          $(if [ "$COVERAGE" == "1" ]; then echo " --cython-coverage"; fi) \
+          $(python -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")') \
+      || exit 1
 fi
 
 if [ "$TEST_CODE_STYLE" != "1" -a -n "${PYTHON_VERSION##pypy*}" ]; then

setup.py

@@ -79,7 +79,7 @@ else:
         scripts = ["cython.py", "cythonize.py", "cygdb.py"]
 
 
-def compile_cython_modules(profile=False, compile_more=False, cython_with_refnanny=False):
+def compile_cython_modules(profile=False, coverage=False, compile_more=False, cython_with_refnanny=False):
     source_root = os.path.abspath(os.path.dirname(__file__))
     compiled_modules = [
         "Cython.Plex.Scanners",
@@ -135,6 +135,8 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
     defines = []
     if cython_with_refnanny:
         defines.append(('CYTHON_REFNANNY', '1'))
+    if coverage:
+        defines.append(('CYTHON_TRACE', '1'))
 
     extensions = []
     for module in compiled_modules:
@@ -168,6 +170,9 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
     if profile:
         get_directive_defaults()['profile'] = True
         sys.stderr.write("Enabled profiling for the Cython binary modules\n")
+    if coverage:
+        get_directive_defaults()['linetrace'] = True
+        sys.stderr.write("Enabled line tracing and profiling for the Cython binary modules\n")
 
     # not using cythonize() directly to let distutils decide whether building extensions was requested
     add_command_class("build_ext", new_build_ext)
@@ -178,6 +183,10 @@ cython_profile = '--cython-profile' in sys.argv
 if cython_profile:
     sys.argv.remove('--cython-profile')
 
+cython_coverage = '--cython-coverage' in sys.argv
+if cython_coverage:
+    sys.argv.remove('--cython-coverage')
+
 try:
     sys.argv.remove("--cython-compile-all")
     cython_compile_more = True
@@ -230,7 +239,7 @@ packages = [
 
 def run_build():
     if compile_cython_itself and (is_cpython or cython_compile_more):
-        compile_cython_modules(cython_profile, cython_compile_more, cython_with_refnanny)
+        compile_cython_modules(cython_profile, cython_coverage, cython_compile_more, cython_with_refnanny)
 
     from Cython import __version__ as version
     setup(