Support for versioned pxd files (GH-3577)

* Support for versioned pxd files like "lib.cython-30.pxd" for a Cython 3.0+ version.
See https://github.com/cython/cython/issues/3573

* Fix test in Py2 by avoiding namespace packages in favour of a normal package.

Cython/Compiler/Main.py

@@ -667,44 +667,40 @@ def search_include_directories(dirs, qualified_name, suffix, pos, include=False)
         names = qualified_name.split('.')
         package_names = tuple(names[:-1])
         module_name = names[-1]
-        module_filename = module_name + suffix
-        package_filename = "__init__" + suffix
 
         # search for standard packages first - PEP420
         namespace_dirs = []
         for dirname in dirs:
             package_dir, is_namespace = Utils.check_package_dir(dirname, package_names)
             if package_dir is not None:
-
                 if is_namespace:
                     namespace_dirs.append(package_dir)
                     continue
-
-                # matches modules of the form: <dir>/foo/bar.pxd
-                path = os.path.join(package_dir, module_filename)
-                if os.path.exists(path):
-                    return path
-
-                # matches modules of the form: <dir>/foo/bar/__init__.pxd
-                path = os.path.join(package_dir, module_name, package_filename)
-                if os.path.exists(path):
+                path = search_module_in_dir(package_dir, module_name, suffix)
+                if path:
                     return path
 
         # search for namespaces second - PEP420
         for package_dir in namespace_dirs:
-            # matches modules of the form: <dir>/foo/bar.pxd
-            path = os.path.join(package_dir, module_filename)
-            if os.path.exists(path):
-                return path
-
-            # matches modules of the form: <dir>/foo/bar/__init__.pxd
-            path = os.path.join(package_dir, module_name, package_filename)
-            if os.path.exists(path):
+            path = search_module_in_dir(package_dir, module_name, suffix)
+            if path:
                 return path
 
     return None
 
 
+@Utils.cached_function
+def search_module_in_dir(package_dir, module_name, suffix):
+    # matches modules of the form: <dir>/foo/bar.pxd
+    path = Utils.find_versioned_file(package_dir, module_name, suffix)
+
+    # matches modules of the form: <dir>/foo/bar/__init__.pxd
+    if not path and suffix:
+        path = Utils.find_versioned_file(os.path.join(package_dir, module_name), "__init__", suffix)
+
+    return path
+
+
 # ------------------------------------------------------------------------
 #
 #  Main command-line entry point

Cython/Utils.py

@@ -20,9 +20,12 @@ import sys
 import re
 import io
 import codecs
+import glob
 import shutil
 from contextlib import contextmanager
 
+from . import __version__ as cython_version
+
 PACKAGE_FILES = ("__init__.py", "__init__.pyc", "__init__.pyx", "__init__.pxd")
 
 modification_time = os.path.getmtime
@@ -197,6 +200,39 @@ def path_exists(path):
     return False
 
 
+_parse_file_version = re.compile(r".*[.]cython-([0-9]+)[.][^./\\]+$").findall
+
+
+@cached_function
+def find_versioned_file(directory, filename, suffix,
+                        _current_version=int(re.sub(r"^([0-9]+)[.]([0-9]+).*", r"\1\2", cython_version))):
+    """
+    Search a directory for versioned pxd files, e.g. "lib.cython-30.pxd" for a Cython 3.0+ version.
+
+    @param directory: the directory to search
+    @param filename: the filename without suffix
+    @param suffix: the filename extension including the dot, e.g. ".pxd"
+    @return: the file path if found, or None
+    """
+    assert not suffix or suffix[:1] == '.'
+    path_prefix = os.path.join(directory, filename)
+
+    matching_files = glob.glob(path_prefix + ".cython-*" + suffix)
+    path = path_prefix + suffix
+    if not os.path.exists(path):
+        path = None
+    best_match = (-1, path)  # last resort, if we do not have versioned .pxd files
+
+    for path in matching_files:
+        versions = _parse_file_version(path)
+        if versions:
+            int_version = int(versions[0])
+            # Let's assume no duplicates.
+            if best_match[0] < int_version <= _current_version:
+                best_match = (int_version, path)
+    return best_match[1]
+
+
 # file name encodings
 
 def decode_filename(filename):

tests/run/versioned_pxds.srctree

+# mode: run
+# tag: pxd
+
+"""
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import runner"
+"""
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+
+from distutils.core import setup, Extension
+
+setup(
+  ext_modules=cythonize([
+    Extension("pkg.m1.a", ["pkg/m1/a.pyx"]),
+    Extension("pkg.m2.b", ["pkg/m2/b.pyx"])
+  ]),
+)
+
+######## pkg/__init__.py ########
+
+######## pkg/m1/__init__.py ########
+
+
+######## pkg/m1/a.pyx ########
+
+cdef class A:
+    def __init__(self):
+        self.x = 5
+
+######## pkg/m1/a.pxd ########
+
+to be ignored if there is a more specific file
+
+######## pkg/m1/a.cython-2.pxd ########
+
+very outdated, not to be picked up
+
+######## pkg/m1/a.cython-20.pxd ########
+
+outdated, not to be picked up
+
+######## pkg/m1/a.cython-29.pxd ########
+
+# closest version should get found!
+
+cdef class A:
+    cdef public float x
+
+######## pkg/m1/a.cython-300000.pxd ########
+
+Invalid distant future syntax right here!
+
+######## pkg/m1/a.cython-100000.pxd ########
+
+Invalid future syntax right here!
+
+
+######## pkg/m2/__init__.py ########
+
+######## pkg/m2/b.pyx ########
+
+from pkg.m1.a cimport A
+
+cdef class B(A):
+    pass
+
+######## runner.py ########
+
+from pkg.m1.a import A
+from pkg.m2.b import B
+
+a = A()
+b = B()
+
+assert a.x == 5
+assert isinstance(a.x, float), type(a.x)