Merge pull request #2910 from mattip/includes

BUG: prefer sys.path before Cython/Include when searching for pxd files

CHANGES.rst

@@ -115,6 +115,12 @@ Other changes
 
 * Support for Python 2.6 was removed.
 
+* The search order for include files was changed. Previously it was
+  ``include_directories``, ``Cython/Includes``, ``sys.path``. Now it is
+  ``include_directories``, ``sys.path``, ``Cython/Includes``. This was done to
+  allow third-party ``*.pxd`` files to override the ones in Cython.
+  (Github issue #2905)
+
 
 0.29.7 (2019-0?-??)
 ===================

Cython/Compiler/Main.py

@@ -40,6 +40,8 @@ module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_
 
 verbose = 0
 
+standard_include_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                        os.path.pardir, 'Includes'))
 
 class Context(object):
     #  This class encapsulates the context needed for compiling
@@ -75,10 +77,6 @@ class Context(object):
         self.pxds = {}  # full name -> node tree
         self._interned = {}  # (type(value), value, *key_args) -> interned_value
 
-        standard_include_path = os.path.abspath(os.path.normpath(
-            os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes')))
-        self.include_directories = include_directories + [standard_include_path]
-
         if language_level is not None:
             self.set_language_level(language_level)
 
@@ -253,8 +251,13 @@ class Context(object):
 
     def search_include_directories(self, qualified_name, suffix, pos,
                                    include=False, sys_path=False):
-        return search_include_directories(
-            tuple(self.include_directories), qualified_name, suffix, pos, include, sys_path)
+        include_dirs = self.include_directories
+        if sys_path:
+            include_dirs = include_dirs + sys.path
+        # include_dirs must be hashable for caching in @cached_function
+        include_dirs = tuple(include_dirs + [standard_include_path])
+        return search_include_directories(include_dirs, qualified_name,
+                                          suffix, pos, include)
 
     def find_root_package_dir(self, file_path):
         return Utils.find_root_package_dir(file_path)
@@ -602,8 +605,7 @@ def compile(source, options = None, full_module_name = None, **kwds):
 
 
 @Utils.cached_function
-def search_include_directories(dirs, qualified_name, suffix, pos,
-                               include=False, sys_path=False):
+def search_include_directories(dirs, qualified_name, suffix, pos, include=False):
     """
     Search the list of include directories for the given file name.
 
@@ -612,10 +614,7 @@ def search_include_directories(dirs, qualified_name, suffix, pos,
     report an error.
 
     The 'include' option will disable package dereferencing.
-    If 'sys_path' is True, also search sys.path.
     """
-    if sys_path:
-        dirs = dirs + tuple(sys.path)
 
     if pos:
         file_desc = pos[0]
@@ -648,7 +647,7 @@ def search_include_directories(dirs, qualified_name, suffix, pos,
                 path = os.path.join(package_dir, module_filename)
                 if os.path.exists(path):
                     return path
-                path = os.path.join(dirname, package_dir, module_name,
+                path = os.path.join(package_dir, module_name,
                                     package_filename)
                 if os.path.exists(path):
                     return path

tests/compile/find_pxd.srctree

@@ -6,12 +6,13 @@ from Cython.Build import cythonize
 from Cython.Distutils.extension import Extension
 
 import sys
-sys.path.append("path")
+sys.path.insert(0, "path")
 
 ext_modules = [
     Extension("a", ["a.pyx"]),
     Extension("b", ["b.pyx"]),
     Extension("c", ["c.pyx"]),
+    Extension("d", ["d.pyx"]),
 ]
 
 ext_modules = cythonize(ext_modules, include_path=["include"])
@@ -37,3 +38,15 @@ ctypedef int my_type
 
 ######## path/c.pxd ########
 +++syntax error just to show that this file is not actually cimported+++
+
+######## path/numpy/__init__.pxd ########
+
+# gh-2905: This should be found before Cython/Inlude/numpy/__init__.pxd
+
+ctypedef int my_type
+
+######## d.pyx ########
+
+cimport numpy
+
+cdef numpy.my_type foo