Make "new_build_ext" the new "build_ext" (GH-4498)

This also solves a difficulty with the Cython import in setuptools' build_ext. We need to inherit from the one in distutils, so that setuptools can inherit from us. That leads to a circular dependency that goes either way depending on which gets imported first by users, and in what way (from-import or module import). This is built to match the code in

https://github.com/pypa/setuptools/blob/9f1822ee910df3df930a98ab99f66d18bb70659b/setuptools/command/build_ext.py#L14-L21

Closes https://github.com/cython/cython/issues/3541

Cython/Distutils/build_ext.py

 import sys
+import os
 
-if 'setuptools' in sys.modules:
-    try:
-        from setuptools.command.build_ext import build_ext as _build_ext
-    except ImportError:
-        # We may be in the process of importing setuptools, which tries
-        # to import this.
-        from distutils.command.build_ext import build_ext as _build_ext
-else:
-    from distutils.command.build_ext import build_ext as _build_ext
+try:
+    from __builtin__ import basestring
+except ImportError:
+    basestring = str
+
+# Always inherit from the "build_ext" in distutils since setuptools already imports
+# it from Cython if available, and does the proper distutils fallback otherwise.
+# https://github.com/pypa/setuptools/blob/9f1822ee910df3df930a98ab99f66d18bb70659b/setuptools/command/build_ext.py#L16
 
+# setuptools imports Cython's "build_ext", so make sure we go first.
+_build_ext_module = sys.modules.get('setuptools.command.build_ext')
+if _build_ext_module is None:
+    import distutils.command.build_ext as _build_ext_module
+
+# setuptools remembers the original distutils "build_ext" as "_du_build_ext"
+_build_ext = getattr(_build_ext_module, '_du_build_ext', None)
+if _build_ext is None:
+    _build_ext = getattr(_build_ext_module, 'build_ext', None)
+if _build_ext is None:
+    from distutils.command.build_ext import build_ext as _build_ext
 
-class new_build_ext(_build_ext, object):
 
-    user_options = _build_ext.user_options[:]
-    boolean_options = _build_ext.boolean_options[:]
+class build_ext(_build_ext, object):
 
-    user_options.extend([
+    user_options = _build_ext.user_options + [
+        ('cython-cplus', None,
+             "generate C++ source files"),
+        ('cython-create-listing', None,
+             "write errors to a listing file"),
+        ('cython-line-directives', None,
+             "emit source line directives"),
+        ('cython-include-dirs=', None,
+             "path to the Cython include files" + _build_ext.sep_by),
         ('cython-c-in-temp', None,
              "put generated C files in temp directory"),
-        ])
+        ('cython-gen-pxi', None,
+            "generate .pxi file for public declarations"),
+        ('cython-directives=', None,
+            "compiler directive overrides"),
+        ('cython-gdb', None,
+             "generate debug information for cygdb"),
+        ('cython-compile-time-env', None,
+            "cython compile time environment"),
+        ]
 
-    boolean_options.extend([
-        'cython-c-in-temp'
-    ])
+    boolean_options = _build_ext.boolean_options + [
+        'cython-cplus', 'cython-create-listing', 'cython-line-directives',
+        'cython-c-in-temp', 'cython-gdb',
+    ]
 
     def initialize_options(self):
-        _build_ext.initialize_options(self)
+        super(build_ext, self).initialize_options()
+        self.cython_cplus = 0
+        self.cython_create_listing = 0
+        self.cython_line_directives = 0
+        self.cython_include_dirs = None
+        self.cython_directives = None
         self.cython_c_in_temp = 0
+        self.cython_gen_pxi = 0
+        self.cython_gdb = False
+        self.cython_compile_time_env = None
+
+    def finalize_options(self):
+        super(build_ext, self).finalize_options()
+        if self.cython_include_dirs is None:
+            self.cython_include_dirs = []
+        elif isinstance(self.cython_include_dirs, basestring):
+            self.cython_include_dirs = \
+                self.cython_include_dirs.split(os.pathsep)
+        if self.cython_directives is None:
+            self.cython_directives = {}
+
+    def get_extension_attr(self, extension, option_name, default=False):
+        return getattr(self, option_name) or getattr(extension, option_name, default)
 
     def build_extension(self, ext):
         from Cython.Build.Dependencies import cythonize
-        if self.cython_c_in_temp:
-            build_dir = self.build_temp
-        else:
-            build_dir = None
-        new_ext = cythonize(ext,force=self.force, quiet=self.verbose == 0, build_dir=build_dir)[0]
+
+        # Set up the include_path for the Cython compiler:
+        #    1.    Start with the command line option.
+        #    2.    Add in any (unique) paths from the extension
+        #        cython_include_dirs (if Cython.Distutils.extension is used).
+        #    3.    Add in any (unique) paths from the extension include_dirs
+        includes = list(self.cython_include_dirs)
+        for include_dir in getattr(ext, 'cython_include_dirs', []):
+            if include_dir not in includes:
+                includes.append(include_dir)
+
+        # In case extension.include_dirs is a generator, evaluate it and keep
+        # result
+        ext.include_dirs = list(ext.include_dirs)
+        for include_dir in ext.include_dirs + list(self.include_dirs):
+            if include_dir not in includes:
+                includes.append(include_dir)
+
+        # Set up Cython compiler directives:
+        #    1. Start with the command line option.
+        #    2. Add in any (unique) entries from the extension
+        #         cython_directives (if Cython.Distutils.extension is used).
+        directives = dict(self.cython_directives)
+        if hasattr(ext, "cython_directives"):
+            directives.update(ext.cython_directives)
+
+        if self.get_extension_attr(ext, 'cython_cplus'):
+            ext.language = 'c++'
+
+        options = {
+            'use_listing_file': self.get_extension_attr(ext, 'cython_create_listing'),
+            'emit_linenums': self.get_extension_attr(ext, 'cython_line_directives'),
+            'include_path': includes,
+            'compiler_directives': directives,
+            'build_dir': self.build_temp if self.get_extension_attr(ext, 'cython_c_in_temp') else None,
+            'generate_pxi': self.get_extension_attr(ext, 'cython_gen_pxi'),
+            'gdb_debug': self.get_extension_attr(ext, 'cython_gdb'),
+            'c_line_in_traceback': not getattr(ext, 'no_c_in_traceback', 0),
+            'compile_time_env': self.get_extension_attr(ext, 'cython_compile_time_env', default=None),
+        }
+
+        new_ext = cythonize(
+            ext,force=self.force, quiet=self.verbose == 0, **options
+        )[0]
+
         ext.sources = new_ext.sources
-        super(new_build_ext, self).build_extension(ext)
+        super(build_ext, self).build_extension(ext)
 
-# This will become new_build_ext in the future.
-from .old_build_ext import old_build_ext as build_ext
+# backward compatibility
+new_build_ext = build_ext

docs/src/userguide/source_files_and_compilation.rst

@@ -415,7 +415,7 @@ Cython's build_ext module which runs ``cythonize`` as part of the build process:
 
     setup(
         extensions = [Extension("*", ["*.pyx"])],
-        cmdclass={'build_ext': Cython.Build.new_build_ext},
+        cmdclass={'build_ext': Cython.Build.build_ext},
         ...
     )
 

pyximport/pyxbuild.py

@@ -10,7 +10,7 @@ from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError
 from distutils.extension import Extension
 from distutils.util import grok_environment_error
 try:
-    from Cython.Distutils.build_ext import new_build_ext as build_ext
+    from Cython.Distutils.build_ext import build_ext
     HAS_CYTHON = True
 except ImportError:
     HAS_CYTHON = False

setup.py

@@ -159,7 +159,7 @@ def compile_cython_modules(profile=False, coverage=False, compile_more=False, cy
         # XXX hack around setuptools quirk for '*.pyx' sources
         extensions[-1].sources[0] = pyx_source_file
 
-    from Cython.Distutils.build_ext import new_build_ext
+    from Cython.Distutils.build_ext import build_ext
     from Cython.Compiler.Options import get_directive_defaults
     get_directive_defaults().update(
         language_level=2,
@@ -175,7 +175,7 @@ def compile_cython_modules(profile=False, coverage=False, compile_more=False, cy
         sys.stderr.write("Enabled line tracing and profiling for the Cython binary modules\n")
 
     # not using cythonize() directly to let distutils decide whether building extensions was requested
-    add_command_class("build_ext", new_build_ext)
+    add_command_class("build_ext", build_ext)
     setup_args['ext_modules'] = extensions
 
 

tests/build/build_ext_cython_c_in_temp.srctree

+
+PYTHON setup.py build_ext --inplace --cython-c-in-temp
+PYTHON -c 'import mymodule; assert mymodule.test_string == "TEST"'
+PYTHON check_paths.py
+
+############# setup.py #############
+
+from Cython.Distutils.extension import Extension
+from Cython.Build import build_ext
+from distutils.core import setup
+
+setup(
+    name='Hello world app',
+    ext_modules = [
+        Extension(
+            name = 'mymodule',
+            sources=['mymodule.pyx'],
+        )
+    ],
+    cmdclass={'build_ext': build_ext},
+)
+
+######## mymodule.pyx ########
+
+test_string = "TEST"
+
+######## check_paths.py ########
+
+import os
+assert not os.path.exists("mymodule.c")

tests/build/build_ext_cython_cplus.srctree

+# tag: cpp
+
+PYTHON setup.py build_ext --inplace --cython-cplus
+PYTHON -c "import a; a.use_vector([1,2,3])"
+
+######## setup.py ########
+
+from Cython.Distutils.extension import Extension
+from Cython.Build import build_ext
+from distutils.core import setup
+
+setup(
+    name='Hello world app',
+    ext_modules = [
+        Extension(
+            name = 'a',
+            sources=['a.pyx'],
+        )
+    ],
+    cmdclass={'build_ext': build_ext},
+)
+
+######## a.pyx ########
+
+from libcpp.vector cimport vector
+
+def use_vector(L):
+    try:
+        v = new vector[int]()
+        for a in L:
+            v.push_back(a)
+        return v.size()
+    finally:
+        del v

tests/build/build_ext_cython_include_dirs.srctree

+
+PYTHON setup.py build_ext --inplace --cython-include-dirs=./headers1 --include-dirs=./headers2
+PYTHON -c 'import mymodule; assert mymodule.test_string == "TEST"; assert mymodule.header_value1 == 1; assert mymodule.header_value2 == 2; assert mymodule.header_value3 == 3; assert mymodule.header_value4 == 4'
+
+############# setup.py #############
+
+from Cython.Distutils.extension import Extension
+from Cython.Build import build_ext
+from distutils.core import setup
+
+setup(
+    name='Hello world app',
+    ext_modules = [
+        Extension(
+            name = 'mymodule',
+            sources=['mymodule.pyx'],
+            cython_include_dirs=['headers3'],
+            include_dirs=['./headers4']
+        )
+    ],
+    cmdclass={'build_ext': build_ext},
+)
+
+######## mymodule.pyx ########
+
+include "myheader1.pxi"
+include "myheader2.pxi"
+include "myheader3.pxi"
+include "myheader4.pxi"
+header_value1 = test_value1
+header_value2 = test_value2
+header_value3 = test_value3
+header_value4 = test_value4
+test_string = "TEST"
+
+######## headers1/myheader1.pxi ########
+
+cdef int test_value1 = 1
+
+######## headers2/myheader2.pxi ########
+
+cdef int test_value2 = 2
+
+######## headers3/myheader3.pxi ########
+
+cdef int test_value3 = 3
+
+######## headers4/myheader4.pxi ########
+
+cdef int test_value4 = 4