Merge pull request #466 from jdemeyer/metadata_hook

Add create_extension() hook

Cython/Build/Dependencies.py

@@ -637,6 +637,20 @@ def create_dependency_tree(ctx=None, quiet=False):
     return _dep_tree
 
 
+# If this changes, change also docs/src/reference/compilation.rst
+# which mentions this function
+def default_create_extension(template, kwds):
+    if 'depends' in kwds:
+        include_dirs = kwds.get('include_dirs', []) + ["."]
+        depends = resolve_depends(kwds['depends'], include_dirs)
+        kwds['depends'] = sorted(set(depends + template.depends))
+
+    t = template.__class__
+    ext = t(**kwds)
+    metadata = dict(distutils=kwds, module_name=kwds['name'])
+    return (ext, metadata)
+
+
 # This may be useful for advanced users?
 def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=False, language=None,
                           exclude_failures=False):
@@ -669,13 +683,16 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
         Extension_distutils = Extension
         class Extension_setuptools(Extension): pass
 
+    # if no create_extension() function is defined, use a simple
+    # default function.
+    create_extension = ctx.options.create_extension or default_create_extension
+
     for pattern in patterns:
         if isinstance(pattern, str):
             filepattern = pattern
-            template = None
+            template = Extension(pattern, [])  # Fake Extension without sources
             name = '*'
             base = None
-            exn_type = Extension
             ext_language = language
         elif isinstance(pattern, (Extension_distutils, Extension_setuptools)):
             cython_sources = [s for s in pattern.sources
@@ -693,7 +710,6 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
             template = pattern
             name = template.name
             base = DistutilsInfo(exn=template)
-            exn_type = template.__class__
             ext_language = None  # do not override whatever the Extension says
         else:
             msg = str("pattern is not of type str nor subclass of Extension (%s)"
@@ -727,39 +743,28 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
                         if key not in kwds:
                             kwds[key] = value
 
-                sources = [file]
-                if template is not None:
-                    sources += [m for m in template.sources if m != filepattern]
+                kwds['name'] = module_name
+
+                sources = [file] + [m for m in template.sources if m != filepattern]
                 if 'sources' in kwds:
                     # allow users to add .c files etc.
                     for source in kwds['sources']:
                         source = encode_filename_in_py2(source)
                         if source not in sources:
                             sources.append(source)
-                    extra_sources = kwds['sources']
-                    del kwds['sources']
-                else:
-                    extra_sources = None
-                if 'depends' in kwds:
-                    depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + ["."])
-                    if template is not None:
-                        # Always include everything from the template.
-                        depends = set(template.depends).union(depends)
-                    # Sort depends to make the metadata dump in the
-                    # Cython-generated C code predictable.
-                    kwds['depends'] = sorted(depends)
+                kwds['sources'] = sources
 
                 if ext_language and 'language' not in kwds:
                     kwds['language'] = ext_language
 
-                module_list.append(exn_type(
-                        name=module_name,
-                        sources=sources,
-                        **kwds))
-                if extra_sources:
-                    kwds['sources'] = extra_sources
-                module_metadata[module_name] = {'distutils': kwds, 'module_name': module_name}
-                m = module_list[-1]
+                # Create the new extension
+                m, metadata = create_extension(template, kwds)
+                module_list.append(m)
+
+                # Store metadata (this will be written as JSON in the
+                # generated C file but otherwise has no purpose)
+                module_metadata[module_name] = metadata
+
                 if file not in m.sources:
                     # Old setuptools unconditionally replaces .pyx with .c
                     m.sources.remove(file.rsplit('.')[0] + '.c')

Cython/Compiler/Main.py

@@ -753,4 +753,5 @@ default_options = dict(
     output_dir=None,
     build_dir=None,
     cache=None,
+    create_extension=None,
 )

docs/src/reference/compilation.rst

@@ -126,11 +126,15 @@ both might disagree about the class to use here.
 
 If your options are static (for example you do not need to call a tool like
 ``pkg-config`` to determine them) you can also provide them directly in your
-.pyx source file using a special comment block at the start of the file::
+.pyx or .pxd source file using a special comment block at the start of the file::
 
     # distutils: libraries = spam eggs
     # distutils: include_dirs = /opt/food/include
 
+If you cimport multiple .pxd files defining libraries, then Cython
+merges the list of libraries, so this works as expected (similarly
+with other options, like ``include_dirs`` above).
+
 If you have some C files that have been wrapped with Cython and you want to
 compile them into your extension, you can define the distutils ``sources``
 parameter::
@@ -160,6 +164,57 @@ to find the ``.h`` and library files when linking to external libraries.
 
 .. _distutils documentation: http://docs.python.org/extending/building.html
 
+Sometimes this is not enough and you need finer customization of the
+distutils :class:`Extension`.
+To do this, you can provide a custom function ``create_extension``
+to create the final :class:`Extension` object after Cython has processed
+the sources, dependencies and ``# distutils`` directives but before the
+file is actually Cythonized.
+This function takes 2 arguments ``template`` and ``kwds``, where
+``template`` is the :class:`Extension` object given as input to Cython
+and ``kwds`` is a :class:`dict` with all keywords which should be used
+to create the :class:`Extension`.
+The function ``create_extension`` must return a 2-tuple
+``(extension, metadata)``, where ``extension`` is the created
+:class:`Extension` and ``metadata`` is metadata which will be written
+as JSON at the top of the generated C files. This metadata is only used
+for debugging purposes, so you can put whatever you want in there
+(as long as it can be converted to JSON).
+The default function (defined in ``Cython.Build.Dependencies``) is::
+
+    def default_create_extension(template, kwds):
+        if 'depends' in kwds:
+            include_dirs = kwds.get('include_dirs', []) + ["."]
+            depends = resolve_depends(kwds['depends'], include_dirs)
+            kwds['depends'] = sorted(set(depends + template.depends))
+
+        t = template.__class__
+        ext = t(**kwds)
+        metadata = dict(distutils=kwds, module_name=kwds['name'])
+        return (ext, metadata)
+
+In case that you pass a string instead of an :class:`Extension` to
+``cythonize()``, the ``template`` will be an :class:`Extension` without
+sources. For example, if you do ``cythonize("*.pyx")``,
+the ``template`` will be ``Extension(name="*.pyx", sources=[])``.
+
+Just as an example, this adds ``mylib`` as library to every extension::
+
+    from Cython.Build.Dependencies import default_create_extension
+
+    def my_create_extension(template, kwds):
+        libs = kwds.get('libraries', []) + ["mylib"]
+        kwds['libraries'] = libs
+        return default_create_extension(template, kwds)
+
+    ext_modules = cythonize(..., create_extension=my_create_extension)
+
+.. note::
+
+    If you Cythonize in parallel (using the ``nthreads`` argument),
+    then the argument to ``create_extension`` must be pickleable.
+    In particular, it cannot be a lambda function.
+
 
 Distributing Cython modules
 ----------------------------

tests/compile/create_extension.srctree

+PYTHON setup.py build_ext --inplace
+
+######## setup.py ########
+
+from Cython.Build import cythonize
+from Cython.Distutils.extension import Extension
+
+ext_modules = [
+    Extension("foo", ["foo.pyx"]),
+]
+
+# Example documented in docs/src/reference/compilation.rst
+from Cython.Build.Dependencies import default_create_extension
+
+def my_create_extension(template, kwds):
+    libs = kwds.get('libraries', []) + ["mylib"]
+    kwds['libraries'] = libs
+    return default_create_extension(template, kwds)
+
+ext_modules = cythonize(ext_modules, create_extension=my_create_extension)
+
+assert ext_modules[0].libraries == ["mylib"]
+
+######## foo.pyx ########