Commit 228fc4c9 authored by Stefan Behnel's avatar Stefan Behnel

implemented 'ignore compile failures' mode in cythonize() to do trial+error...

implemented 'ignore compile failures' mode in cythonize() to do trial+error compilation of .py files

--HG--
extra : rebase_source : d415ce5d3dc3a3504b2bf4e8fc1ffd0cd16de64e
parent 09a5e0fd
...@@ -565,7 +565,29 @@ def create_extension_list(patterns, exclude=[], ctx=None, aliases=None, quiet=Fa ...@@ -565,7 +565,29 @@ def create_extension_list(patterns, exclude=[], ctx=None, aliases=None, quiet=Fa
return module_list return module_list
# This is the user-exposed entry point. # This is the user-exposed entry point.
def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False, **options): def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False,
exclude_failures=False, **options):
"""
Compile a set of source modules into C/C++ files and return a list of distutils
Extension objects for them.
As module list, pass either a glob pattern, a list of glob patterns or a list of
Extension objects. The latter allows you to configure the extensions separately
through the normal distutils options.
When using glob patterns, you can exclude certain module names explicitly
by passing them into the 'exclude' option.
For parallel compilation, set the 'nthreads' option to the number of
concurrent builds.
For a broad 'try to compile' mode that ignores compilation failures and
simply excludes the failed extensions, pass 'exclude_failures=True'. Note
that this only really makes sense for compiling .py files which can also
be used without compilation.
Additional compilation options can be passed as keyword arguments.
"""
if 'include_path' not in options: if 'include_path' not in options:
options['include_path'] = ['.'] options['include_path'] = ['.']
c_options = CompilationOptions(**options) c_options = CompilationOptions(**options)
...@@ -578,6 +600,7 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo ...@@ -578,6 +600,7 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
quiet=quiet, quiet=quiet,
aliases=aliases) aliases=aliases)
deps = create_dependency_tree(ctx, quiet=quiet) deps = create_dependency_tree(ctx, quiet=quiet)
modules_by_cfile = {}
to_compile = [] to_compile = []
for m in module_list: for m in module_list:
new_sources = [] new_sources = []
...@@ -614,8 +637,13 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo ...@@ -614,8 +637,13 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
fingerprint = deps.transitive_fingerprint(source, extra) fingerprint = deps.transitive_fingerprint(source, extra)
else: else:
fingerprint = None fingerprint = None
to_compile.append((priority, source, c_file, fingerprint, quiet, options)) to_compile.append((priority, source, c_file, fingerprint, quiet,
options, not exclude_failures))
new_sources.append(c_file) new_sources.append(c_file)
if c_file not in modules_by_cfile:
modules_by_cfile[c_file] = [m]
else:
modules_by_cfile[c_file].append(m)
else: else:
new_sources.append(source) new_sources.append(source)
m.sources = new_sources m.sources = new_sources
...@@ -635,12 +663,19 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo ...@@ -635,12 +663,19 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
if not nthreads: if not nthreads:
for args in to_compile: for args in to_compile:
cythonize_one(*args[1:]) cythonize_one(*args[1:])
if exclude_failures:
failed_modules = set()
for c_file, modules in modules_by_cfile.iteritems():
if not os.path.exists(c_file):
failed_modules.update(modules)
for module in failed_modules:
module_list.remove(module)
if hasattr(options, 'cache'): if hasattr(options, 'cache'):
cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100)) cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
return module_list return module_list
# TODO: Share context? Issue: pyx processing leaks into pxd module # TODO: Share context? Issue: pyx processing leaks into pxd module
def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None): def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True):
from Cython.Compiler.Main import compile, default_options from Cython.Compiler.Main import compile, default_options
from Cython.Compiler.Errors import CompileError, PyrexError from Cython.Compiler.Errors import CompileError, PyrexError
...@@ -683,9 +718,9 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None): ...@@ -683,9 +718,9 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None):
except (EnvironmentError, PyrexError), e: except (EnvironmentError, PyrexError), e:
sys.stderr.write('%s\n' % e) sys.stderr.write('%s\n' % e)
any_failures = 1 any_failures = 1
if any_failures: if any_failures and raise_on_failure:
raise CompileError(None, pyx_file) raise CompileError(None, pyx_file)
if fingerprint: if fingerprint and not any_failures:
f = open(c_file, 'rb') f = open(c_file, 'rb')
try: try:
g = gzip_open(fingerprint_file, 'wb') g = gzip_open(fingerprint_file, 'wb')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment