Commit 8e18e1c0 authored by Robert Bradshaw's avatar Robert Bradshaw

Parallel cythonization.

parent 345db432
......@@ -340,9 +340,7 @@ def create_extension_list(filepatterns, ctx=None):
module_list.append(Extension(name=name, sources=[file], **deps.distutils_info(file).values))
return module_list
cython_py = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../cython.py'))
def cythonize(module_list, ctx=None):
def cythonize(module_list, ctx=None, nthreads=0):
deps = create_dependency_tree(ctx)
to_compile = []
for m in module_list:
......@@ -358,6 +356,8 @@ def cythonize(module_list, ctx=None):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
......@@ -373,8 +373,27 @@ def cythonize(module_list, ctx=None):
m.sources = new_sources
to_compile.sort()
# TODO: invoke directly
for priority, pyx_file, c_file in to_compile:
cmd = "%s %s %s -o %s" % (sys.executable, cython_py, pyx_file, c_file)
print cmd
os.system(cmd)
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
except ImportError:
print "multiprocessing required for parallel cythonization"
nthreads = 0
pool = multiprocessing.Pool(nthreads)
pool.map(cythonoize_one_helper, to_compile)
if not nthreads:
for priority, pyx_file, c_file in to_compile:
cythonoize_one(pyx_file, c_file)
return module_list
cython_py = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../cython.py'))
def cythonoize_one(pyx_file, c_file):
cmd = "%s %s %s -o %s" % (sys.executable, cython_py, pyx_file, c_file)
print cmd
if os.system(cmd) != 0:
raise CompilerError, pyx_file
def cythonoize_one_helper(m):
return cythonoize_one(*m[1:])
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment