Commit b16f44a4 authored by Stefan Behnel's avatar Stefan Behnel

applied pyximport patch from ticket 312

parent a7e5883e
......@@ -6,7 +6,6 @@ out_fname = pyx_to_dll("foo.pyx")
import os
import sys
import distutils
from distutils.dist import Distribution
from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError
from distutils.extension import Extension
......@@ -16,11 +15,13 @@ try:
HAS_CYTHON = True
except ImportError:
HAS_CYTHON = False
import shutil
DEBUG = 0
_reloads={}
def pyx_to_dll(filename, ext = None, force_rebuild = 0,
build_in_temp=False, pyxbuild_dir=None):
build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False):
"""Compile a PYX file to a DLL and return the name of the generated .so
or .dll ."""
assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename)
......@@ -37,7 +38,8 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
if not pyxbuild_dir:
pyxbuild_dir = os.path.join(path, "_pyxbld")
if DEBUG:
script_args=setup_args.get("script_args",[])
if DEBUG or "--verbose" in script_args:
quiet = "--verbose"
else:
quiet = "--quiet"
......@@ -46,7 +48,11 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
args.append("--force")
if HAS_CYTHON and build_in_temp:
args.append("--pyrex-c-in-temp")
dist = Distribution({"script_name": None, "script_args": args})
sargs = setup_args.copy()
sargs.update(
{"script_name": None,
"script_args": args + script_args} )
dist = Distribution(sargs)
if not dist.ext_modules:
dist.ext_modules = []
dist.ext_modules.append(ext)
......@@ -60,6 +66,10 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
except ValueError: pass
dist.parse_config_files(config_files)
cfgfiles = dist.find_config_files()
try: cfgfiles.remove('setup.cfg')
except ValueError: pass
dist.parse_config_files(cfgfiles)
try:
ok = dist.parse_command_line()
except DistutilsArgError:
......@@ -73,7 +83,39 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
try:
dist.run_commands()
return dist.get_command_obj("build_ext").get_outputs()[0]
obj_build_ext = dist.get_command_obj("build_ext")
so_path = obj_build_ext.get_outputs()[0]
if obj_build_ext.inplace:
# Python distutils get_outputs()[ returns a wrong so_path
# when --inplace ; see http://bugs.python.org/issue5977
# workaround:
so_path = os.path.join(os.path.dirname(filename),
os.path.basename(so_path))
if reload_support:
org_path = so_path
timestamp = os.path.getmtime(org_path)
global _reloads
last_timestamp, last_path, count = _reloads.get(org_path, (None,None,0) )
if last_timestamp == timestamp:
so_path = last_path
else:
basename = os.path.basename(org_path)
while count < 100:
count += 1
r_path = os.path.join(obj_build_ext.build_lib,
basename + '.reload%s'%count)
try:
import shutil # late import / reload_support is: debugging
shutil.copy2(org_path, r_path)
so_path = r_path
except IOError:
continue
break
else:
# used up all 100 slots
raise ImportError("reload count for %s reached maximum"%org_path)
_reloads[org_path]=(timestamp, so_path, count)
return so_path
except KeyboardInterrupt:
sys.exit(1)
except (IOError, os.error):
......@@ -83,15 +125,6 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
if DEBUG:
sys.stderr.write(error + "\n")
raise
else:
raise RuntimeError(error)
except (DistutilsError, CCompilerError):
if DEBUG:
raise
else:
exc = sys.exc_info()[1]
raise RuntimeError(repr(exc))
if __name__=="__main__":
pyx_to_dll("dummy.pyx")
......
......@@ -14,6 +14,22 @@ For instance on the Mac with a non-system Python 2.3, you could create
sitecustomize.py with only those two lines at
/usr/local/lib/python2.3/site-packages/sitecustomize.py .
A custom distutils.core.Extension instance and setup() args
(Distribution) for for the build can be defined by a <modulename>.pyxbld
file like:
# examplemod.pyxbdl
def make_ext(modname, pyxfilename):
from distutils.extension import Extension
return Extension(name = modname,
sources=[pyxfilename, 'hello.c'],
include_dirs=['/myinclude'] )
def make_setup_args():
return dict(script_args=["--compiler=mingw32"])
Extra dependencies can be defined by a <modulename>.pyxdep .
See README.
Since Cython 0.11, the :mod:`pyximport` module also has experimental
compilation support for normal Python modules. This allows you to
automatically run Cython on every .pyx and .py module that Python
......@@ -30,18 +46,11 @@ the documentation.
This code is based on the Py2.3+ import protocol as described in PEP 302.
"""
import sys
import os
import glob
import imp
import pyxbuild
from distutils.dep_util import newer
from distutils.extension import Extension
try:
import hashlib
except ImportError:
import md5 as hashlib
mod_name = "pyximport"
......@@ -64,30 +73,43 @@ def _load_pyrex(name, filename):
"Load a pyrex file given a name and filename."
def get_distutils_extension(modname, pyxfilename):
extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
# try:
# import hashlib
# except ImportError:
# import md5 as hashlib
# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
# modname = modname + extra
extension_mod = handle_special_build(modname, pyxfilename)
extension_mod,setup_args = handle_special_build(modname, pyxfilename)
if not extension_mod:
from distutils.extension import Extension
extension_mod = Extension(name = modname, sources=[pyxfilename])
return extension_mod
return extension_mod,setup_args
def handle_special_build(modname, pyxfilename):
special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
if not os.path.exists(special_build):
ext = None
else:
globls = {}
locs = {}
setup_args={}
if os.path.exists(special_build):
# globls = {}
# locs = {}
# execfile(special_build, globls, locs)
# ext = locs["make_ext"](modname, pyxfilename)
mod = imp.load_source("XXXX", special_build, open(special_build))
ext = mod.make_ext(modname, pyxfilename)
make_ext = getattr(mod,'make_ext',None)
if make_ext:
ext = make_ext(modname, pyxfilename)
assert ext and ext.sources, ("make_ext in %s did not return Extension"
% special_build)
make_setup_args = getattr(mod,'make_setup_args',None)
if make_setup_args:
setup_args = make_setup_args()
assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict"
% special_build)
assert set or setup_args, ("neither make_ext nor make_setup_args %s"
% special_build)
ext.sources = [os.path.join(os.path.dirname(special_build), source)
for source in ext.sources]
return ext
return ext, setup_args
def handle_dependencies(pyxfilename):
dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
......@@ -110,12 +132,13 @@ def handle_dependencies(pyxfilename):
files.extend(glob.glob(fullpath))
# only for unit testing to see we did the right thing
_test_files[:] = []
_test_files[:] = [] #$pycheck_no
# if any file that the pyxfile depends upon is newer than
# the pyx file, 'touch' the pyx file so that distutils will
# be tricked into rebuilding it.
for file in files:
from distutils.dep_util import newer
if newer(file, pyxfilename):
print("Rebuilding because of ", file)
filetime = os.path.getmtime(file)
......@@ -127,14 +150,21 @@ def build_module(name, pyxfilename, pyxbuild_dir=None):
"Path does not exist: %s" % pyxfilename)
handle_dependencies(pyxfilename)
extension_mod = get_distutils_extension(name, pyxfilename)
extension_mod,setup_args = get_distutils_extension(name, pyxfilename)
build_in_temp=pyxargs.build_in_temp
sargs=pyxargs.setup_args.copy()
sargs.update(setup_args)
build_in_temp=sargs.pop('build_in_temp',build_in_temp)
import pyxbuild
so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
build_in_temp=True,
pyxbuild_dir=pyxbuild_dir)
build_in_temp=build_in_temp,
pyxbuild_dir=pyxbuild_dir,
setup_args=sargs,
reload_support=pyxargs.reload_support)
assert os.path.exists(so_path), "Cannot find: %s" % so_path
junkpath = os.path.join(os.path.dirname(so_path), name+"_*")
junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ?
junkstuff = glob.glob(junkpath)
for path in junkstuff:
if path!=so_path:
......@@ -151,7 +181,9 @@ def load_module(name, pyxfilename, pyxbuild_dir=None):
mod = imp.load_dynamic(name, so_path)
assert mod.__file__ == so_path, (mod.__file__, so_path)
except Exception, e:
raise ImportError("Building module failed: %s" % e)
import traceback
raise ImportError("Building module failed: %s" %
traceback.format_exception_only(*sys.exc_info()[:2])),None,sys.exc_info()[2]
return mod
......@@ -165,16 +197,33 @@ class PyxImporter(object):
self.pyxbuild_dir = pyxbuild_dir
def find_module(self, fullname, package_path=None):
if fullname in sys.modules:
if fullname in sys.modules and not pyxargs.reload_support:
return None # only here when reload()
try:
fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
if fp: fp.close() # Python should offer a Default-Loader to avoid this double find/open!
if ty!=imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
return None
if DEBUG_IMPORT:
print("SEARCHING", fullname, package_path)
if '.' in fullname:
# find .pyx fast, when .so/.pyd exist --inplace
pyxpath = os.path.splitext(pathname)[0]+self.extension
if os.path.isfile(pyxpath):
return PyxLoader(fullname, pyxpath,
pyxbuild_dir=self.pyxbuild_dir)
# .so/.pyd's on PATH should not be remote from .pyx's
# think no need to implement PyxArgs.importer_search_remote here?
except ImportError:
pass
# searching sys.path ...
#if DEBUG_IMPORT: print "SEARCHING", fullname, package_path
if '.' in fullname: # only when package_path anyway?
mod_parts = fullname.split('.')
package = '.'.join(mod_parts[:-1])
module_name = mod_parts[-1]
else:
package = None
module_name = fullname
pyx_module_name = module_name + self.extension
# this may work, but it returns the file content, not its path
......@@ -187,24 +236,15 @@ class PyxImporter(object):
paths = sys.path
join_path = os.path.join
is_file = os.path.isfile
is_dir = os.path.isdir
#is_dir = os.path.isdir
sep = os.path.sep
for path in paths:
if not is_dir(path):
if not path:
path = os.getcwd()
else:
continue
for filename in os.listdir(path):
if filename == pyx_module_name:
return PyxLoader(fullname, join_path(path, filename),
pyxbuild_dir=self.pyxbuild_dir)
elif filename == module_name:
package_path = join_path(path, filename)
init_path = join_path(package_path,
'__init__' + self.extension)
if is_file(init_path):
return PyxLoader(fullname, package_path, init_path,
if is_file(path+sep+pyx_module_name):
return PyxLoader(fullname, join_path(path, pyx_module_name),
pyxbuild_dir=self.pyxbuild_dir)
# not found, normal package, not a .pyx file, none of our business
return None
......@@ -289,7 +329,16 @@ class PyxLoader(object):
return module
def install(pyximport=True, pyimport=False, build_dir=None):
#install args
class PyxArgs(object):
build_dir=True
build_in_temp=True
setup_args={} #None
##pyxargs=None
def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
setup_args={}, reload_support=False ):
"""Main entry point. Call this to install the .pyx import hook in
your meta-path for a single Python process. If you want it to be
installed whenever you use Python, add it to your sitecustomize
......@@ -303,10 +352,32 @@ def install(pyximport=True, pyimport=False, build_dir=None):
By default, compiled modules will end up in a ``.pyxbld``
directory in the user's home directory. Passing a different path
as ``build_dir`` will override this.
``build_in_temp=False`` will produce the C files locally. Working
with complex dependencies and debugging becomes more easy. This
can principally interfere with existing files of the same name.
build_in_temp can be overriden by <modulename>.pyxbld/make_setup_args()
by a dict item of 'build_in_temp'
``setup_args``: dict of arguments for Distribution - see
distutils.core.setup() . They are extended/overriden by those of
<modulename>.pyxbld/make_setup_args()
``reload_support``: Enables support for dynamic
reload(<pyxmodulename>), e.g. after a change in the Cython code.
Additional files <so_path>.reloadNN may arise on that account, when
the previously loaded module file cannot be overwritten.
"""
if not build_dir:
build_dir = os.path.expanduser('~/.pyxbld')
global pyxargs
pyxargs = PyxArgs() #$pycheck_no
pyxargs.build_dir = build_dir
pyxargs.build_in_temp = build_in_temp
pyxargs.setup_args = (setup_args or {}).copy()
pyxargs.reload_support = reload_support
has_py_importer = False
has_pyx_importer = False
for importer in sys.meta_path:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment