Commit d1de0001 authored by Stefan Behnel's avatar Stefan Behnel

rewrite of pyximport module based on PEP 302 import hooks

parent e6f712b6
""" """
Import hooks; when installed (with the install()) function, these hooks Import hooks; when installed with the install() function, these hooks
allow importing .pyx files as if they were Python modules. allow importing .pyx files as if they were Python modules.
If you want the hook installed every time you run Python If you want the hook installed every time you run Python
you can add it to your Python version by adding these lines to you can add it to your Python version by adding these lines to
sitecustomize.py (which you can create from scratch in site-packages sitecustomize.py (which you can create from scratch in site-packages
if it doesn't exist there are somewhere else on your python path) if it doesn't exist there or somewhere else on your python path)::
import pyximport import pyximport
pyximport.install() pyximport.install()
For instance on the Mac with Python 2.3 built from CVS, you could For instance on the Mac with a non-system Python 2.3, you could create
create sitecustomize.py with only those two lines at sitecustomize.py with only those two lines at
/usr/local/lib/python2.3/site-packages/sitecustomize.py . /usr/local/lib/python2.3/site-packages/sitecustomize.py .
Running this module as a top-level script will run a test and then print Running this module as a top-level script will run a test and then print
the documentation. the documentation.
This code was modeled on Quixote's ptl_import. This code is based on the Py2.3+ import protocol as described in PEP 302.
""" """
import sys, os, shutil import sys
import imp, ihooks, glob import os
import __builtin__ import glob
import imp
import pyxbuild import pyxbuild
from distutils.dep_util import newer from distutils.dep_util import newer
from distutils.extension import Extension from distutils.extension import Extension
...@@ -33,19 +34,11 @@ except ImportError: ...@@ -33,19 +34,11 @@ except ImportError:
mod_name = "pyximport" mod_name = "pyximport"
assert sys.hexversion >= 0x20000b1, "need Python 2.0b1 or later" assert sys.hexversion >= 0x2030000, "need Python 2.3 or later"
PYX_FILE_TYPE = 1011
PYX_EXT = ".pyx" PYX_EXT = ".pyx"
PYXDEP_EXT = ".pyxdep" PYXDEP_EXT = ".pyxdep"
PYXBLD_EXT = ".pyxbld" PYXBLD_EXT = ".pyxbld"
_test_files = []
class PyxHooks (ihooks.Hooks):
"""Import hook that declares our suffixes. Let install() install it."""
def get_suffixes (self):
# add our suffixes
return imp.get_suffixes() + [(PYX_EXT, "r", PYX_FILE_TYPE)]
# Performance problem: for every PYX file that is imported, we will # Performance problem: for every PYX file that is imported, we will
# invoke the whole distutils infrastructure even if the module is # invoke the whole distutils infrastructure even if the module is
...@@ -58,33 +51,29 @@ def _load_pyrex(name, filename): ...@@ -58,33 +51,29 @@ def _load_pyrex(name, filename):
"Load a pyrex file given a name and filename." "Load a pyrex file given a name and filename."
def get_distutils_extension(modname, pyxfilename): def get_distutils_extension(modname, pyxfilename):
extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest() extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
# modname = modname + extra # modname = modname + extra
extension_mod = handle_special_build(modname, pyxfilename) extension_mod = handle_special_build(modname, pyxfilename)
if not extension_mod: if not extension_mod:
extension_mod = Extension(name = modname, sources=[pyxfilename]) extension_mod = Extension(name = modname, sources=[pyxfilename])
return extension_mod return extension_mod
def handle_special_build(modname, pyxfilename): def handle_special_build(modname, pyxfilename):
special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
if not os.path.exists(special_build): if not os.path.exists(special_build):
ext = None ext = None
else: else:
globls = {} globls = {}
locs = {} locs = {}
# execfile(special_build, globls, locs) # execfile(special_build, globls, locs)
# ext = locs["make_ext"](modname, pyxfilename) # ext = locs["make_ext"](modname, pyxfilename)
mod = imp.load_source("XXXX", special_build, open(special_build)) mod = imp.load_source("XXXX", special_build, open(special_build))
ext = mod.make_ext(modname, pyxfilename) ext = mod.make_ext(modname, pyxfilename)
assert ext and ext.sources, ("make_ext in %s did not return Extension" assert ext and ext.sources, ("make_ext in %s did not return Extension"
% special_build) % special_build)
ext.sources = [os.path.join(os.path.dirname(special_build), source) ext.sources = [os.path.join(os.path.dirname(special_build), source)
for source in ext.sources] for source in ext.sources]
return ext return ext
def handle_dependencies(pyxfilename): def handle_dependencies(pyxfilename):
...@@ -96,33 +85,33 @@ def handle_dependencies(pyxfilename): ...@@ -96,33 +85,33 @@ def handle_dependencies(pyxfilename):
# but we know more about dependencies so force a rebuild if # but we know more about dependencies so force a rebuild if
# some of the dependencies are newer than the pyxfile. # some of the dependencies are newer than the pyxfile.
if os.path.exists(dependfile): if os.path.exists(dependfile):
depends = open(dependfile).readlines() depends = open(dependfile).readlines()
depends = [depend.strip() for depend in depends] depends = [depend.strip() for depend in depends]
# gather dependencies in the "files" variable # gather dependencies in the "files" variable
# the dependency file is itself a dependency # the dependency file is itself a dependency
files = [dependfile] files = [dependfile]
for depend in depends: for depend in depends:
fullpath = os.path.join(os.path.dirname(dependfile), fullpath = os.path.join(os.path.dirname(dependfile),
depend) depend)
files.extend(glob.glob(fullpath)) files.extend(glob.glob(fullpath))
# only for unit testing to see we did the right thing # only for unit testing to see we did the right thing
_test_files[:] = [] _test_files[:] = []
# if any file that the pyxfile depends upon is newer than # if any file that the pyxfile depends upon is newer than
# the pyx file, 'touch' the pyx file so that distutils will # the pyx file, 'touch' the pyx file so that distutils will
# be tricked into rebuilding it. # be tricked into rebuilding it.
for file in files: for file in files:
if newer(file, pyxfilename): if newer(file, pyxfilename):
print "Rebuilding because of ", file print "Rebuilding because of ", file
filetime = os.path.getmtime(file) filetime = os.path.getmtime(file)
os.utime(pyxfilename, (filetime, filetime)) os.utime(pyxfilename, (filetime, filetime))
_test_files.append(file) _test_files.append(file)
def build_module(name, pyxfilename): def build_module(name, pyxfilename):
assert os.path.exists(pyxfilename), ( assert os.path.exists(pyxfilename), (
"Path does not exist: %s" % pyxfilename) "Path does not exist: %s" % pyxfilename)
handle_dependencies(pyxfilename) handle_dependencies(pyxfilename)
extension_mod = get_distutils_extension(name, pyxfilename) extension_mod = get_distutils_extension(name, pyxfilename)
...@@ -133,11 +122,11 @@ def build_module(name, pyxfilename): ...@@ -133,11 +122,11 @@ def build_module(name, pyxfilename):
junkpath = os.path.join(os.path.dirname(so_path), name+"_*") junkpath = os.path.join(os.path.dirname(so_path), name+"_*")
junkstuff = glob.glob(junkpath) junkstuff = glob.glob(junkpath)
for path in junkstuff: for path in junkstuff:
if path!=so_path: if path!=so_path:
try: try:
os.remove(path) os.remove(path)
except IOError: except IOError:
print "Couldn't remove ", path print "Couldn't remove ", path
return so_path return so_path
...@@ -147,53 +136,64 @@ def load_module(name, pyxfilename): ...@@ -147,53 +136,64 @@ def load_module(name, pyxfilename):
assert mod.__file__ == so_path, (mod.__file__, so_path) assert mod.__file__ == so_path, (mod.__file__, so_path)
return mod return mod
class PyxLoader (ihooks.ModuleLoader):
"""Load a module. It checks whether a file is a .pyx and returns it. # import hooks
Otherwise it lets the ihooks base class handle it. Let install()
install it.""" class PyxImporter(object):
def __init__(self, so_cache_dir):
def load_module (self, name, stuff): self.so_cache_dir = so_cache_dir
# If it's a Pyrex file, load it specially. self.so_name_cache = {}
if stuff[2][2] == PYX_FILE_TYPE:
file, pyxfilename, info = stuff def find_module(self, fullname, package_path=None):
(suff, mode, type) = info print "SEARCHING", fullname, package_path
if file: if '.' in fullname:
file.close() mod_parts = fullname.split('.')
return load_module(name, pyxfilename) package = '.'.join(mod_parts[:-1])
module_name = mod_parts[-1]
else:
package = None
module_name = fullname
pyx_module_name = module_name + PYX_EXT
# this may work, but it returns the file content, not its path
#import pkgutil
#pyx_source = pkgutil.get_data(package, pyx_module_name)
if package_path:
paths = package_path
else: else:
# Otherwise, use the default handler for loading paths = sys.path
return ihooks.ModuleLoader.load_module( self, name, stuff) join_path = os.path.join
is_file = os.path.isfile
for path in filter(os.path.isdir, paths):
for filename in os.listdir(path):
if filename == pyx_module_name:
return PyxLoader(fullname, join_path(path, filename))
elif filename == module_name:
package_path = join_path(path, filename)
init_path = join_path(package_path, '__init__' + PYX_EXT)
if is_file(init_path):
return PyxLoader(fullname, package_path, init_path)
# not found, normal package, not a .pyx file, none of our business
return None
class PyxLoader(object):
def __init__(self, fullname, path, init_path=None):
self.fullname, self.path, self.init_path = fullname, path, init_path
def load_module(self, fullname):
assert self.fullname == fullname, (
"invalid module, expected %s, got %s" % (
self.fullname, fullname))
if self.init_path:
# package
print "PACKAGE", fullname
module = load_module(fullname, self.init_path)
module.__path__ = [self.path]
else:
print "MODULE", fullname
module = load_module(fullname, self.path)
return module
try:
import cimport
except ImportError:
cimport = None
class cModuleImporter(ihooks.ModuleImporter):
"""This was just left in from the Quixote implementation. I think
it allows a performance enhancement if you have the cimport module
from Quixote. Let install() install it."""
def __init__(self, loader=None):
self.loader = loader or ihooks.ModuleLoader()
cimport.set_loader(self.find_import_module)
def find_import_module(self, fullname, subname, path):
stuff = self.loader.find_module(subname, path)
if not stuff:
return None
return self.loader.load_module(fullname, stuff)
def install(self):
self.save_import_module = __builtin__.__import__
self.save_reload = __builtin__.reload
if not hasattr(__builtin__, 'unload'):
__builtin__.unload = None
self.save_unload = __builtin__.unload
__builtin__.__import__ = cimport.import_module
__builtin__.reload = cimport.reload_module
__builtin__.unload = self.unload
_installed = 0
def install(): def install():
"""Main entry point. call this to install the import hook in your """Main entry point. call this to install the import hook in your
...@@ -201,22 +201,14 @@ def install(): ...@@ -201,22 +201,14 @@ def install():
you use Python, add it to your sitecustomize (as described above). you use Python, add it to your sitecustomize (as described above).
""" """
global _installed for importer in sys.meta_path:
if not _installed: if isinstance(importer, PyxImporter):
hooks = PyxHooks() return
loader = PyxLoader(hooks) importer = PyxImporter('~/.pyxbuild')
if cimport is not None: sys.meta_path.append(importer)
importer = cModuleImporter(loader)
else:
importer = ihooks.ModuleImporter(loader) # MAIN
ihooks.install(importer)
_installed = 1
def on_remove_file_error(func, path, excinfo):
print "Sorry! Could not remove a temp file:", path
print "Extra information."
print func, excinfo
print "You may want to delete this yourself when you get a chance."
def show_docs(): def show_docs():
import __main__ import __main__
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment