Commit d1de0001 authored by Stefan Behnel's avatar Stefan Behnel

rewrite of pyximport module based on PEP 302 import hooks

parent e6f712b6
""" """
Import hooks; when installed (with the install()) function, these hooks Import hooks; when installed with the install() function, these hooks
allow importing .pyx files as if they were Python modules. allow importing .pyx files as if they were Python modules.
If you want the hook installed every time you run Python If you want the hook installed every time you run Python
you can add it to your Python version by adding these lines to you can add it to your Python version by adding these lines to
sitecustomize.py (which you can create from scratch in site-packages sitecustomize.py (which you can create from scratch in site-packages
if it doesn't exist there are somewhere else on your python path) if it doesn't exist there or somewhere else on your python path)::
import pyximport import pyximport
pyximport.install() pyximport.install()
For instance on the Mac with Python 2.3 built from CVS, you could For instance on the Mac with a non-system Python 2.3, you could create
create sitecustomize.py with only those two lines at sitecustomize.py with only those two lines at
/usr/local/lib/python2.3/site-packages/sitecustomize.py . /usr/local/lib/python2.3/site-packages/sitecustomize.py .
Running this module as a top-level script will run a test and then print Running this module as a top-level script will run a test and then print
the documentation. the documentation.
This code was modeled on Quixote's ptl_import. This code is based on the Py2.3+ import protocol as described in PEP 302.
""" """
import sys, os, shutil import sys
import imp, ihooks, glob import os
import __builtin__ import glob
import imp
import pyxbuild import pyxbuild
from distutils.dep_util import newer from distutils.dep_util import newer
from distutils.extension import Extension from distutils.extension import Extension
...@@ -33,19 +34,11 @@ except ImportError: ...@@ -33,19 +34,11 @@ except ImportError:
mod_name = "pyximport" mod_name = "pyximport"
assert sys.hexversion >= 0x20000b1, "need Python 2.0b1 or later" assert sys.hexversion >= 0x2030000, "need Python 2.3 or later"
PYX_FILE_TYPE = 1011
PYX_EXT = ".pyx" PYX_EXT = ".pyx"
PYXDEP_EXT = ".pyxdep" PYXDEP_EXT = ".pyxdep"
PYXBLD_EXT = ".pyxbld" PYXBLD_EXT = ".pyxbld"
_test_files = []
class PyxHooks (ihooks.Hooks):
"""Import hook that declares our suffixes. Let install() install it."""
def get_suffixes (self):
# add our suffixes
return imp.get_suffixes() + [(PYX_EXT, "r", PYX_FILE_TYPE)]
# Performance problem: for every PYX file that is imported, we will # Performance problem: for every PYX file that is imported, we will
# invoke the whole distutils infrastructure even if the module is # invoke the whole distutils infrastructure even if the module is
...@@ -58,15 +51,11 @@ def _load_pyrex(name, filename): ...@@ -58,15 +51,11 @@ def _load_pyrex(name, filename):
"Load a pyrex file given a name and filename." "Load a pyrex file given a name and filename."
def get_distutils_extension(modname, pyxfilename): def get_distutils_extension(modname, pyxfilename):
extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest() extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
# modname = modname + extra # modname = modname + extra
extension_mod = handle_special_build(modname, pyxfilename) extension_mod = handle_special_build(modname, pyxfilename)
if not extension_mod: if not extension_mod:
extension_mod = Extension(name = modname, sources=[pyxfilename]) extension_mod = Extension(name = modname, sources=[pyxfilename])
return extension_mod return extension_mod
def handle_special_build(modname, pyxfilename): def handle_special_build(modname, pyxfilename):
...@@ -147,53 +136,64 @@ def load_module(name, pyxfilename): ...@@ -147,53 +136,64 @@ def load_module(name, pyxfilename):
assert mod.__file__ == so_path, (mod.__file__, so_path) assert mod.__file__ == so_path, (mod.__file__, so_path)
return mod return mod
class PyxLoader (ihooks.ModuleLoader):
"""Load a module. It checks whether a file is a .pyx and returns it.
Otherwise it lets the ihooks base class handle it. Let install()
install it."""
def load_module (self, name, stuff):
# If it's a Pyrex file, load it specially.
if stuff[2][2] == PYX_FILE_TYPE:
file, pyxfilename, info = stuff
(suff, mode, type) = info
if file:
file.close()
return load_module(name, pyxfilename)
else:
# Otherwise, use the default handler for loading
return ihooks.ModuleLoader.load_module( self, name, stuff)
try: # import hooks
import cimport
except ImportError: class PyxImporter(object):
cimport = None def __init__(self, so_cache_dir):
self.so_cache_dir = so_cache_dir
class cModuleImporter(ihooks.ModuleImporter): self.so_name_cache = {}
"""This was just left in from the Quixote implementation. I think
it allows a performance enhancement if you have the cimport module def find_module(self, fullname, package_path=None):
from Quixote. Let install() install it.""" print "SEARCHING", fullname, package_path
def __init__(self, loader=None): if '.' in fullname:
self.loader = loader or ihooks.ModuleLoader() mod_parts = fullname.split('.')
cimport.set_loader(self.find_import_module) package = '.'.join(mod_parts[:-1])
module_name = mod_parts[-1]
def find_import_module(self, fullname, subname, path): else:
stuff = self.loader.find_module(subname, path) package = None
if not stuff: module_name = fullname
pyx_module_name = module_name + PYX_EXT
# this may work, but it returns the file content, not its path
#import pkgutil
#pyx_source = pkgutil.get_data(package, pyx_module_name)
if package_path:
paths = package_path
else:
paths = sys.path
join_path = os.path.join
is_file = os.path.isfile
for path in filter(os.path.isdir, paths):
for filename in os.listdir(path):
if filename == pyx_module_name:
return PyxLoader(fullname, join_path(path, filename))
elif filename == module_name:
package_path = join_path(path, filename)
init_path = join_path(package_path, '__init__' + PYX_EXT)
if is_file(init_path):
return PyxLoader(fullname, package_path, init_path)
# not found, normal package, not a .pyx file, none of our business
return None return None
return self.loader.load_module(fullname, stuff)
def install(self): class PyxLoader(object):
self.save_import_module = __builtin__.__import__ def __init__(self, fullname, path, init_path=None):
self.save_reload = __builtin__.reload self.fullname, self.path, self.init_path = fullname, path, init_path
if not hasattr(__builtin__, 'unload'):
__builtin__.unload = None def load_module(self, fullname):
self.save_unload = __builtin__.unload assert self.fullname == fullname, (
__builtin__.__import__ = cimport.import_module "invalid module, expected %s, got %s" % (
__builtin__.reload = cimport.reload_module self.fullname, fullname))
__builtin__.unload = self.unload if self.init_path:
# package
print "PACKAGE", fullname
module = load_module(fullname, self.init_path)
module.__path__ = [self.path]
else:
print "MODULE", fullname
module = load_module(fullname, self.path)
return module
_installed = 0
def install(): def install():
"""Main entry point. call this to install the import hook in your """Main entry point. call this to install the import hook in your
...@@ -201,22 +201,14 @@ def install(): ...@@ -201,22 +201,14 @@ def install():
you use Python, add it to your sitecustomize (as described above). you use Python, add it to your sitecustomize (as described above).
""" """
global _installed for importer in sys.meta_path:
if not _installed: if isinstance(importer, PyxImporter):
hooks = PyxHooks() return
loader = PyxLoader(hooks) importer = PyxImporter('~/.pyxbuild')
if cimport is not None: sys.meta_path.append(importer)
importer = cModuleImporter(loader)
else:
importer = ihooks.ModuleImporter(loader) # MAIN
ihooks.install(importer)
_installed = 1
def on_remove_file_error(func, path, excinfo):
print "Sorry! Could not remove a temp file:", path
print "Extra information."
print func, excinfo
print "You may want to delete this yourself when you get a chance."
def show_docs(): def show_docs():
import __main__ import __main__
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment