pyximport for compiling .pyx files on import

pyximport/PKG-INFO

+Metadata-Version: 1.0
+Name: pyximport
+Version: 1.0
+Summary: Hooks to build and run Pyrex files as if they were simple Python files
+Home-page: http://www.prescod.net/pyximport
+Author: Paul Prescod
+Author-email: paul@prescod.net
+License: Python
+Description: UNKNOWN
+Keywords: pyrex import hook
+Platform: UNKNOWN

pyximport/README

+
+ == Pyximport == 
+
+Download: pyx-import-1.0.tar.gz
+<http://www.prescod.net/pyximport/pyximport-1.0.tar.gz>
+
+Pyrex is a compiler. Therefore it is natural that people tend to go
+through an edit/compile/test cycle with Pyrex modules. But my personal
+opinion is that one of the deep insights in Python's implementation is
+that a language can be compiled (Python modules are compiled to .pyc)
+files and hide that compilation process from the end-user so that they
+do not have to worry about it. Pyximport does this for Pyrex modules.
+For instance if you write a Pyrex module called "foo.pyx", with
+Pyximport you can import it in a regular Python module like this:
+
+
+import pyximport; pyximport.install()
+import foo
+
+Doing so will result in the compilation of foo.pyx (with appropriate
+exceptions if it has an error in it).
+
+If you would always like to import pyrex files without building them
+specially, you can also the first line above to your sitecustomize.py.
+That will install the hook every time you run Python. Then you can use
+Pyrex modules just with simple import statements. I like to test my
+Pyrex modules like this:
+
+
+python -c "import foo"
+
+ == Dependency Handling == 
+
+In Pyximport 1.1 it is possible to declare that your module depends on
+multiple files, (likely ".h" and ".pxd" files). If your Pyrex module is
+named "foo" and thus has the filename "foo.pyx" then you should make
+another file in the same directory called "foo.pyxdep". The
+"modname.pyxdep" file can be a list of filenames or "globs" (like
+"*.pxd" or "include/*.h"). Each filename or glob must be on a separate
+line. Pyximport will check the file date for each of those files before
+deciding whether to rebuild the module. In order to keep track of the
+fact that the dependency has been handled, Pyximport updates the
+modification time of your ".pyx" source file. Future versions may do
+something more sophisticated like informing distutils of the
+dependencies directly.
+
+ == Limitations == 
+
+Pyximport does not give you any control over how your Pyrex file is
+compiled. Usually the defaults are fine. You might run into problems if
+you wanted to write your program in half-C, half-Pyrex and build them
+into a single library. Pyximport 1.2 will probably do this.
+
+Pyximport does not hide the Distutils/GCC warnings and errors generated
+by the import process. Arguably this will give you better feedback if
+something went wrong and why. And if nothing went wrong it will give you
+the warm fuzzy that pyximport really did rebuild your module as it was
+supposed to.
+
+ == For further thought and discussion == 
+
+I don't think that Python's "reload" will do anything for changed .SOs
+on some (all?) platforms. It would require some (easy) experimentation
+that I haven't gotten around to. But reload is rarely used in
+applications outside of the Python interactive interpreter and certainly
+not used much for C extension modules. Info about Windows
+<http://mail.python.org/pipermail/python-list/2001-July/053798.html>
+
+"setup.py install" does not modify sitecustomize.py for you. Should it?
+Modifying Python's "standard interpreter" behaviour may be more than
+most people expect of a package they install..
+
+Pyximport puts your ".c" file beside your ".pyx" file (analogous to
+".pyc" beside ".py"). But it puts the platform-specific binary in a
+build directory as per normal for Distutils. If I could wave a magic
+wand and get Pyrex or distutils or whoever to put the build directory I
+might do it but not necessarily: having it at the top level is VERY
+HELPFUL for debugging Pyrex problems.
+

pyximport/Setup.py

+from distutils.core import setup
+import sys, os
+from StringIO import StringIO
+
+if "sdist" in sys.argv:
+    try:
+        os.remove("MANIFEST")
+    except (IOError, OSError):
+        pass
+
+    import html2text
+    out = StringIO()
+    html2text.convert_files(open("index.html"), out)
+    out.write("\n\n")
+    open("README", "w").write(out.getvalue())
+
+setup(
+    name = "pyximport",
+    fullname = "Pyrex Import Hooks",
+    version = "1.0",
+    description = "Hooks to build and run Pyrex files as if they were simple Python files", 
+    author = "Paul Prescod",
+    author_email = "paul@prescod.net",
+    url = "http://www.prescod.net/pyximport",
+    license = "Python",
+    keywords = "pyrex import hook",
+    scripts = ["pyxrun"],
+    data_files = [("examples/multi_file_extension", 
+		  ["README", "ccode.c", "test.pyx", "test.pyxbld"]),
+		 ("examples/dependencies",
+		  ["README", "test.pyx", "test.pyxdep", "header.h",
+			"header2.h", "header3.h", "header4.h"])
+		],
+    py_modules = ["pyximport", "pyxbuild"])
+

pyximport/pyxbuild.py

+"""Build a Pyrex file from .pyx source to .so loadable module using
+the installed distutils infrastructure. Call:
+
+out_fname = pyx_to_dll("foo.pyx")
+"""
+import os, md5
+
+import distutils
+from distutils.dist import Distribution
+from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError
+from distutils.extension import Extension
+from distutils.util import grok_environment_error
+from Pyrex.Distutils import build_ext
+import shutil
+
+DEBUG = 0
+def pyx_to_dll(filename, ext = None, force_rebuild = 0):
+    """Compile a PYX file to a DLL and return the name of the generated .so 
+       or .dll ."""
+    assert os.path.exists(filename)
+
+    path, name = os.path.split(filename)
+
+    if not ext:
+        modname, extension = os.path.splitext(name)
+	assert extension == ".pyx", extension
+        ext = Extension(name=modname, sources=[filename])
+
+    if DEBUG:
+        quiet = "--verbose"
+    else:
+	quiet = "--quiet"
+    args = [quiet, "build_ext"]
+    if force_rebuild:
+        args.append("--force")
+    dist = Distribution({"script_name": None, "script_args": args})
+    if not dist.ext_modules:
+        dist.ext_modules = []
+    dist.ext_modules.append(ext)
+    dist.cmdclass = {'build_ext': build_ext}
+    build = dist.get_command_obj('build')
+    build.build_base = os.path.join(path, "_pyxbld")
+
+    try:
+        ok = dist.parse_command_line()
+    except DistutilsArgError, msg:
+        raise
+
+    if DEBUG:
+        print "options (after parsing command line):"
+        dist.dump_option_dicts()
+    assert ok
+
+
+    try:
+        dist.run_commands()
+        return dist.get_command_obj("build_ext").get_outputs()[0]
+    except KeyboardInterrupt:
+        raise SystemExit, "interrupted"
+    except (IOError, os.error), exc:
+        error = grok_environment_error(exc)
+
+        if DEBUG:
+            sys.stderr.write(error + "\n")
+            raise
+        else:
+            raise SystemExit, error
+
+    except (DistutilsError,
+        CCompilerError), msg:
+        if DEBUG:
+            raise
+        else:
+            raise SystemExit, "error: " + str(msg)
+
+if __name__=="__main__":
+    pyx_to_dll("dummy.pyx")
+    import test
+

pyximport/pyximport.py

+"""
+Import hooks; when installed (with the install()) function, these hooks 
+allow importing .pyx files as if they were Python modules.
+
+If you want the hook installed every time you run Python
+you can add it to your Python version by adding these lines to
+sitecustomize.py (which you can create from scratch in site-packages 
+if it doesn't exist there are somewhere else on your python path)
+
+import pyximport
+pyximport.install()
+
+For instance on  the Mac with Python 2.3 built from CVS, you could 
+create sitecustomize.py with only those two lines at
+/usr/local/lib/python2.3/site-packages/sitecustomize.py .
+
+Running this module as a top-level script will run a test and then print
+the documentation.
+
+This code was modeled on Quixote's ptl_import.
+"""
+import sys, os, shutil
+import imp, ihooks, glob, md5
+import __builtin__
+import pyxbuild
+from distutils.dep_util import newer
+from distutils.extension import Extension
+
+mod_name = "pyximport"
+
+assert sys.hexversion >= 0x20000b1, "need Python 2.0b1 or later"
+
+PYX_FILE_TYPE = 1011
+PYX_EXT = ".pyx"
+PYXDEP_EXT = ".pyxdep"
+PYXBLD_EXT = ".pyxbld"
+_test_files = []
+
+class PyxHooks (ihooks.Hooks):
+    """Import hook that declares our suffixes. Let install() install it."""
+    def get_suffixes (self):
+        # add our suffixes
+        return imp.get_suffixes() + [(PYX_EXT, "r", PYX_FILE_TYPE)]
+
+# Performance problem: for every PYX file that is imported, we will 
+# invoke the whole distutils infrastructure even if the module is 
+# already built. It might be more efficient to only do it when the 
+# mod time of the .pyx is newer than the mod time of the .so but
+# the question is how to get distutils to tell me the name of the .so
+# before it builds it. Maybe it is easy...but maybe the peformance
+# issue isn't real.
+def _load_pyrex(name, filename):
+    "Load a pyrex file given a name and filename."
+
+def get_distutils_extension(modname, pyxfilename):
+
+    extra = "_" + md5.md5(open(pyxfilename).read()).hexdigest()  
+    modname = modname + extra
+
+    extension_mod = handle_special_build(modname, pyxfilename)
+
+    if not extension_mod:
+	extension_mod = Extension(name = modname, sources=[pyxfilename])
+
+    return extension_mod
+
+def handle_special_build(modname, pyxfilename):
+    special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
+    
+    if not os.path.exists(special_build): 
+        ext = None
+    else:
+	globls = {}
+	locs = {}
+	# execfile(special_build, globls, locs)
+	# ext = locs["make_ext"](modname, pyxfilename)
+        mod = imp.load_source("XXXX", special_build, open(special_build))
+        ext = mod.make_ext(modname, pyxfilename)
+	assert ext and ext.sources, ("make_ext in %s did not return Extension" 
+			% special_build)
+        ext.sources = [os.path.join(os.path.dirname(special_build), source) 
+				for source in ext.sources]
+    return ext
+
+def handle_dependencies(pyxfilename):
+    dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
+
+    # by default let distutils decide whether to rebuild on its own
+    # (it has a better idea of what the output file will be)
+
+    # but we know more about dependencies so force a rebuild if 
+    # some of the dependencies are newer than the pyxfile.
+    if os.path.exists(dependfile):
+	depends = open(dependfile).readlines()
+	depends = [depend.strip() for depend in depends]
+
+	# gather dependencies in the "files" variable
+	# the dependency file is itself a dependency
+	files = [dependfile]
+	for depend in depends:
+	    fullpath = os.path.join(os.path.dirname(dependfile),
+				      depend) 
+	    files.extend(glob.glob(fullpath))
+
+	# only for unit testing to see we did the right thing
+	_test_files[:] = []
+
+	# if any file that the pyxfile depends upon is newer than
+	# the pyx file, 'touch' the pyx file so that distutils will
+	# be tricked into rebuilding it.
+	for file in files:
+	    if newer(file, pyxfilename):
+		print "Rebuilding because of ", file
+		filetime = os.path.getmtime(file)
+		os.utime(pyxfilename, (filetime, filetime))
+		_test_files.append(file)
+
+def build_module(name, pyxfilename):
+    assert os.path.exists(pyxfilename), (
+		"Path does not exist: %s" % pyxfilename)
+    handle_dependencies(pyxfilename)
+
+    extension_mod = get_distutils_extension(name, pyxfilename)
+
+    so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod)
+    assert os.path.exists(so_path), "Cannot find: %s" % so_path
+
+    junkpath = os.path.join(os.path.dirname(so_path), name+"_*")
+    junkstuff = glob.glob(junkpath)
+    for path in junkstuff:
+	if path!=so_path:
+	    try:
+		os.remove(path)
+	    except IOError:
+		print "Couldn't remove ", path
+
+    return so_path
+
+def load_module(name, pyxfilename):
+    so_path = build_module(name, pyxfilename)
+    mod = imp.load_dynamic(name, so_path)
+    assert mod.__file__ == so_path, (mod.__file__, so_path)
+    return mod
+
+class PyxLoader (ihooks.ModuleLoader):
+    """Load a module. It checks whether a file is a .pyx and returns it.
+    Otherwise it lets the ihooks base class handle it. Let install() 
+    install it."""
+
+    def load_module (self, name, stuff):
+        # If it's a Pyrex file, load it specially.
+        if stuff[2][2] == PYX_FILE_TYPE:
+            file, pyxfilename, info = stuff
+            (suff, mode, type) = info
+            if file:
+                file.close()
+	    return load_module(name, pyxfilename)
+        else:
+            # Otherwise, use the default handler for loading
+            return ihooks.ModuleLoader.load_module( self, name, stuff)
+
+try:
+    import cimport
+except ImportError:
+    cimport = None
+
+class cModuleImporter(ihooks.ModuleImporter):
+    """This was just left in from the Quixote implementation. I think
+    it allows a performance enhancement if you have the cimport module
+    from Quixote. Let install() install it."""
+    def __init__(self, loader=None):
+        self.loader = loader or ihooks.ModuleLoader()
+        cimport.set_loader(self.find_import_module)
+
+    def find_import_module(self, fullname, subname, path):
+        stuff = self.loader.find_module(subname, path)
+        if not stuff:
+            return None
+        return self.loader.load_module(fullname, stuff)
+
+    def install(self):
+        self.save_import_module = __builtin__.__import__
+        self.save_reload = __builtin__.reload
+        if not hasattr(__builtin__, 'unload'):
+            __builtin__.unload = None
+        self.save_unload = __builtin__.unload
+        __builtin__.__import__ = cimport.import_module
+        __builtin__.reload = cimport.reload_module
+        __builtin__.unload = self.unload
+
+_installed = 0
+
+def install():
+    """Main entry point. call this to install the import hook in your
+    for a single Python process. If you want it to be installed whenever
+    you use Python, add it to your sitecustomize (as described above).
+
+    """
+    global _installed
+    if not _installed:
+        hooks = PyxHooks()
+        loader = PyxLoader(hooks)
+        if cimport is not None:
+            importer = cModuleImporter(loader)
+        else:
+            importer = ihooks.ModuleImporter(loader)
+        ihooks.install(importer)
+        _installed = 1
+
+def on_remove_file_error(func, path, excinfo):
+    print "Sorry! Could not remove a temp file:", path
+    print "Extra information."
+    print func, excinfo
+    print "You may want to delete this yourself when you get a chance."
+
+def show_docs():
+    import __main__
+    __main__.__name__ = mod_name
+    for name in dir(__main__):
+        item = getattr(__main__, name)
+        try:
+            setattr(item, "__module__", mod_name)
+        except (AttributeError, TypeError):
+            pass
+    help(__main__)
+
+if __name__ == '__main__':
+    show_docs()

pyximport/test/test_pyximport.py

+import pyximport; pyximport.install()
+import os, sys
+import time, shutil
+import tempfile
+
+def make_tempdir():
+    tempdir = os.path.join(tempfile.gettempdir(), "pyrex_temp")
+    if os.path.exists(tempdir):
+        remove_tempdir(tempdir)
+
+    os.mkdir(tempdir)
+    return tempdir
+
+def remove_tempdir(tempdir):
+    shutil.rmtree(tempdir, 0, on_remove_file_error)
+
+def on_remove_file_error(func, path, excinfo):
+    print "Sorry! Could not remove a temp file:", path
+    print "Extra information."
+    print func, excinfo
+    print "You may want to delete this yourself when you get a chance."
+
+def test():
+    tempdir = make_tempdir()
+    sys.path.append(tempdir)
+    filename = os.path.join(tempdir, "dummy.pyx")
+    open(filename, "w").write("print 'Hello world from the Pyrex install hook'")
+    import dummy
+    reload(dummy)
+
+    depend_filename = os.path.join(tempdir, "dummy.pyxdep")
+    depend_file = open(depend_filename, "w")
+    depend_file.write("*.txt\nfoo.bar")
+    depend_file.close()
+
+    build_filename = os.path.join(tempdir, "dummy.pyxbld")
+    build_file = open(build_filename, "w")
+    build_file.write("""
+from distutils.extension import Extension
+def make_ext(name, filename):
+	return Extension(name=name, sources=[filename]) 
+""")
+    build_file.close()
+
+    open(os.path.join(tempdir, "foo.bar"), "w").write(" ")
+    open(os.path.join(tempdir, "1.txt"), "w").write(" ")
+    open(os.path.join(tempdir, "abc.txt"), "w").write(" ")
+    reload(dummy)
+    assert len(pyximport._test_files)==1, pyximport._test_files
+    reload(dummy)
+
+    time.sleep(1) # sleep a second to get safer mtimes
+    open(os.path.join(tempdir, "abc.txt"), "w").write(" ")
+    print "Here goes the reolad"
+    reload(dummy)
+    assert len(pyximport._test_files) == 1, pyximport._test_files
+
+    reload(dummy)
+    assert len(pyximport._test_files) ==0, pyximport._test_files
+    remove_tempdir(tempdir)
+
+if __name__=="__main__":
+	test()
+

pyximport/test/test_reload.py

+# reload seems to work for Python 2.3 but not 2.2. 
+import time, os, sys
+import test_pyximport
+
+# debugging the 2.2 problem
+if 1:
+	from distutils import sysconfig
+	try:
+		sysconfig.set_python_build()
+	except AttributeError:
+		pass
+	import pyxbuild
+	print pyxbuild.distutils.sysconfig == sysconfig
+
+def test():
+	tempdir = test_pyximport.make_tempdir()
+	sys.path.append(tempdir)
+	hello_file = os.path.join(tempdir, "hello.pyx")
+	open(hello_file, "w").write("x = 1; print x; before = 'before'\n")
+	import hello
+        assert hello.x == 1
+
+	time.sleep(1) # sleep to make sure that new "hello.pyx" has later
+		      # timestamp than object file.
+
+	open(hello_file, "w").write("x = 2; print x; after = 'after'\n")
+	reload(hello)
+	assert hello.x == 2, "Reload should work on Python 2.3 but not 2.2"
+	test_pyximport.remove_tempdir(tempdir)
+
+if __name__=="__main__":
+	test()
+