Commit 4c0bd32b authored by jim's avatar jim

Added support controlling how eggs with extensions are built.


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@69383 62d5b8a3-27da-0310-9561-8e5933582275
parent 7abf0a57
...@@ -20,10 +20,15 @@ installed. ...@@ -20,10 +20,15 @@ installed.
$Id$ $Id$
""" """
import logging, os, re, sys import logging, os, re, tempfile, sys
import pkg_resources import pkg_resources, setuptools.command.setopt
import zc.buildout import zc.buildout
# XXX we could potentially speed this up quite a bit by keeping our
# own PackageIndex to analyse whether there are newer dists. A hitch
# is that the package index seems to go out of its way to only handle
# one Python version at a time. :(
logger = logging.getLogger('zc.buildout.easy_install') logger = logging.getLogger('zc.buildout.easy_install')
# Include buildout and setuptools eggs in paths # Include buildout and setuptools eggs in paths
...@@ -131,15 +136,15 @@ def _call_easy_install(spec, dest, links=(), ...@@ -131,15 +136,15 @@ def _call_easy_install(spec, dest, links=(),
logger.debug('Running easy_install:\n%s "%s"\npath=%s\n', logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
executable, '" "'.join(args), path) executable, '" "'.join(args), path)
args += (dict(PYTHONPATH=path), ) args += (dict(os.environ, PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first sys.stdout.flush() # We want any pending output first
exit_code = os.spawnle(os.P_WAIT, executable, executable, *args) exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
assert exit_code == 0
# We may overwrite distributions, so clear importer # We may overwrite distributions, so clear importer
# cache. # cache.
sys.path_importer_cache.clear() sys.path_importer_cache.clear()
assert exit_code == 0
def _get_dist(requirement, env, ws, def _get_dist(requirement, env, ws,
...@@ -235,6 +240,77 @@ def install(specs, dest, ...@@ -235,6 +240,77 @@ def install(specs, dest,
return ws return ws
def _editable(spec, dest, links=(), index = None, executable=sys.executable):
prefix = sys.exec_prefix + os.path.sep
path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)])
args = (
'-c', 'from setuptools.command.easy_install import main; main()',
'-eb', dest)
if links:
args += ('-f', ' '.join(links))
if index:
args += ('-i', index)
level = logger.getEffectiveLevel()
if level > logging.DEBUG:
args += ('-q', )
elif level < logging.DEBUG:
args += ('-v', )
args += (spec, )
if level <= logging.DEBUG:
logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
executable, '" "'.join(args), path)
args += (dict(os.environ, PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first
exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
assert exit_code == 0
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
path=None):
# XXX we're going to download and build the egg every stinking time.
# We need to not do that.
logger.debug('Building %r', spec)
path = path and path[:] or []
if dest is not None:
path.insert(0, dest)
path += buildout_and_setuptools_path
links = list(links) # make copy, because we may need to mutate
# For each spec, see if it is already installed. We create a working
# set to keep track of what we've collected and to make sue than the
# distributions assembled are consistent.
env = pkg_resources.Environment(path, python=_get_version(executable))
requirement = pkg_resources.Requirement.parse(spec)
dist = _satisfied(requirement, env)
if dist is not None:
return dist
# Get an editable version of the package to a temporary directory:
tmp = tempfile.mkdtemp('editable')
_editable(spec, tmp, links, index, executable)
setup_cfg = os.path.join(tmp, requirement.key, 'setup.cfg')
if not os.path.exists(setup_cfg):
f = open(setup_cfg, 'w')
f.close()
setuptools.command.setopt.edit_config(setup_cfg, dict(build_ext=build_ext))
# Now run easy_install for real:
_call_easy_install(
os.path.join(tmp, requirement.key),
dest, links, index, executable, True)
def working_set(specs, executable, path): def working_set(specs, executable, path):
return install(specs, None, executable=executable, path=path) return install(specs, None, executable=executable, path=path)
......
...@@ -18,7 +18,10 @@ easy_install command that provides some additional semantics: ...@@ -18,7 +18,10 @@ easy_install command that provides some additional semantics:
look for additional distributions. We always give preference to look for additional distributions. We always give preference to
develop eggs. develop eggs.
The easy_install module provides a single method, install. The - Distutils options for building extensions can be passed.
The easy_install module provides a method, install, for installing one
or more packages and their dependencies. The
install function takes 2 positional arguments: install function takes 2 positional arguments:
- An iterable of setuptools requirement strings for the distributions - An iterable of setuptools requirement strings for the distributions
...@@ -73,6 +76,7 @@ We have a link server that has a number of eggs: ...@@ -73,6 +76,7 @@ We have a link server that has a number of eggs:
<a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br> <a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br>
<a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br> <a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br>
<a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br> <a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br>
<a href="extdemo-1.4.tar.gz">extdemo-1.4.tar.gz</a><br>
<a href="index/">index/</a><br> <a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br> <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br> <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
...@@ -267,3 +271,94 @@ original script names to new script names. ...@@ -267,3 +271,94 @@ original script names to new script names.
>>> print system(os.path.join(bin, 'run')), >>> print system(os.path.join(bin, 'run')),
3 1 3 1
Handling custom build options for extensions
--------------------------------------------
Sometimes, we need to control how extension modules are built. The
build method provides this level of control. It takes a single
package specification, downloads a source distribution, and builds it
with specified custom build options.
The build method takes 3 positional arguments:
spec
A package specification
dest
A destination directory
build_ext
A dictionary of options to be passed to the distutils build_ext
command when building extensions.
It supports a number of optional keyword arguments:
links
a sequence of URLs, file names, or directories to look for
links to distributions,
index
The URL of an index server, or almost any other valid URL. :)
If not specified, the Python Package Index,
http://cheeseshop.python.org/pypi, is used. You can specify an
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
executable
A path to a Python executable. Distributions will ne installed
using this executable and will be for the matching Python version.
path
A list of additional directories to search for locally-installed
distributions.
always_unzip
A flag indicating that newly-downloaded distributions should be
directories even if they could be installed as zip files.
Our link server included a source distribution that includes a simple
extension, extdemo.c::
#include <Python.h>
#include <extdemo.h>
static PyMethodDef methods[] = {};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
}
The extension depends on a system-dependnt include file, extdemo.h,
that defines a constant, EXTDEMO, that is exposed by the extension.
We'll add an include directory to our sample buildout and add the
needed include file to it:
>>> mkdir(sample_buildout, 'include')
>>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
... "#define EXTDEMO 42\n")
Now, we can use the build function to create an egg from the source
distribution:
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/')
Now if we look in our destination directory, we see we have an extdemo egg:
>>> ls(dest)
d demo-0.3-py2.4.egg
d demoneeded-1.1-py2.4.egg
d extdemo-1.4-py2.3-unix-i686.egg
...@@ -200,6 +200,49 @@ def multi_python(test): ...@@ -200,6 +200,49 @@ def multi_python(test):
test.globs['python2_4_executable'] = p24 test.globs['python2_4_executable'] = p24
extdemo_c = """
#include <Python.h>
#include <extdemo.h>
static PyMethodDef methods[] = {};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
}
"""
extdemo_setup_py = """
from distutils.core import setup, Extension
setup(name = "extdemo", version = "1.4", url="http://www.zope.org",
author="Demo", author_email="demo@demo.com",
ext_modules = [Extension('extdemo', ['extdemo.c'])],
)
"""
def add_source_dist(test):
import tarfile
tmp = tempfile.mkdtemp('test-sdist')
open(os.path.join(tmp, 'extdemo.c'), 'w').write(extdemo_c);
open(os.path.join(tmp, 'setup.py'), 'w').write(extdemo_setup_py);
open(os.path.join(tmp, 'README'), 'w').write("");
open(os.path.join(tmp, 'MANIFEST.in'), 'w').write("include *.c\n");
here = os.getcwd()
os.chdir(tmp)
status = os.spawnl(os.P_WAIT, sys.executable, sys.executable,
os.path.join(tmp, 'setup.py'), '-q', 'sdist')
os.chdir(here)
assert status == 0
shutil.move(
os.path.join(tmp, 'dist', 'extdemo-1.4.tar.gz'),
os.path.join(test.globs['sample_eggs'], 'extdemo-1.4.tar.gz'),
)
def make_tree(test): def make_tree(test):
sample_eggs = test.globs['sample_eggs'] sample_eggs = test.globs['sample_eggs']
tree = dict( tree = dict(
...@@ -266,6 +309,8 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler): ...@@ -266,6 +309,8 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
self.send_header('Content-Length', len(out)) self.send_header('Content-Length', len(out))
if name.endswith('.egg'): if name.endswith('.egg'):
self.send_header('Content-Type', 'application/zip') self.send_header('Content-Type', 'application/zip')
elif name.endswith('.gz'):
self.send_header('Content-Type', 'application/x-gzip')
else: else:
self.send_header('Content-Type', 'text/html') self.send_header('Content-Type', 'text/html')
self.end_headers() self.end_headers()
......
...@@ -141,6 +141,12 @@ def linkerSetUp(test): ...@@ -141,6 +141,12 @@ def linkerSetUp(test):
zc.buildout.testing.multi_python(test) zc.buildout.testing.multi_python(test)
zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test)) zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
def easy_install_SetUp(test):
zc.buildout.testing.buildoutSetUp(test, clear_home=False)
zc.buildout.testing.multi_python(test)
zc.buildout.testing.add_source_dist(test)
zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
class PythonNormalizing(renormalizing.RENormalizing): class PythonNormalizing(renormalizing.RENormalizing):
def _transform(self, want, got): def _transform(self, want, got):
...@@ -216,14 +222,15 @@ def test_suite(): ...@@ -216,14 +222,15 @@ def test_suite():
doctest.DocFileSuite( doctest.DocFileSuite(
'easy_install.txt', 'easy_install.txt',
setUp=linkerSetUp, tearDown=zc.buildout.testing.buildoutTearDown, setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=PythonNormalizing([ checker=PythonNormalizing([
(re.compile("'%(sep)s\S+sample-install%(sep)s(dist%(sep)s)?" (re.compile("'%(sep)s\S+sample-install%(sep)s(dist%(sep)s)?"
% dict(sep=os.path.sep)), % dict(sep=os.path.sep)),
'/sample-eggs/'), '/sample-eggs/'),
(re.compile("(- (demo(needed)?|other)" (re.compile("([d-] ((ext)?demo(needed)?|other)"
"-\d[.]\d-py)\d[.]\d[.]egg"), "-\d[.]\d-py)\d[.]\d(-[^. \t\n]+)?[.]egg"),
'\\1V.V.egg'), '\\1V.V.egg'),
]), ]),
), ),
......
...@@ -35,6 +35,12 @@ scripts ...@@ -35,6 +35,12 @@ scripts
disabled. If the option isn't given at all, then all scripts disabled. If the option isn't given at all, then all scripts
defined by the named eggs will be generated. defined by the named eggs will be generated.
Custom eggs
-----------
The zc.recipe.egg:custom recipe supports building custom eggs,
currently with specialized options for building extensions.
To do To do
----- -----
......
...@@ -19,6 +19,9 @@ setup( ...@@ -19,6 +19,9 @@ setup(
install_requires = ['zc.buildout', 'setuptools'], install_requires = ['zc.buildout', 'setuptools'],
tests_require = ['zope.testing'], tests_require = ['zope.testing'],
test_suite = name+'.tests.test_suite', test_suite = name+'.tests.test_suite',
entry_points = {'zc.buildout': ['default = %s:Egg' % name]}, entry_points = {'zc.buildout': ['default = %s:Egg' % name,
'custom = %s:Custom' % name,
]
},
dependency_links = ['http://download.zope.org/distribution/'], dependency_links = ['http://download.zope.org/distribution/'],
) )
from zc.recipe.egg.egg import Egg from zc.recipe.egg.egg import Egg
from zc.recipe.egg.custom import Custom
##############################################################################
#
# Copyright (c) 2006 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Install packages as eggs
$Id$
"""
import os, re, zipfile
import zc.buildout.easy_install
class Custom:
def __init__(self, buildout, name, options):
self.buildout = buildout
self.name = name
self.options = options
links = options.get('find-links',
buildout['buildout'].get('find-links'))
if links:
links = links.split()
options['find-links'] = '\n'.join(links)
else:
links = ()
self.links = links
index = options.get('index', buildout['buildout'].get('index'))
if index is not None:
options['index'] = index
self.index = index
options['_b'] = buildout['buildout']['bin-directory']
options['_e'] = buildout['buildout']['eggs-directory']
options['_d'] = buildout['buildout']['develop-eggs-directory']
assert options.get('unzip') in ('true', 'false', None)
python = options.get('python', buildout['buildout']['python'])
options['executable'] = buildout[python]['executable']
build_ext = {}
for be_option in ('include-dirs', 'library-dirs', 'rpath'):
value = options.get(be_option)
if value is None:
continue
value = [
os.path.join(
buildout['buildout']['directory'],
v.strip()
)
for v in value.strip().split('\n')
if v.strip()
]
build_ext[be_option] = ':'.join(value)
options[be_option] = ':'.join(value)
self.build_ext = build_ext
def install(self):
if self.buildout['buildout'].get('offline') == 'true':
return
options = self.options
distribution = options.get('eggs', self.name).strip()
build_ext = dict([
(k, options[k])
for k in ('include-dirs', 'library-dirs', 'rpath')
if k in options
])
zc.buildout.easy_install.build(
distribution, options['_d'], self.build_ext,
self.links, self.index, options['executable'], [options['_e']],
)
Custon eggs
===========
Sometimes, It's necessary to provide extra control over how an egg is
created. This is commonly true for eggs with extension modules that
need to access libraries or include files.
The zc.recipe.egg:custom recipe can be used to define an egg with
custom build parameters. The currently defined parameters are:
include-dirs
A new-line separated list of directories to search for include
files.
library-dirs
A new-line separated list of directories to search for libraries
to link with.
rpath
A new-line separated list of directories to search for dynamic libraries
at run time.
In addition, the following options can be used to specify the egg:
egg
An eggs to install given as a setuptools requirement string.
This defaults to the part name.
find-links
A list of URLs, files, or directories to search for distributions.
index
The URL of an index server, or almost any other valid URL. :)
If not specified, the Python Package Index,
http://cheeseshop.python.org/pypi, is used. You can specify an
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
python
The name of a section to get the Python executable from.
If not specified, then the buildout python option is used. The
Python executable is found in the executable option of the named
section.
To illustrate this, we'll define a buildout that builds an egg for a
package that has a simple extension module::
#include <Python.h>
#include <extdemo.h>
static PyMethodDef methods[] = {};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
}
The extension depends on a system-dependnt include file, extdemo.h,
that defines a constant, EXTDEMO, that is exposed by the extension.
The extension module is available as a source distribution,
extdemo-1.4.tar.gz, on a distribution server.
We have a sample buildout that we'll add an include directory to with
the necessary include file:
>>> mkdir(sample_buildout, 'include')
>>> import os
>>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
... "#define EXTDEMO 42\n")
We'll also update the buildout configuration file to define a part for
the egg:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... parts = extdemo
...
... [extdemo]
... recipe = zc.recipe.egg:custom
... find-links = %(server)s
... index = %(server)s/index
... include-dirs = include
... """ % dict(server=link_server))
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print system(buildout),
zip_safe flag not set; analyzing archive contents...
We got the zip_safe warning because the source distribution we used
wasn't setuptools based and thus didn't set the option.
The egg is created in the develop-eggs directory *not* the eggs
directory because it depends on buildout-specific parameters and the
eggs directory can be shared across multiple buildouts.
>>> ls(sample_buildout, 'develop-eggs')
d extdemo-1.4-py2.4-unix-i686.egg
- zc.recipe.egg.egg-link
Note that no scripts or dependencies are installed. To install
dependencies or scripts for a custom egg, define another part and use
the zc.recipe.egg recipe, listing the custom egg as one of the eggs to
be installed. The zc.recipe.egg recipe will use the installed egg.
...@@ -41,6 +41,16 @@ def setUpPython(test): ...@@ -41,6 +41,16 @@ def setUpPython(test):
zc.buildout.testing.multi_python(test) zc.buildout.testing.multi_python(test)
zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test)) zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
def setUpCustom(test):
zc.buildout.testing.buildoutSetUp(test)
open(os.path.join(test.globs['sample_buildout'],
'develop-eggs', 'zc.recipe.egg.egg-link'),
'w').write(dirname(__file__, 4))
zc.buildout.testing.create_sample_eggs(test)
zc.buildout.testing.add_source_dist(test)
zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
def test_suite(): def test_suite():
return unittest.TestSuite(( return unittest.TestSuite((
...@@ -87,7 +97,17 @@ def test_suite(): ...@@ -87,7 +97,17 @@ def test_suite():
r'/sample-\1/\2'), r'/sample-\1/\2'),
(re.compile('\S+sample-(\w+)'), r'/sample-\1'), (re.compile('\S+sample-(\w+)'), r'/sample-\1'),
]), ]),
), ),
doctest.DocFileSuite(
'custom.txt',
setUp=setUpCustom, tearDown=zc.buildout.testing.buildoutTearDown,
checker=renormalizing.RENormalizing([
(re.compile("(d ((ext)?demo(needed)?|other)"
"-\d[.]\d-py)\d[.]\d(-[^. \t\n]+)?[.]egg"),
'\\1V.V.egg'),
]),
),
)) ))
if __name__ == '__main__': if __name__ == '__main__':
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment