Skip to content

S
slapos.buildout
Commit a4291845 authored by jim's avatar jim
Browse files
Options

Merged revisions 71277 to 71397 from dev branch: 

Feature Changes
---------------

- Variable substitutions now reflect option data written by recipes.

- A part referenced by a part in a parts list is now added to the parts
  list before the referencing part.  This means that you can omit
  parts from the parts list if they are referenced by other parts.

- Added a develop function to the easy_install module to aid in
  creating develop eggs with custom build_ext options.

- The build and develop functions in the easy_install module now
  return the path of the egg or egg link created.

- Removed the limitation that parts named in the install command can
  only name configured parts.

- Removed support ConfigParser-style variable substitutions
  (e.g. %(foo)s). Only the string-template style of variable
  (e.g. ${section:option}) substitutions will be supported.
  Supporting both violates "there's only one way to do it".

- Deprecated the buildout-section extendedBy option.


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@71398 62d5b8a3-27da-0310-9561-8e5933582275
parent b296bd0d
Hide whitespace changes
Inline Side-by-side
Showing with 1082 additions and 539 deletions
CHANGES.txt
View file @ a4291845
......@@ -20,9 +20,34 @@ priorities include:
Change History
**************
1.0.0b12 (2006-10-?)
1.0.0b13 (2006-12-04)
=====================
Feature Changes
---------------
- Variable substitutions now reflect option data written by recipes.
- A part referenced by a part in a parts list is now added to the parts
list before the referencing part. This means that you can omit
parts from the parts list if they are referenced by other parts.
- Added a develop function to the easy_install module to aid in
creating develop eggs with custom build_ext options.
- The build and develop functions in the easy_install module now
return the path of the egg or egg link created.
- Removed the limitation that parts named in the install command can
only name configured parts.
- Removed support ConfigParser-style variable substitutions
(e.g. %(foo)s). Only the string-template style of variable
(e.g. ${section:option}) substitutions will be supported.
Supporting both violates "there's only one way to do it".
- Deprecated the buildout-section extendedBy option.
Bugs Fixed
----------
......
setup.py
View file @ a4291845
......@@ -25,6 +25,8 @@ setup(
+ '\n' +
read('src', 'zc', 'buildout', 'testing.txt')
+ '\n' +
read('src', 'zc', 'buildout', 'easy_install.txt')
+ '\n' +
'Download\n'
'**********************\n'
),
......
src/zc/buildout/buildout.py
View file @ a4291845
......@@ -22,14 +22,22 @@ import os
import pprint
import re
import shutil
import cStringIO
import sys
import tempfile
import ConfigParser
import UserDict
import pkg_resources
import zc.buildout
import zc.buildout.easy_install
try:
realpath = os.path.realpath
except AttributeError:
def realpath(path):
return path
pkg_resources_loc = pkg_resources.working_set.find(
pkg_resources.Requirement.parse('setuptools')).location
......@@ -42,30 +50,11 @@ class MissingSection(zc.buildout.UserError, KeyError):
"""A required section is missinh
"""
class Options(dict):
def __init__(self, buildout, section, data):
self.buildout = buildout
self.section = section
super(Options, self).__init__(data)
def __getitem__(self, option):
try:
return super(Options, self).__getitem__(option)
except KeyError:
raise MissingOption("Missing option: %s:%s"
% (self.section, option))
def __str__(self):
return "The referenced section, %r, was not defined." % self[0]
# XXX need test
def __setitem__(self, option, value):
if not isinstance(value, str):
raise TypeError('Option values must be strings', value)
super(Options, self).__setitem__(option, value)
def copy(self):
return Options(self.buildout, self.section, self)
class Buildout(dict):
class Buildout(UserDict.DictMixin):
def __init__(self, config_file, cloptions, windows_restart=False):
config_file = os.path.abspath(config_file)
......@@ -75,8 +64,6 @@ class Buildout(dict):
print 'Warning: creating', config_file
open(config_file, 'w').write('[buildout]\nparts = \n')
super(Buildout, self).__init__()
# default options
data = dict(buildout={
'directory': os.path.dirname(config_file),
......@@ -110,19 +97,9 @@ class Buildout(dict):
options[option] = value
# The egg dire
# do substitutions
converted = {}
for section, options in data.iteritems():
for option, value in options.iteritems():
if '$' in value:
value = self._dosubs(section, option, value,
data, converted, [])
options[option] = value
converted[(section, option)] = value
# copy data into self:
for section, options in data.iteritems():
self[section] = Options(self, section, options)
self._raw = data
self._data = {}
self._parts = []
# initialize some attrs and buildout directories.
options = self['buildout']
......@@ -140,72 +117,11 @@ class Buildout(dict):
self._setup_logging()
def _dosubs(self, section, option, value, data, converted, seen):
key = section, option
r = converted.get(key)
if r is not None:
return r
if key in seen:
raise zc.buildout.UserError(
"Circular reference in substitutions.\n"
"We're evaluating %s\nand are referencing: %s.\n"
% (", ".join([":".join(k) for k in seen]),
":".join(key)
)
)
seen.append(key)
value = '$$'.join([self._dosubs_esc(s, data, converted, seen)
for s in value.split('$$')
])
seen.pop()
return value
_template_split = re.compile('([$]{[^}]*})').split
_simple = re.compile('[-a-zA-Z0-9 ._]+$').match
_valid = re.compile('[-a-zA-Z0-9 ._]+:[-a-zA-Z0-9 ._]+$').match
def _dosubs_esc(self, value, data, converted, seen):
value = self._template_split(value)
subs = []
for ref in value[1::2]:
s = tuple(ref[2:-1].split(':'))
if not self._valid(ref):
if len(s) < 2:
raise zc.buildout.UserError("The substitution, %s,\n"
"doesn't contain a colon."
% ref)
if len(s) > 2:
raise zc.buildout.UserError("The substitution, %s,\n"
"has too many colons."
% ref)
if not self._simple(s[0]):
raise zc.buildout.UserError(
"The section name in substitution, %s,\n"
"has invalid characters."
% ref)
if not self._simple(s[1]):
raise zc.buildout.UserError(
"The option name in substitution, %s,\n"
"has invalid characters."
% ref)
v = converted.get(s)
if v is None:
options = data.get(s[0])
if options is None:
raise MissingSection(
"Referenced section does not exist", s[0])
v = options.get(s[1])
if v is None:
raise MissingOption("Referenced option does not exist:",
*s)
if '$' in v:
v = self._dosubs(s[0], s[1], v, data, converted, seen)
options[s[1]] = v
converted[s] = v
subs.append(v)
subs.append('')
offline = options.get('offline', 'false')
if offline not in ('true', 'false'):
self._error('Invalid value for offline option: %s', offline)
options['offline'] = offline
return ''.join([''.join(v) for v in zip(value[::2], subs)])
def _buildout_path(self, *names):
return os.path.join(self._buildout_dir, *names)
......@@ -268,25 +184,26 @@ class Buildout(dict):
conf_parts = conf_parts and conf_parts.split() or []
installed_parts = installed_part_options['buildout']['parts']
installed_parts = installed_parts and installed_parts.split() or []
# If install_parts is given, then they must be listed in parts
# and we don't uninstall anything. Otherwise, we install
# the configured parts and uninstall anything else.
if install_parts:
extra = [p for p in install_parts if p not in conf_parts]
if extra:
self._error(
'Invalid install parts: %s.\n'
'Install parts must be listed in the configuration.',
' '.join(extra))
uninstall_missing = False
else:
install_parts = conf_parts
uninstall_missing = True
# load recipes
recipes = self._load_recipes(install_parts)
# load and initialize recipes
[self[part]['recipe'] for part in install_parts]
install_parts = self._parts
if self._log_level <= logging.DEBUG:
sections = list(self)
sections.sort()
print
print 'Configuration data:'
for section in self._data:
_save_options(section, self[section], sys.stdout)
print
# compute new part recipe signatures
self._compute_part_signatures(install_parts)
......@@ -339,19 +256,20 @@ class Buildout(dict):
for part in install_parts:
signature = self[part].pop('__buildout_signature__')
saved_options = self[part].copy()
recipe = self[part].recipe
if part in installed_parts:
self._logger.info('Updating %s', part)
old_options = installed_part_options[part]
old_installed_files = old_options['__buildout_installed__']
try:
update = recipes[part].update
update = recipe.update
except AttributeError:
update = recipes[part].install
update = recipe.install
self._logger.warning(
"The recipe for %s doesn't define an update "
"method. Using its install method",
part)
try:
installed_files = update()
except:
......@@ -364,7 +282,7 @@ class Buildout(dict):
else:
self._logger.info('Installing %s', part)
installed_files = recipes[part].install()
installed_files = recipe.install()
if installed_files is None:
self._logger.warning(
"The %s install returned None. A path or "
......@@ -380,15 +298,12 @@ class Buildout(dict):
] = '\n'.join(installed_files)
saved_options['__buildout_signature__'] = signature
if part not in installed_parts:
installed_parts.append(part)
installed_parts = [p for p in installed_parts if p != part]
installed_parts.append(part)
finally:
installed_part_options['buildout']['parts'] = ' '.join(
[p for p in conf_parts if p in installed_parts]
+
[p for p in installed_parts if p not in conf_parts]
)
installed_part_options['buildout']['parts'] = (
' '.join(installed_parts))
installed_part_options['buildout']['installed_develop_eggs'
] = installed_develop_eggs
......@@ -419,46 +334,8 @@ class Buildout(dict):
try:
for setup in develop.split():
setup = self._buildout_path(setup)
if os.path.isdir(setup):
setup = os.path.join(setup, 'setup.py')
self._logger.info("Develop: %s", setup)
fd, tsetup = tempfile.mkstemp()
try:
os.write(fd, runsetup_template % dict(
setuptools=pkg_resources_loc,
setupdir=os.path.dirname(setup),
setup=setup,
__file__ = setup,
))
args = [
zc.buildout.easy_install._safe_arg(tsetup),
'-q', 'develop', '-mxN',
'-f', zc.buildout.easy_install._safe_arg(
' '.join(self._links)
),
'-d', zc.buildout.easy_install._safe_arg(dest),
]
if self._log_level <= logging.DEBUG:
if self._log_level == logging.DEBUG:
del args[1]
else:
args[1] == '-v'
self._logger.debug("in: %s\n%r",
os.path.dirname(setup), args)
assert os.spawnl(
os.P_WAIT, sys.executable, sys.executable,
*args) == 0
finally:
os.close(fd)
os.remove(tsetup)
zc.buildout.easy_install.develop(setup, dest)
except:
# if we had an error, we need to roll back changes, by
# removing any files we created.
......@@ -490,90 +367,34 @@ class Buildout(dict):
self._logger.warning(
"Unexpected entry, %s, in develop-eggs directory", f)
def _load_recipes(self, parts):
recipes = {}
if not parts:
return recipes
recipes_requirements = []
pkg_resources.working_set.add_entry(
self['buildout']['develop-eggs-directory'])
pkg_resources.working_set.add_entry(self['buildout']['eggs-directory'])
# Gather requirements
for part in parts:
options = self.get(part)
if options is None:
raise MissingSection("No section was specified for part", part)
recipe, entry = self._recipe(part, options)
if recipe not in recipes_requirements:
recipes_requirements.append(recipe)
# Install the recipe distros
offline = self['buildout'].get('offline', 'false')
if offline not in ('true', 'false'):
self._error('Invalid value for offline option: %s', offline)
if offline == 'false':
dest = self['buildout']['eggs-directory']
else:
dest = None
ws = zc.buildout.easy_install.install(
recipes_requirements, dest,
links=self._links,
index=self['buildout'].get('index'),
path=[self['buildout']['develop-eggs-directory'],
self['buildout']['eggs-directory'],
],
working_set=pkg_resources.working_set,
)
# instantiate the recipes
for part in parts:
options = self[part]
recipe, entry = self._recipe(part, options)
recipe_class = pkg_resources.load_entry_point(
recipe, 'zc.buildout', entry)
recipes[part] = recipe_class(self, part, options)
return recipes
def _compute_part_signatures(self, parts):
# Compute recipe signature and add to options
for part in parts:
options = self.get(part)
if options is None:
options = self[part] = {}
recipe, entry = self._recipe(part, options)
recipe, entry = _recipe(options)
req = pkg_resources.Requirement.parse(recipe)
sig = _dists_sig(pkg_resources.working_set.resolve([req]))
options['__buildout_signature__'] = ' '.join(sig)
def _recipe(self, part, options):
recipe = options['recipe']
if ':' in recipe:
recipe, entry = recipe.split(':')
else:
entry = 'default'
return recipe, entry
def _read_installed_part_options(self):
old = self._installed_path()
if os.path.isfile(old):
parser = ConfigParser.SafeConfigParser(_spacey_defaults)
parser = ConfigParser.RawConfigParser()
parser.optionxform = lambda s: s
parser.read(old)
return dict([
(section,
Options(self, section,
[item for item in parser.items(section)
if item[0] not in _spacey_defaults]
)
)
for section in parser.sections()])
result = {}
for section in parser.sections():
options = {}
for option, value in parser.items(section):
if '%(' in value:
for k, v in _spacey_defaults:
value = value.replace(k, v)
options[option] = value
result[section] = Options(self, section, options)
return result
else:
return {'buildout': Options(self, 'buildout', {'parts': ''})}
......@@ -592,7 +413,7 @@ class Buildout(dict):
def _install(self, part):
options = self[part]
recipe, entry = self._recipe(part, options)
recipe, entry = _recipe(options)
recipe_class = pkg_resources.load_entry_point(
recipe, 'zc.buildout', entry)
installed = recipe_class(self, part, options).install()
......@@ -642,14 +463,6 @@ class Buildout(dict):
root_logger.setLevel(level)
self._log_level = level
if level <= logging.DEBUG:
sections = list(self)
sections.sort()
print 'Configuration data:'
for section in sections:
_save_options(section, self[section], sys.stdout)
print
def _maybe_upgrade(self):
# See if buildout or setuptools need to be upgraded.
# If they do, do the upgrade and restart the buildout process.
......@@ -677,10 +490,25 @@ class Buildout(dict):
if not upgraded:
return
if (os.path.abspath(sys.argv[0])
!= os.path.join(os.path.abspath(self['buildout']['bin-directory']),
'buildout')
if (realpath(os.path.abspath(sys.argv[0]))
!=
realpath(
os.path.join(os.path.abspath(
self['buildout']['bin-directory']
),
'buildout',
)
)
):
self._logger.debug("Running %r", realpath(sys.argv[0]))
self._logger.debug(
"Local buildout is %r",
realpath(
os.path.join(
os.path.abspath(self['buildout']['bin-directory']),
'buildout')
)
)
self._logger.warn("Not upgrading because not running a local "
"buildout command")
return
......@@ -745,7 +573,7 @@ class Buildout(dict):
fd, tsetup = tempfile.mkstemp()
try:
os.write(fd, runsetup_template % dict(
os.write(fd, zc.buildout.easy_install.runsetup_template % dict(
setuptools=pkg_resources_loc,
setupdir=os.path.dirname(setup),
setup=setup,
......@@ -758,20 +586,180 @@ class Buildout(dict):
os.close(fd)
os.remove(tsetup)
runsetup = setup # backward compat
runsetup = setup # backward compat.
def __getitem__(self, section):
try:
return self._data[section]
except KeyError:
pass
try:
data = self._raw[section]
except KeyError:
raise MissingSection(section)
options = Options(self, section, data)
self._data[section] = options
options._initialize()
return options
def __setitem__(self, key, value):
raise NotImplementedError('__setitem__')
def __delitem__(self, key):
raise NotImplementedError('__delitem__')
def keys(self):
return self._raw.keys()
def __iter__(self):
return iter(self._raw)
class Options(UserDict.DictMixin):
def __init__(self, buildout, section, data):
self.buildout = buildout
self.name = section
self._raw = data
self._data = {}
def _initialize(self):
# force substitutions
for k in self._raw:
self.get(k)
recipe = self.get('recipe')
if not recipe:
return
runsetup_template = """
import sys
sys.path.insert(0, %(setuptools)r)
import os, setuptools
reqs, entry = _recipe(self._data)
req = pkg_resources.Requirement.parse(reqs)
buildout = self.buildout
if pkg_resources.working_set.find(req) is None:
offline = buildout['buildout']['offline'] == 'true'
if offline:
dest = None
path = [buildout['buildout']['develop-eggs-directory'],
buildout['buildout']['eggs-directory'],
]
else:
dest = buildout['buildout']['eggs-directory']
path = [buildout['buildout']['develop-eggs-directory']]
zc.buildout.easy_install.install(
[reqs], dest,
links=buildout._links,
index=buildout['buildout'].get('index'),
path=path,
working_set=pkg_resources.working_set,
)
recipe_class = pkg_resources.load_entry_point(
req.project_name, 'zc.buildout', entry)
__file__ = %(__file__)r
self.recipe = recipe_class(buildout, self.name, self)
buildout._parts.append(self.name)
os.chdir(%(setupdir)r)
sys.argv[0] = %(setup)r
execfile(%(setup)r)
"""
def get(self, option, default=None, seen=None):
try:
return self._data[option]
except KeyError:
pass
v = self._raw.get(option)
if v is None:
return default
if '${' in v:
key = self.name, option
if seen is None:
seen = [key]
elif key in seen:
raise zc.buildout.UserError(
"Circular reference in substitutions.\n"
"We're evaluating %s\nand are referencing: %s.\n"
% (", ".join([":".join(k) for k in seen]),
":".join(key)
)
)
else:
seen.append(key)
v = '$$'.join([self._sub(s, seen) for s in v.split('$$')])
seen.pop()
self._data[option] = v
return v
_template_split = re.compile('([$]{[^}]*})').split
_simple = re.compile('[-a-zA-Z0-9 ._]+$').match
_valid = re.compile('[-a-zA-Z0-9 ._]+:[-a-zA-Z0-9 ._]+$').match
def _sub(self, template, seen):
value = self._template_split(template)
subs = []
for ref in value[1::2]:
s = tuple(ref[2:-1].split(':'))
if not self._valid(ref):
if len(s) < 2:
raise zc.buildout.UserError("The substitution, %s,\n"
"doesn't contain a colon."
% ref)
if len(s) > 2:
raise zc.buildout.UserError("The substitution, %s,\n"
"has too many colons."
% ref)
if not self._simple(s[0]):
raise zc.buildout.UserError(
"The section name in substitution, %s,\n"
"has invalid characters."
% ref)
if not self._simple(s[1]):
raise zc.buildout.UserError(
"The option name in substitution, %s,\n"
"has invalid characters."
% ref)
v = self.buildout[s[0]].get(s[1], None, seen)
if v is None:
raise MissingOption("Referenced option does not exist:", *s)
subs.append(v)
subs.append('')
return ''.join([''.join(v) for v in zip(value[::2], subs)])
def __getitem__(self, key):
try:
return self._data[key]
except KeyError:
pass
v = self.get(key)
if v is None:
raise MissingOption("Missing option: %s:%s"
% (self.name, key))
return v
def __setitem__(self, option, value):
if not isinstance(value, str):
raise TypeError('Option values must be strings', value)
self._data[option] = value
def __delitem__(self, key):
if key in self._raw:
del self._raw[key]
if key in self._data:
del self._data[key]
elif key in self._data:
del self._data[key]
else:
raise KeyError, key
def keys(self):
raw = self._raw
return list(self._raw) + [k for k in self._data if k not in raw]
def copy(self):
return dict([(k, self[k]) for k in self.keys()])
_spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*'
'|'
......@@ -780,6 +768,14 @@ _spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*'
'[ \t\r\f\v]+$'
)
_spacey_defaults = [
('%(__buildout_space__)s', ' '),
('%(__buildout_space_n__)s', '\n'),
('%(__buildout_space_r__)s', '\r'),
('%(__buildout_space_f__)s', '\f'),
('%(__buildout_space_v__)s', '\v'),
]
def _quote_spacey_nl(match):
match = match.group(0).split('\n', 1)
result = '\n\t'.join(
......@@ -794,20 +790,11 @@ def _quote_spacey_nl(match):
)
return result
_spacey_defaults = dict(
__buildout_space__ = ' ',
__buildout_space_r__ = '\r',
__buildout_space_f__ = '\f',
__buildout_space_v__ = '\v',
__buildout_space_n__ = '\n',
)
def _save_options(section, options, f):
print >>f, '[%s]' % section
items = options.items()
items.sort()
for option, value in items:
value = value.replace('%', '%%')
value = _spacey_nl.sub(_quote_spacey_nl, value)
if value.startswith('\n\t'):
value = '%(__buildout_space_n__)s' + value[2:]
......@@ -832,7 +819,7 @@ def _open(base, filename, seen):
result = {}
parser = ConfigParser.SafeConfigParser()
parser = ConfigParser.RawConfigParser()
parser.optionxform = lambda s: s
parser.readfp(open(filename))
extends = extended_by = None
......@@ -850,6 +837,9 @@ def _open(base, filename, seen):
result = _update(_open(base, fname, seen), result)
if extended_by:
self._logger.warn(
"The extendedBy option is deprecated. Stop using it."
)
for fname in extended_by.split():
result = _update(result, _open(base, fname, seen))
......@@ -887,6 +877,15 @@ def _update(d1, d2):
d1[section] = d2[section]
return d1
def _recipe(options):
recipe = options['recipe']
if ':' in recipe:
recipe, entry = recipe.split(':')
else:
entry = 'default'
return recipe, entry
def _error(*message):
sys.stderr.write('Error: ' + ' '.join(message) +'\n')
sys.exit(1)
......
src/zc/buildout/buildout.txt
View file @ a4291845
......@@ -286,7 +286,7 @@ buildout:
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Installing data-dir
data-dir: Creating directory mystuff
......@@ -335,7 +335,7 @@ we'll see that the directory gets removed and recreated:
... """)
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling data-dir
buildout: Installing data-dir
data-dir: Creating directory mydata
......@@ -355,7 +355,7 @@ the part will be reinstalled:
>>> rmdir(sample_buildout, 'mydata')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling data-dir
buildout: Installing data-dir
data-dir: Creating directory mydata
......@@ -386,7 +386,7 @@ the directory in:
We'll get a user error, not a traceback.
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
data-dir: Cannot create /xxx/mydata. /xxx is not a directory.
Error: Invalid Path
......@@ -418,9 +418,8 @@ characters, hyphens, and periods.
Variable substitutions
----------------------
Buildout configuration files support two kinds of substitutions,
standard ConfigParser substitutions, and string-template
substitutions. To illustrate this, we'll create an debug recipe to
Buildout configuration files support variable substitution.
To illustrate this, we'll create an debug recipe to
allow us to see interactions with the buildout:
>>> write(sample_buildout, 'recipes', 'debug.py',
......@@ -442,10 +441,9 @@ allow us to see interactions with the buildout:
... update = install
... """)
In this example, we've used a simple base class that provides a
boilerplate constructor. This recipe doesn't actually create
anything. The install method doesn't return anything, because it
didn't create any files or directories.
This recipe doesn't actually create anything. The install method
doesn't return anything, because it didn't create any files or
directories.
We also have to update our setup script:
......@@ -478,16 +476,11 @@ examples:
... [debug]
... recipe = recipes:debug
... File 1 = ${data-dir:path}/file
... File 2 = %(File 1)s.out
... File 3 = %(base)s/file3
... File 4 = ${debug:File 3}/log
... File 2 = ${debug:File 1}/log
...
... [data-dir]
... recipe = recipes:mkdir
... path = mydata
...
... [DEFAULT]
... base = var
... """)
In this example, we've used ConfigParser substitutions for file2 and
......@@ -504,18 +497,18 @@ Now, if we run the buildout, we'll see the options with the values
substituted.
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling data-dir
buildout: Installing data-dir
data-dir: Creating directory mydata
buildout: Installing debug
File 1 mydata/file
File 2 mydata/file.out
File 3 var/file3
File 4 var/file3/log
base var
File 1 /sample-buildout/mydata/file
File 2 /sample-buildout/mydata/file/log
recipe recipes:debug
Note that the substitution of the data-dir path option reflects the
update to the option performed by the mkdir recipe.
It might seem surprising that mydata was created again. This is
because we changed our recipes package by adding the debug module.
The buildout system didn't know if this module could effect the mkdir
......@@ -523,14 +516,11 @@ recipe, so it assumed it could and reinstalled mydata. If we rerun
the buildout:
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Updating data-dir
buildout: Updating debug
File 1 mydata/file
File 2 mydata/file.out
File 3 var/file3
File 4 var/file3/log
base var
File 1 /sample-buildout/mydata/file
File 2 /sample-buildout/mydata/file/log
recipe recipes:debug
We can see that mydata was not recreated.
......@@ -542,23 +532,96 @@ Section and option names in variable substitutions are only allowed to
contain alphanumeric characters, hyphens, periods and spaces. This
restriction might be relaxed in future releases.
Multiple configuration files
----------------------------
You can use multiple configuration files. From your main
configuration file, you can include other configuration files in 2
ways:
Automatic part selection and ordering
-------------------------------------
When a section with a recipe is refered to, either through variable
substitution or by an initializing recipe, the section is treated as a
part and added to the part list before the referencing part. For
example, we can leave data-dir out of the parts list:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
... log-level = INFO
...
... [debug]
... recipe = recipes:debug
... File 1 = ${data-dir:path}/file
... File 2 = ${debug:File 1}/log
...
... [data-dir]
... recipe = recipes:mkdir
... path = mydata
... """)
It will still be treated as a part:
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Updating data-dir
buildout: Updating debug
File 1 /sample-buildout/mydata/file
File 2 /sample-buildout/mydata/file/log
recipe recipes:debug
>>> cat('.installed.cfg') # doctest: +ELLIPSIS
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
parts = data-dir debug
...
Note that the data-dir part is included *before* the debug part,
because the debug part refers to the data-dir part. Even if we list
the data-dir part after the debug part, it will be included before:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug data-dir
... log-level = INFO
...
... [debug]
... recipe = recipes:debug
... File 1 = ${data-dir:path}/file
... File 2 = ${debug:File 1}/log
...
... [data-dir]
... recipe = recipes:mkdir
... path = mydata
... """)
- Your configuration file can "extend" another configuration file.
Option are read from the other configuration file if they aren't
already defined by your configuration file.
It will still be treated as a part:
- Your configuration file can be "extended-by" another configuration
file, In this case, the options in the other configuration file
override options in your configuration file.
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Updating data-dir
buildout: Updating debug
File 1 /sample-buildout/mydata/file
File 2 /sample-buildout/mydata/file/log
recipe recipes:debug
The configuration files your file extends or is extended by can extend
or be extended by other configuration files. The same file may be
>>> cat('.installed.cfg') # doctest: +ELLIPSIS
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
parts = data-dir debug
...
Multiple configuration files
----------------------------
A configuration file can "extend" another configuration file.
Options are read from the other configuration file if they aren't
already defined by your configuration file.
The configuration files your file extends can extend
other configuration files. The same file may be
used more than once although, of course, cycles aren't allowed.
To see how this works, we use an example:
......@@ -584,7 +647,7 @@ To see how this works, we use an example:
... """)
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling debug
buildout: Uninstalling data-dir
buildout: Installing debug
......@@ -598,20 +661,16 @@ customization.
Here is a more elaborate example.
>>> extensions = tmpdir('extensions')
>>> other = tmpdir('other')
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... extends = b1.cfg b2.cfg
... extended-by = e1.cfg %(e2)s
... extends = b1.cfg b2.cfg %(b3)s
...
... [debug]
... op = %%(name)s
...
... [DEFAULT]
... name = buildout
... """ % dict(e2=os.path.join(extensions, 'e2.cfg')))
... op = buildout
... """ % dict(b3=os.path.join(other, 'b3.cfg')))
>>> write(sample_buildout, 'b1.cfg',
... """
......@@ -619,12 +678,8 @@ Here is a more elaborate example.
... extends = base.cfg
...
... [debug]
... op1 = %(name)s 1
... op2 = %(name)s 2
... op3 = %(name)s 3
...
... [DEFAULT]
... name = b1
... op1 = b1 1
... op2 = b1 2
... """)
>>> write(sample_buildout, 'b2.cfg',
......@@ -633,90 +688,57 @@ Here is a more elaborate example.
... extends = base.cfg
...
... [debug]
... op3 = %(name)s 3
... op4 = %(name)s 4
... op5 = %(name)s 5
...
... [DEFAULT]
... name = b2
... op2 = b2 2
... op3 = b2 3
... """)
>>> write(sample_buildout, 'base.cfg',
>>> write(other, 'b3.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
... extends = b3base.cfg
...
... [debug]
... recipe = recipes:debug
... name = base
... op4 = b3 4
... """)
>>> write(sample_buildout, 'e1.cfg',
>>> write(other, 'b3base.cfg',
... """
... [debug]
... op1 = %(name)s 1
...
... [DEFAULT]
... name = e1
... op5 = b3base 5
... """)
>>> write(extensions, 'e2.cfg',
>>> write(sample_buildout, 'base.cfg',
... """
... [buildout]
... extends = eb.cfg
... extended-by = ee.cfg
... """)
>>> write(extensions, 'eb.cfg',
... """
... [debug]
... op5 = %(name)s 5
... develop = recipes
... parts = debug
...
... [DEFAULT]
... name = eb
... """)
>>> write(extensions, 'ee.cfg',
... """
... [debug]
... op6 = %(name)s 6
...
... [DEFAULT]
... name = ee
... recipe = recipes:debug
... name = base
... """)
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling debug
buildout: Installing debug
name ee
name base
op buildout
op1 e1 1
op2 b1 2
op1 b1 1
op2 b2 2
op3 b2 3
op4 b2 4
op5 eb 5
op6 ee 6
op4 b3 4
op5 b3base 5
recipe recipes:debug
There are several things to note about this example:
- We can name multiple files in an extends or extended-by option.
- We can name multiple files in an extends option.
- We can reference files recursively.
- DEFAULT sections only directly affect the configuration file they're
used in, but they can have secondary effects. For example, the name
option showed up in the debug section because it was defined in the
debug sections in several of the input files by virtue of being in
their DEFAULT sections.
- Relative file names in extended and extended-by options are
interpreted relative to the directory containing the referencing
configuration file. The files eb.cfg and ee.cfg were found in the
extensions directory because they were referenced from a file in
that directory.
- Relative file names in extended options are interpreted relative to
the directory containing the referencing configuration file.
User defaults
-------------
......@@ -737,17 +759,16 @@ delimiter.)
>>> os.environ['HOME'] = home
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling debug
buildout: Installing debug
name ee
name base
op buildout
op1 e1 1
op2 b1 2
op1 b1 1
op2 b2 2
op3 b2 3
op4 b2 4
op5 eb 5
op6 ee 6
op4 b3 4
op5 b3base 5
op7 7
recipe recipes:debug
......@@ -765,16 +786,13 @@ set the logging level to WARNING
... [buildout]
... log-level = WARNING
... extends = b1.cfg b2.cfg
... extended-by = e1.cfg
... """)
>>> print system(buildout),
name e1
op1 e1 1
op2 b1 2
name base
op1 b1 1
op2 b2 2
op3 b2 3
op4 b2 4
op5 b2 5
recipe recipes:debug
Command-line usage
......@@ -821,7 +839,7 @@ Note that we used the installed buildout option to specify an
alternate file to store information about installed parts.
>>> print system(buildout+' -c other.cfg debug:op1=foo -v'),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Installing debug
name other
op1 foo
......@@ -834,7 +852,7 @@ WARNING.
Options can also be combined in the usual Unix way, as in:
>>> print system(buildout+' -vcother.cfg debug:op1=foo'),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Updating debug
name other
op1 foo
......@@ -876,7 +894,7 @@ the buildout in the usual way:
... """)
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling debug
buildout: Installing debug
recipe recipes:debug
......@@ -898,7 +916,6 @@ the buildout in the usual way:
d d2
d d3
d develop-eggs
- e1.cfg
d eggs
d parts
d recipes
......@@ -959,7 +976,7 @@ Now we'll update our configuration file:
and run the buildout specifying just d3 and d4:
>>> print system(buildout+' install d3 d4'),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling d3
buildout: Installing d3
d3: Creating directory data3
......@@ -978,7 +995,6 @@ and run the buildout specifying just d3 and d4:
d data3
d data4
d develop-eggs
- e1.cfg
d eggs
d parts
d recipes
......@@ -991,13 +1007,19 @@ The .installed.cfg is only updated for the recipes that ran:
>>> cat(sample_buildout, '.installed.cfg')
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
parts = debug d2 d3 d4 d1
parts = debug d1 d2 d3 d4
<BLANKLINE>
[debug]
__buildout_installed__ =
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
recipe = recipes:debug
<BLANKLINE>
[d1]
__buildout_installed__ = /sample-buildout/d1
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /sample-buildout/d1
recipe = recipes:mkdir
<BLANKLINE>
[d2]
__buildout_installed__ = /sample-buildout/d2
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
......@@ -1015,12 +1037,6 @@ The .installed.cfg is only updated for the recipes that ran:
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /sample-buildout/data4
recipe = recipes:mkdir
<BLANKLINE>
[d1]
__buildout_installed__ = /sample-buildout/d1
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /sample-buildout/d1
recipe = recipes:mkdir
Note that the installed data for debug, d1, and d2 haven't changed,
because we didn't install those parts and that the d1 and d2
......@@ -1029,9 +1045,9 @@ directories are still there.
Now, if we run the buildout without the install command:
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Uninstalling d1
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling d2
buildout: Uninstalling d1
buildout: Uninstalling debug
buildout: Installing debug
recipe recipes:debug
......@@ -1055,7 +1071,6 @@ also see that d1 and d2 have gone away:
d data3
d data4
d develop-eggs
- e1.cfg
d eggs
d parts
d recipes
......@@ -1090,7 +1105,7 @@ provide alternate locations, and even names for these directories.
buildout: Creating directory /sample-alt/work
buildout: Creating directory /sample-alt/basket
buildout: Creating directory /sample-alt/developbasket
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling d4
buildout: Uninstalling d3
buildout: Uninstalling d2
......@@ -1126,7 +1141,7 @@ You can also specify an alternate buildout directory:
buildout: Creating directory /sample-alt/parts
buildout: Creating directory /sample-alt/eggs
buildout: Creating directory /sample-alt/develop-eggs
buildout: Develop: /sample-buildout/recipes/setup.py
buildout: Develop: /sample-buildout/recipes
>>> ls(alt)
- .installed.cfg
......@@ -1162,12 +1177,11 @@ of changing the format:
... parts =
... log-level = 25
... verbosity = 5
... log-format = %%(levelname)s %%(message)s
... log-format = %(levelname)s %(message)s
... """)
Here, we've changed the format to include the log-level name, rather
than the logger name. Note that we had to double percent signs,
because configuration options allow ConfigParser variable substitution.
than the logger name.
We've also illustrated, with a contrived example, that the log level
can be a numeric value and that the verbosity can be specified in the
......@@ -1175,7 +1189,7 @@ configuration file. Because the verbosity is subtracted from the log
level, we get a final log level of 20, which is the INFO level.
>>> print system(buildout),
INFO Develop: /sample-buildout/recipes/setup.py
INFO Develop: /sample-buildout/recipes
Predefined buildout options
---------------------------
......@@ -1193,6 +1207,11 @@ database is shown.
... """)
>>> print system(buildout+' -v'),
zc.buildout.easy_install: Installing ['zc.buildout', 'setuptools']
zc.buildout.easy_install: We have a develop egg for zc.buildout
zc.buildout.easy_install: We have the best distribution that satisfies
setuptools
<BLANKLINE>
Configuration data:
[buildout]
bin-directory = /sample-buildout/bin
......@@ -1201,18 +1220,15 @@ database is shown.
eggs-directory = /sample-buildout/eggs
executable = /usr/local/bin/python2.3
installed = /sample-buildout/.installed.cfg
log-format = %%(name)s: %%(message)s
log-format = %(name)s: %(message)s
log-level = INFO
offline = false
parts =
parts-directory = /sample-buildout/parts
python = buildout
verbosity = 10
<BLANKLINE>
zc.buildout.easy_install: Installing ['zc.buildout', 'setuptools']
zc.buildout.easy_install: We have a develop egg for zc.buildout
zc.buildout.easy_install: We have the best distribution that satisfies
setuptools
All of these options can be overridden by configuration files or by
command-line assignments. We've discussed most of these options
already, but let's review them and touch on some we haven't discussed:
......@@ -1377,7 +1393,7 @@ egg to be built:
>>> os.chdir(sample_bootstrapped)
>>> print system(os.path.join(sample_bootstrapped, 'bin', 'buildout')),
buildout: Develop: /sample-bootstrapped/demo/setup.py
buildout: Develop: /sample-bootstrapped/demo
Now we can add the extensions option. We were a bit tricly and ran
the buildout once with the demo develop egg defined but without the
......@@ -1398,7 +1414,7 @@ We see that out extension is loaded and executed:
>>> print system(os.path.join(sample_bootstrapped, 'bin', 'buildout')),
ext ['buildout']
buildout: Develop: /sample-bootstrapped/demo/setup.py
buildout: Develop: /sample-bootstrapped/demo
......
src/zc/buildout/easy_install.py
View file @ a4291845
......@@ -34,10 +34,13 @@ logger = logging.getLogger('zc.buildout.easy_install')
url_match = re.compile('[a-z0-9+.-]+://').match
setuptools_loc = pkg_resources.working_set.find(
pkg_resources.Requirement.parse('setuptools')
).location
# Include buildout and setuptools eggs in paths
buildout_and_setuptools_path = [
pkg_resources.working_set.find(
pkg_resources.Requirement.parse('setuptools')).location,
setuptools_loc,
pkg_resources.working_set.find(
pkg_resources.Requirement.parse('zc.buildout')).location,
]
......@@ -404,12 +407,15 @@ def build(spec, dest, build_ext,
dist = _satisfied(requirement, env, dest, executable, index_url, links)
if dist is not None:
return dist
return [dist.location]
# Get an editable version of the package to a temporary directory:
tmp = tempfile.mkdtemp('editable')
tmp2 = tempfile.mkdtemp('editable')
undo = []
try:
tmp = tempfile.mkdtemp('build')
undo.append(lambda : shutil.rmtree(tmp))
tmp2 = tempfile.mkdtemp('build')
undo.append(lambda : shutil.rmtree(tmp2))
index = _get_index(executable, index_url, links)
dist = index.fetch_distribution(requirement, tmp2, False, True)
if dist is None:
......@@ -442,13 +448,105 @@ def build(spec, dest, build_ext,
setuptools.command.setopt.edit_config(
setup_cfg, dict(build_ext=build_ext))
# Now run easy_install for real:
tmp3 = tempfile.mkdtemp('build', dir=dest)
undo.append(lambda : shutil.rmtree(tmp3))
_call_easy_install(base, env, pkg_resources.WorkingSet(),
dest, links, index_url, executable, True)
tmp3, links, index_url, executable, True)
return _copyeggs(tmp3, dest, '.egg', undo)
finally:
shutil.rmtree(tmp)
shutil.rmtree(tmp2)
undo.reverse()
[f() for f in undo]
def _rm(*paths):
for path in paths:
if os.path.isdir(path):
shutil.rmtree(path)
elif os.path.exists(path):
os.remove(path)
def _copyeggs(src, dest, suffix, undo):
result = []
undo.append(lambda : _rm(*result))
for name in os.listdir(src):
if name.endswith(suffix):
new = os.path.join(dest, name)
_rm(new)
os.rename(os.path.join(src, name), new)
result.append(new)
assert len(result) == 1
undo.pop()
return result[0]
def develop(setup, dest,
build_ext=None,
executable=sys.executable):
if os.path.isdir(setup):
directory = setup
setup = os.path.join(directory, 'setup.py')
else:
directory = os.path.dirname(setup)
undo = []
try:
if build_ext:
setup_cfg = os.path.join(directory, 'setup.cfg')
if os.path.exists(setup_cfg):
os.rename(setup_cfg, setup_cfg+'-develop-aside')
def restore_old_setup():
if os.path.exists(setup_cfg):
os.remove(setup_cfg)
os.rename(setup_cfg+'-develop-aside', setup_cfg)
undo.append(restore_old_setup)
else:
open(setup_cfg, 'w')
undo.append(lambda: os.remove(setup_cfg))
setuptools.command.setopt.edit_config(
setup_cfg, dict(build_ext=build_ext))
fd, tsetup = tempfile.mkstemp()
undo.append(lambda: os.remove(tsetup))
undo.append(lambda: os.close(fd))
os.write(fd, runsetup_template % dict(
setuptools=setuptools_loc,
setupdir=directory,
setup=setup,
__file__ = setup,
))
tmp3 = tempfile.mkdtemp('build', dir=dest)
undo.append(lambda : shutil.rmtree(tmp3))
args = [
zc.buildout.easy_install._safe_arg(tsetup),
'-q', 'develop', '-mxN',
'-d', _safe_arg(tmp3),
]
log_level = logger.getEffectiveLevel()
if log_level <= logging.DEBUG:
if log_level == logging.DEBUG:
del args[1]
else:
args[1] == '-v'
logger.debug("in: %s\n%r", directory, args)
assert os.spawnl(os.P_WAIT, executable, executable, *args) == 0
return _copyeggs(tmp3, dest, '.egg-link', undo)
finally:
undo.reverse()
[f() for f in undo]
def working_set(specs, executable, path):
return install(specs, None, executable=executable, path=path)
......@@ -595,6 +693,15 @@ if _interactive:
import code
code.interact(banner="", local=globals())
'''
runsetup_template = """
import sys
sys.path.insert(0, %(setuptools)r)
import os, setuptools
__file__ = %(__file__)r
os.chdir(%(setupdir)r)
sys.argv[0] = %(setup)r
execfile(%(setup)r)
"""
src/zc/buildout/easy_install.txt
View file @ a4291845
Minimal Python interface to easy_install
========================================
Python API for egg and script installation
==========================================
The easy_install module provides a minimal interface to the setuptools
easy_install command that provides some additional semantics:
The easy_install module provides some functions to provide support for
egg and script installation. It provides functionality at the python
level that is similar to easy_install, with a few exceptions:
- By default, we look for new packages *and* the packages that
they depend on. This is somewhat like (and uses) the --upgrade
......@@ -21,8 +22,8 @@ easy_install command that provides some additional semantics:
- Distutils options for building extensions can be passed.
The easy_install module provides a method, install, for installing one
or more packages and their dependencies. The
install function takes 2 positional arguments:
or more packages and their dependencies. The install function takes 2
positional arguments:
- An iterable of setuptools requirement strings for the distributions
to be installed, and
......@@ -426,18 +427,18 @@ You can also pass script initialization code:
eggrecipedemo.main(1, 2)
Handling custom build options for extensions
--------------------------------------------
Handling custom build options for extensions provided in source distributions
-----------------------------------------------------------------------------
Sometimes, we need to control how extension modules are built. The
build method provides this level of control. It takes a single
build function provides this level of control. It takes a single
package specification, downloads a source distribution, and builds it
with specified custom build options.
The build method takes 3 positional arguments:
The build function takes 3 positional arguments:
spec
A package specification
A package specification for a source distribution
dest
A destination directory
......@@ -486,9 +487,13 @@ extension, extdemo.c::
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
PyObject *m;
m = Py_InitModule3("extdemo", methods, "");
#ifdef TWO
PyModule_AddObject(m, "val", PyInt_FromLong(2));
#else
PyModule_AddObject(m, "val", PyInt_FromLong(EXTDEMO));
#endif
}
The extension depends on a system-dependnt include file, extdemo.h,
......@@ -497,9 +502,11 @@ that defines a constant, EXTDEMO, that is exposed by the extension.
We'll add an include directory to our sample buildout and add the
needed include file to it:
>>> mkdir(sample_buildout, 'include')
>>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
... "#define EXTDEMO 42\n")
>>> mkdir('include')
>>> write('include', 'extdemo.h',
... """
... #define EXTDEMO 42
... """)
Now, we can use the build function to create an egg from the source
distribution:
......@@ -508,6 +515,9 @@ distribution:
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/')
'/sample-install/extdemo-1.4-py2.4-unix-i686.egg'
The function returns the list of eggs
Now if we look in our destination directory, we see we have an extdemo egg:
......@@ -516,3 +526,68 @@ Now if we look in our destination directory, we see we have an extdemo egg:
d demoneeded-1.1-py2.4.egg
d extdemo-1.4-py2.4-unix-i686.egg
Handling custom build options for extensions in develop eggs
------------------------------------------------------------
The develop function is similar to the build function, except that,
rather than building an egg from a source directory containing a
setup.py script.
The develop function takes 2 positional arguments:
setup
The path to a setup script, typically named "setup.py", or a
directory containing a setup.py script.
dest
The directory to install the egg link to
It supports some optional keyword argument:
build_ext
A dictionary of options to be passed to the distutils build_ext
command when building extensions.
executable
A path to a Python executable. Distributions will ne installed
using this executable and will be for the matching Python version.
We have a local directory containing the extdemo source:
>>> ls(extdemo)
- MANIFEST
- MANIFEST.in
- README
- extdemo.c
- setup.py
Now, we can use the develop function to create a develop egg from the source
distribution:
>>> zc.buildout.easy_install.develop(
... extdemo, dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')})
'/sample-install/extdemo.egg-link'
The name of the egg link created is returned.
Now if we look in our destination directory, we see we have an extdemo
egg link:
>>> ls(dest)
d demo-0.3-py2.4.egg
d demoneeded-1.1-py2.4.egg
d extdemo-1.4-py2.4-linux-i686.egg
- extdemo.egg-link
And that the source directory contains the compiled extension:
>>> ls(extdemo)
- MANIFEST
- MANIFEST.in
- README
d build
- extdemo.c
d extdemo.egg-info
- extdemo.so
- setup.py
src/zc/buildout/tests.py
View file @ a4291845
......@@ -45,7 +45,7 @@ We should be able to deal with setup scripts that aren't setuptools based.
... ''')
>>> print system(join('bin', 'buildout')),
buildout: Develop: /sample-buildout/foo/setup.py
buildout: Develop: /sample-buildout/foo
>>> ls('develop-eggs')
- foo.egg-link
......@@ -71,9 +71,8 @@ We should be able to deal with setup scripts that aren't setuptools based.
... ''')
>>> print system(join('bin', 'buildout')+' -v'), # doctest: +ELLIPSIS
Configuration data:
...
buildout: Develop: /sample-buildout/foo/setup.py
zc.buildout...
buildout: Develop: /sample-buildout/foo
...
Installed /sample-buildout/foo
...
......@@ -86,7 +85,7 @@ We should be able to deal with setup scripts that aren't setuptools based.
def buildout_error_handling():
r"""Buildout error handling
Asking for a section that doesn't exist, yields a key error:
Asking for a section that doesn't exist, yields a missing section error:
>>> import os
>>> os.chdir(sample_buildout)
......@@ -95,7 +94,7 @@ Asking for a section that doesn't exist, yields a key error:
>>> buildout['eek']
Traceback (most recent call last):
...
KeyError: 'eek'
MissingSection: The referenced section, 'eek', was not defined.
Asking for an option that doesn't exist, a MissingOption error is raised:
......@@ -109,8 +108,7 @@ It is an error to create a variable-reference cycle:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = data_dir debug
... parts =
... x = ${buildout:y}
... y = ${buildout:z}
... z = ${buildout:x}
......@@ -183,7 +181,7 @@ Al parts have to have a section:
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
Error: No section was specified for part x
Error: The referenced section, 'x', was not defined.
and all parts have to have a specified recipe:
......@@ -265,15 +263,15 @@ def test_comparing_saved_options_with_funny_characters():
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print system(buildout), # doctest: +ELLIPSIS
buildout: Develop: ...setup.py
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Installing debug
If we run the buildout again, we shoudn't get a message about
uninstalling anything because the configuration hasn't changed.
>>> print system(buildout), # doctest: +ELLIPSIS
buildout: Develop: ...setup.py
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Updating debug
"""
......@@ -317,20 +315,21 @@ Then try to install it again:
"""
def error_for_indefined_install_parts():
"""
Any parts we pass to install on the command line must be
listed in the configuration.
# Why?
## def error_for_undefined_install_parts():
## """
## Any parts we pass to install on the command line must be
## listed in the configuration.
>>> print system(join('bin', 'buildout') + ' install foo'),
buildout: Invalid install parts: foo.
Install parts must be listed in the configuration.
## >>> print system(join('bin', 'buildout') + ' install foo'),
## buildout: Invalid install parts: foo.
## Install parts must be listed in the configuration.
>>> print system(join('bin', 'buildout') + ' install foo bar'),
buildout: Invalid install parts: foo bar.
Install parts must be listed in the configuration.
## >>> print system(join('bin', 'buildout') + ' install foo bar'),
## buildout: Invalid install parts: foo bar.
## Install parts must be listed in the configuration.
"""
## """
bootstrap_py = os.path.join(
......@@ -515,7 +514,7 @@ Create a develop egg:
... """)
>>> print system(join('bin', 'buildout')),
buildout: Develop: /sample-buildout/foo/setup.py
buildout: Develop: /sample-buildout/foo
>>> ls('develop-eggs')
- foox.egg-link
......@@ -536,8 +535,8 @@ Create another:
... """)
>>> print system(join('bin', 'buildout')),
buildout: Develop: /sample-buildout/foo/setup.py
buildout: Develop: /sample-buildout/bar/setup.py
buildout: Develop: /sample-buildout/foo
buildout: Develop: /sample-buildout/bar
>>> ls('develop-eggs')
- foox.egg-link
......@@ -552,7 +551,7 @@ Remove one:
... parts =
... """)
>>> print system(join('bin', 'buildout')),
buildout: Develop: /sample-buildout/bar/setup.py
buildout: Develop: /sample-buildout/bar
It is gone
......@@ -611,7 +610,7 @@ a devlop egg, we will also generate a warning.
... """)
>>> print system(join('bin', 'buildout')),
buildout: Develop: /sample-buildout/foo/setup.py
buildout: Develop: /sample-buildout/foo
Now, if we generate a working set using the egg link, we will get a warning
and we will get setuptools included in the working set.
......@@ -659,7 +658,6 @@ We do not get a warning, but we do get setuptools included in the working set:
... ])]
['foox', 'setuptools']
>>> print handler,
We get the same behavior if the it is a depedency that uses a
......@@ -682,8 +680,8 @@ namespace package.
... """)
>>> print system(join('bin', 'buildout')),
buildout: Develop: /sample-buildout/foo/setup.py
buildout: Develop: /sample-buildout/bar/setup.py
buildout: Develop: /sample-buildout/foo
buildout: Develop: /sample-buildout/bar
>>> [dist.project_name
... for dist in zc.buildout.easy_install.working_set(
......@@ -703,6 +701,52 @@ namespace package.
>>> handler.uninstall()
'''
def develop_preserves_existing_setup_cfg():
"""
See "Handling custom build options for extensions in develop eggs" in
easy_install.txt. This will be very similar except that we'll have an
existing setup.cfg:
>>> write(extdemo, "setup.cfg",
... '''
... # sampe cfg file
...
... [foo]
... bar = 1
...
... [build_ext]
... define = X,Y
... ''')
>>> mkdir('include')
>>> write('include', 'extdemo.h',
... '''
... #define EXTDEMO 42
... ''')
>>> dest = tmpdir('dest')
>>> zc.buildout.easy_install.develop(
... extdemo, dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')})
'/tmp/tmp7AFYXv/_TEST_/dest/extdemo.egg-link'
>>> ls(dest)
- extdemo.egg-link
>>> cat(extdemo, "setup.cfg")
<BLANKLINE>
# sampe cfg file
<BLANKLINE>
[foo]
bar = 1
<BLANKLINE>
[build_ext]
define = X,Y
"""
def create_sample_eggs(test, executable=sys.executable):
write = test.globs['write']
......@@ -762,9 +806,13 @@ static PyMethodDef methods[] = {{NULL}};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
PyObject *m;
m = Py_InitModule3("extdemo", methods, "");
#ifdef TWO
PyModule_AddObject(m, "val", PyInt_FromLong(2));
#else
PyModule_AddObject(m, "val", PyInt_FromLong(EXTDEMO));
#endif
}
"""
......@@ -778,8 +826,8 @@ setup(name = "extdemo", version = "1.4", url="http://www.zope.org",
"""
def add_source_dist(test):
import tarfile
tmp = tempfile.mkdtemp('test-sdist')
tmp = test.globs['extdemo'] = test.globs['tmpdir']('extdemo')
write = test.globs['write']
try:
write(tmp, 'extdemo.c', extdemo_c);
......@@ -930,7 +978,7 @@ def test_suite():
]),
),
doctest.DocTestSuite(
setUp=zc.buildout.testing.buildoutSetUp,
setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=renormalizing.RENormalizing([
zc.buildout.testing.normalize_path,
......
src/zc/buildout/update.txt
View file @ a4291845
......@@ -74,7 +74,7 @@ new versions found in new releases:
zc.buildout version 99.99,
setuptools version 99.99;
restarting.
buildout: Develop: /sample-buildout/showversions/setup.py
buildout: Develop: /sample-buildout/showversions
buildout: Installing show-versions
zc.buildout 99.99
setuptools 99.99
......@@ -122,7 +122,7 @@ We'll actually "upgrade" to an earlier version.
zc.buildout version 1.0.0,
setuptools version 0.6;
restarting.
buildout: Develop: /sample-buildout/showversions/setup.py
buildout: Develop: /sample-buildout/showversions
buildout: Updating show-versions
zc.buildout 1.0.0
setuptools 0.6
......@@ -143,7 +143,7 @@ We won't upgrade in offline mode:
... """ % dict(new_releases=new_releases))
>>> print system(buildout),
buildout: Develop: /sample-buildout/showversions/setup.py
buildout: Develop: /sample-buildout/showversions
buildout: Updating show-versions
zc.buildout 1.0.0
setuptools 0.6
......@@ -174,4 +174,3 @@ directory:
buildout: Not upgrading because not running a local buildout command
>>> ls('bin')
zc.recipe.egg_/CHANGES.txt
View file @ a4291845
......@@ -8,6 +8,27 @@ To do
Change History
**************
1.0.0b3 (2006-12-04)
====================
Feature Changes
---------------
- Added a develop recipe for creating develop eggs.
This is useful to:
- Specify custom extension building options,
- Specify a version of Python to use, and to
- Cause develop eggs to be created after other parts.
- The develop and build recipes now return the paths created, so that
created eggs or egg links are removed when a part is removed (or
changed).
1.0.0b2 (2006-10-16)
====================
......
zc.recipe.egg_/setup.py
View file @ a4291845
......@@ -42,7 +42,9 @@ setup(
tests_require = ['zope.testing'],
test_suite = name+'.tests.test_suite',
entry_points = {'zc.buildout': ['default = %s:Egg' % name,
'script = %s:Egg' % name,
'custom = %s:Custom' % name,
'develop = %s:Develop' % name,
]
},
zip_safe=False,
......
zc.recipe.egg_/src/zc/recipe/egg/__init__.py
View file @ a4291845
from zc.recipe.egg.egg import Egg
from zc.recipe.egg.custom import Custom
from zc.recipe.egg.custom import Custom, Develop
zc.recipe.egg_/src/zc/recipe/egg/custom.py
View file @ a4291845
......@@ -19,12 +19,27 @@ $Id$
import os, re, zipfile
import zc.buildout.easy_install
class Custom:
class Base:
def __init__(self, buildout, name, options):
self.buildout = buildout
self.name = name
self.options = options
self.name, self.options = name, options
options['_d'] = buildout['buildout']['develop-eggs-directory']
python = options.get('python', buildout['buildout']['python'])
options['executable'] = buildout[python]['executable']
self.build_ext = build_ext(buildout, options)
def update(self):
return self.install()
class Custom(Base):
def __init__(self, buildout, name, options):
Base.__init__(self, buildout, name, options)
links = options.get('find-links',
buildout['buildout'].get('find-links'))
if links:
......@@ -39,47 +54,66 @@ class Custom:
options['index'] = index
self.index = index
options['_b'] = buildout['buildout']['bin-directory']
options['_e'] = buildout['buildout']['eggs-directory']
options['_d'] = buildout['buildout']['develop-eggs-directory']
assert options.get('unzip') in ('true', 'false', None)
python = options.get('python', buildout['buildout']['python'])
options['executable'] = buildout[python]['executable']
build_ext = {}
for be_option in ('include-dirs', 'library-dirs', 'rpath'):
value = options.get(be_option)
if value is None:
continue
value = [
os.path.join(
buildout['buildout']['directory'],
v.strip()
)
for v in value.strip().split('\n')
if v.strip()
]
build_ext[be_option] = ':'.join(value)
options[be_option] = ':'.join(value)
self.build_ext = build_ext
if buildout['buildout'].get('offline') == 'true':
self.install = lambda: ()
def install(self):
if self.buildout['buildout'].get('offline') == 'true':
return ()
options = self.options
distribution = options.get('eggs', self.name).strip()
build_ext = dict([
(k, options[k])
for k in ('include-dirs', 'library-dirs', 'rpath')
if k in options
])
zc.buildout.easy_install.build(
return zc.buildout.easy_install.build(
distribution, options['_d'], self.build_ext,
self.links, self.index, options['executable'], [options['_e']],
)
return ()
class Develop(Base):
def __init__(self, buildout, name, options):
Base.__init__(self, buildout, name, options)
options['setup'] = os.path.join(buildout['buildout']['directory'],
options['setup'])
def install(self):
options = self.options
return zc.buildout.easy_install.develop(
options['setup'], options['_d'], self.build_ext,
options['executable'],
)
def build_ext(buildout, options):
result = {}
for be_option in ('include-dirs', 'library-dirs', 'rpath'):
value = options.get(be_option)
if value is None:
continue
value = [
os.path.join(
buildout['buildout']['directory'],
v.strip()
)
for v in value.strip().split('\n')
if v.strip()
]
result[be_option] = os.pathsep.join(value)
options[be_option] = os.pathsep.join(value)
swig = options.get('swig')
if swig:
options['swig'] = result['swig'] = os.path.join(
buildout['buildout']['directory'],
swig,
)
for be_option in ('define', 'undef', 'libraries', 'link-objects',
'debug', 'force', 'compiler', 'swig-cpp', 'swig-opts',
):
value = options.get(be_option)
if value is None:
continue
result[be_option] = value
update = install
return result
zc.recipe.egg_/src/zc/recipe/egg/custom.txt
View file @ a4291845
Custon eggs
===========
Creating eggs with extensions neededing custom build settings
=============================================================
Sometimes, It's necessary to provide extra control over how an egg is
created. This is commonly true for eggs with extension modules that
......@@ -20,6 +20,42 @@ rpath
A new-line separated list of directories to search for dynamic libraries
at run time.
define
A comma-separated list of names of C preprocessor variables to
define.
undef
A comman separated list of names of C preprocessor variables to
undefine.
libraries
The name of an additional library to link with. Due to limitations
in distutils and desprite the option name, only a single library
can be specified.
link-objects
The name of an link object to link afainst. Due to limitations
in distutils and desprite the option name, only a single link object
can be specified.
debug
Compile/link with debugging information
force
Forcibly build everything (ignore file timestamps)
compiler
Specify the compiler type
swig
The path to the swig executable
swig-cpp
Make SWIG create C++ files (default is C)
swig-opts
List of SWIG command line options
In addition, the following options can be used to specify the egg:
egg
......@@ -57,9 +93,13 @@ package that has a simple extension module::
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
PyObject *m;
m = Py_InitModule3("extdemo", methods, "");
#ifdef TWO
PyModule_AddObject(m, "val", PyInt_FromLong(2));
#else
PyModule_AddObject(m, "val", PyInt_FromLong(EXTDEMO));
#endif
}
The extension depends on a system-dependnt include file, extdemo.h,
......@@ -71,10 +111,11 @@ extdemo-1.4.tar.gz, on a distribution server.
We have a sample buildout that we'll add an include directory to with
the necessary include file:
>>> mkdir(sample_buildout, 'include')
>>> import os
>>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
... "#define EXTDEMO 42\n")
>>> mkdir('include')
>>> write('include', 'extdemo.h',
... """
... #define EXTDEMO 42
... """)
We'll also update the buildout configuration file to define a part for
the egg:
......@@ -91,8 +132,7 @@ the egg:
... include-dirs = include
... """ % dict(server=link_server))
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> buildout = join('bin', 'buildout')
>>> print system(buildout),
buildout: Installing extdemo
......@@ -113,3 +153,177 @@ Note that no scripts or dependencies are installed. To install
dependencies or scripts for a custom egg, define another part and use
the zc.recipe.egg recipe, listing the custom egg as one of the eggs to
be installed. The zc.recipe.egg recipe will use the installed egg.
Let's define a script that uses out ext demo:
>>> mkdir('demo')
>>> write('demo', 'demo.py',
... """
... import extdemo
... def main():
... print extdemo.val
... """)
>>> write('demo', 'setup.py',
... """
... from setuptools import setup
... setup(name='demo')
... """)
>>> write('buildout.cfg',
... """
... [buildout]
... develop = demo
... parts = extdemo demo
...
... [extdemo]
... recipe = zc.recipe.egg:custom
... find-links = %(server)s
... index = %(server)s/index
... include-dirs = include
...
... [demo]
... recipe = zc.recipe.egg
... eggs = demo
... extdemo
... entry-points = demo=demo:main
... """ % dict(server=link_server))
>>> print system(buildout),
buildout: Develop: /sample-buildout/demo
buildout: Updating extdemo
buildout: Installing demo
When we run the script, we'll 42 printed:
>>> print system(join('bin', 'demo')),
42
Controlling develop-egg generation
==================================
If you want to provide custom build options for a develop egg, you can
use the develop recipe. The recipe has the following options:
path
The path to a setup script or directory containing a startup
script. This is required.
include-dirs
A new-line separated list of directories to search for include
files.
library-dirs
A new-line separated list of directories to search for libraries
to link with.
rpath
A new-line separated list of directories to search for dynamic libraries
at run time.
define
A comma-separated list of names of C preprocessor variables to
define.
undef
A comman separated list of names of C preprocessor variables to
undefine.
libraries
The name of an additional library to link with. Due to limitations
in distutils and desprite the option name, only a single library
can be specified.
link-objects
The name of an link object to link afainst. Due to limitations
in distutils and desprite the option name, only a single link object
can be specified.
debug
Compile/link with debugging information
force
Forcibly build everything (ignore file timestamps)
compiler
Specify the compiler type
swig
The path to the swig executable
swig-cpp
Make SWIG create C++ files (default is C)
swig-opts
List of SWIG command line options
python
The name of a section to get the Python executable from.
If not specified, then the buildout python option is used. The
Python executable is found in the executable option of the named
section.
To illustrate this, we'll use a directory containing the extdemo
example from the earlier section:
>>> ls(extdemo)
- MANIFEST
- MANIFEST.in
- README
- extdemo.c
- setup.py
>>> write('buildout.cfg',
... """
... [buildout]
... develop = demo
... parts = extdemo demo
...
... [extdemo]
... setup = %(extdemo)s
... recipe = zc.recipe.egg:develop
... include-dirs = include
... define = TWO
...
... [demo]
... recipe = zc.recipe.egg
... eggs = demo
... extdemo
... entry-points = demo=demo:main
... """ % dict(extdemo=extdemo))
Note that we added a define option to cause the preprocessor variable
TWO to be defined. This will cause the module-variable, 'val', to be
set with a value of 2.
>>> print system(buildout),
buildout: Develop: /tmp/tmpCXjRps/_TEST_/sample-buildout/demo
buildout: Uninstalling extdemo
buildout: Installing extdemo
buildout: Updating demo
Our develop-eggs now includes an egg link for extdemo:
>>> ls('develop-eggs')
- demo.egg-link
- extdemo.egg-link
- zc.recipe.egg.egg-link
and the extdemo now has a built extension:
>>> ls(extdemo)
- MANIFEST
- MANIFEST.in
- README
d build
- extdemo.c
d extdemo.egg-info
- extdemo.so
- setup.py
Because develop eggs take precedence over non-develop eggs, the demo
script will use the new develop egg:
>>> print system(join('bin', 'demo')),
2
zc.recipe.egg_/src/zc/recipe/egg/tests.py
View file @ a4291845
......@@ -78,6 +78,7 @@ def test_suite():
'custom.txt',
setUp=setUp, tearDown=zc.buildout.testing.buildoutTearDown,
checker=renormalizing.RENormalizing([
zc.buildout.testing.normalize_path,
(re.compile("(d ((ext)?demo(needed)?|other)"
"-\d[.]\d-py)\d[.]\d(-\S+)?[.]egg"),
'\\1V.V.egg'),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7