Skip to content

S
slapos.buildout
Commit e4f4f913 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Factor out setuptools support from the buildout

parent 2495c2b8
Show whitespace changes
Inline Side-by-side
Showing with 3580 additions and 0 deletions
zc.setuptools/zc/__init__.py 0 → 100644
View file @ e4f4f913
try:
__import__('pkg_resources').declare_namespace(__name__)
except:
# bootstrapping
pass
zc.setuptools/zc/buildout/buildout.py 0 → 100644
View file @ e4f4f913
#############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Buildout main script
$Id$
"""
import logging
import md5
import os
import pprint
import re
import shutil
import sys
import ConfigParser
import zc.buildout.easy_install
import pkg_resources
import zc.buildout.easy_install
class UserError(Exception):
"""Errors made by a user
"""
def __str__(self):
return " ".join(map(str, self))
class MissingOption(UserError, KeyError):
"""A required option was missing
"""
class MissingSection(UserError, KeyError):
"""A required section is missinh
"""
class Options(dict):
def __init__(self, buildout, section, data):
self.buildout = buildout
self.section = section
super(Options, self).__init__(data)
def __getitem__(self, option):
try:
return super(Options, self).__getitem__(option)
except KeyError:
raise MissingOption("Missing option: %s:%s"
% (self.section, option))
# XXX need test
def __setitem__(self, option, value):
if not isinstance(value, str):
raise TypeError('Option values must be strings', value)
super(Options, self).__setitem__(option, value)
def copy(self):
return Options(self.buildout, self.section, self)
class Buildout(dict):
def __init__(self, config_file, cloptions):
config_file = os.path.abspath(config_file)
self._config_file = config_file
if not os.path.exists(config_file):
print 'Warning: creating', config_file
open(config_file, 'w').write('[buildout]\nparts = \n')
super(Buildout, self).__init__()
# default options
data = dict(buildout={
'directory': os.path.dirname(config_file),
'eggs-directory': 'eggs',
'develop-eggs-directory': 'develop-eggs',
'bin-directory': 'bin',
'parts-directory': 'parts',
'installed': '.installed.cfg',
'python': 'buildout',
'executable': sys.executable,
'log-level': 'WARNING',
'log-format': '%(name)s: %(message)s',
})
# load user defaults, which override defaults
if 'HOME' in os.environ:
user_config = os.path.join(os.environ['HOME'],
'.buildout', 'default.cfg')
if os.path.exists(user_config):
_update(data, _open(os.path.dirname(user_config), user_config,
[]))
# load configuration files
_update(data, _open(os.path.dirname(config_file), config_file, []))
# apply command-line options
for (section, option, value) in cloptions:
options = data.get(section)
if options is None:
options = self[section] = {}
options[option] = value
# The egg dire
# do substitutions
converted = {}
for section, options in data.iteritems():
for option, value in options.iteritems():
if '$' in value:
value = self._dosubs(section, option, value,
data, converted, [])
options[option] = value
converted[(section, option)] = value
# copy data into self:
for section, options in data.iteritems():
self[section] = Options(self, section, options)
# initialize some attrs and buildout directories.
options = self['buildout']
links = options.get('find-links', '')
self._links = links and links.split() or ()
self._buildout_dir = options['directory']
for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
d = self._buildout_path(options[name+'-directory'])
options[name+'-directory'] = d
options['installed'] = os.path.join(options['directory'],
options['installed'])
self._setup_logging()
def _dosubs(self, section, option, value, data, converted, seen):
key = section, option
r = converted.get(key)
if r is not None:
return r
if key in seen:
raise UserError("Circular reference in substitutions.\n"
"We're evaluating %s\nand are referencing: %s.\n"
% (", ".join([":".join(k) for k in seen]),
":".join(key)
)
)
seen.append(key)
value = '$$'.join([self._dosubs_esc(s, data, converted, seen)
for s in value.split('$$')
])
seen.pop()
return value
_template_split = re.compile('([$]{[^}]*})').split
_simple = re.compile('[-a-zA-Z0-9 ._]+$').match
_valid = re.compile('[-a-zA-Z0-9 ._]+:[-a-zA-Z0-9 ._]+$').match
def _dosubs_esc(self, value, data, converted, seen):
value = self._template_split(value)
subs = []
for ref in value[1::2]:
s = tuple(ref[2:-1].split(':'))
if not self._valid(ref):
if len(s) < 2:
raise UserError("The substitution, %s,\n"
"doesn't contain a colon."
% ref)
if len(s) > 2:
raise UserError("The substitution, %s,\n"
"has too many colons."
% ref)
if not self._simple(s[0]):
raise UserError("The section name in substitution, %s,\n"
"has invalid characters."
% ref)
if not self._simple(s[1]):
raise UserError("The option name in substitution, %s,\n"
"has invalid characters."
% ref)
v = converted.get(s)
if v is None:
options = data.get(s[0])
if options is None:
raise MissingSection(
"Referenced section does not exist", s[0])
v = options.get(s[1])
if v is None:
raise MissingOption("Referenced option does not exist:",
*s)
if '$' in v:
v = self._dosubs(s[0], s[1], v, data, converted, seen)
options[s[1]] = v
converted[s] = v
subs.append(v)
subs.append('')
return ''.join([''.join(v) for v in zip(value[::2], subs)])
def _buildout_path(self, *names):
return os.path.join(self._buildout_dir, *names)
def bootstrap(self, args):
self._setup_directories()
# Now copy buildout and setuptools eggs, amd record destination eggs:
entries = []
for name in 'setuptools', 'zc.buildout':
r = pkg_resources.Requirement.parse(name)
dist = pkg_resources.working_set.find(r)
if dist.precedence == pkg_resources.DEVELOP_DIST:
dest = os.path.join(self['buildout']['eggs-directory'],
name+'.egg-link')
open(dest, 'w').write(dist.location)
entries.append(dist.location)
else:
dest = os.path.join(self['buildout']['eggs-directory'],
os.path.basename(dist.location))
entries.append(dest)
if not os.path.exists(dest):
if os.path.isdir(dist.location):
shutil.copytree(dist.location, dest)
else:
shutil.copy2(dist.location, dest)
# Create buildout script
ws = pkg_resources.WorkingSet(entries)
ws.require('zc.buildout')
zc.buildout.easy_install.scripts(
['zc.buildout'], ws, sys.executable,
self['buildout']['bin-directory'])
def install(self, install_parts):
self._setup_directories()
# Add develop-eggs directory to path so that it gets searched
# for eggs:
sys.path.insert(0, self['buildout']['develop-eggs-directory'])
# Build develop eggs
self._develop()
# load installed data
installed_part_options = self._read_installed_part_options()
# get configured and installed part lists
conf_parts = self['buildout']['parts']
conf_parts = conf_parts and conf_parts.split() or []
installed_parts = installed_part_options['buildout']['parts']
installed_parts = installed_parts and installed_parts.split() or []
# If install_parts is given, then they must be listed in parts
# and we don't uninstall anything. Otherwise, we install
# the configured parts and uninstall anything else.
if install_parts:
extra = [p for p in install_parts if p not in conf_parts]
if extra:
self._error('Invalid install parts:', *extra)
uninstall_missing = False
else:
install_parts = conf_parts
uninstall_missing = True
# load recipes
recipes = self._load_recipes(install_parts)
# compute new part recipe signatures
self._compute_part_signatures(install_parts)
try:
# uninstall parts that are no-longer used or who's configs
# have changed
for part in reversed(installed_parts):
if part in install_parts:
old_options = installed_part_options[part].copy()
old_options.pop('__buildout_installed__')
new_options = self.get(part)
if old_options == new_options:
continue
for k in old_options:
if k not in new_options:
self._logger.debug("Part: %s, dropped option %s",
part, k)
elif old_options[k] != new_options[k]:
self._logger.debug(
"Part: %s, option %s, %r != %r",
part, k, new_options[k], old_options[k],
)
for k in new_options:
if k not in old_options:
self._logger.debug("Part: %s, new option %s",
part, k)
elif not uninstall_missing:
continue
# ununstall part
self._logger.info('Uninstalling %s', part)
self._uninstall(
installed_part_options[part]['__buildout_installed__'])
installed_parts = [p for p in installed_parts if p != part]
# install new parts
for part in install_parts:
self._logger.info('Installing %s', part)
installed_part_options[part] = self[part].copy()
del self[part]['__buildout_signature__']
installed_files = recipes[part].install() or ()
if isinstance(installed_files, str):
installed_files = [installed_files]
installed_part_options[part]['__buildout_installed__'] = (
'\n'.join(installed_files)
)
if part not in installed_parts:
installed_parts.append(part)
finally:
installed_part_options['buildout']['parts'] = ' '.join(
[p for p in conf_parts if p in installed_parts]
+
[p for p in installed_parts if p not in conf_parts]
)
self._save_installed_options(installed_part_options)
def _setup_directories(self):
# Create buildout directories
for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
d = self['buildout'][name+'-directory']
if not os.path.exists(d):
self._logger.info('Creating directory %s', d)
os.mkdir(d)
def _develop(self):
"""Install sources by running setup.py develop on them
"""
develop = self['buildout'].get('develop')
if develop:
here = os.getcwd()
try:
for setup in develop.split():
setup = self._buildout_path(setup)
if os.path.isdir(setup):
setup = os.path.join(setup, 'setup.py')
self._logger.info("Running %s -q develop ...", setup)
os.chdir(os.path.dirname(setup))
os.spawnle(
os.P_WAIT, sys.executable, sys.executable,
zc.buildout.easy_install._safe_arg(setup),
'-q', 'develop', '-m', '-x', '-N',
'-f', zc.buildout.easy_install._safe_arg(
' '.join(self._links)
),
'-d', zc.buildout.easy_install._safe_arg(
self['buildout']['develop-eggs-directory']
),
{'PYTHONPATH':
os.path.dirname(pkg_resources.__file__)},
)
finally:
os.chdir(here)
def _load_recipes(self, parts):
recipes = {}
if not parts:
return recipes
recipes_requirements = []
pkg_resources.working_set.add_entry(
self['buildout']['develop-eggs-directory'])
pkg_resources.working_set.add_entry(self['buildout']['eggs-directory'])
# Gather requirements
for part in parts:
options = self.get(part)
if options is None:
raise MissingSection("No section was specified for part", part)
recipe, entry = self._recipe(part, options)
recipes_requirements.append(recipe)
# Install the recipe distros
offline = self['buildout'].get('offline', 'false')
if offline not in ('true', 'false'):
self._error('Invalif value for offline option: %s', offline)
if offline == 'true':
ws = zc.buildout.easy_install.working_set(
recipes_requirements, sys.executable,
[self['buildout']['eggs-directory'],
self['buildout']['develop-eggs-directory'],
],
)
else:
ws = zc.buildout.easy_install.install(
recipes_requirements, self['buildout']['eggs-directory'],
links=self._links, index=self['buildout'].get('index'),
path=[self['buildout']['develop-eggs-directory']])
# Add the distros to the working set
pkg_resources.require(recipes_requirements)
# instantiate the recipes
for part in parts:
options = self[part]
recipe, entry = self._recipe(part, options)
recipe_class = pkg_resources.load_entry_point(
recipe, 'zc.buildout', entry)
recipes[part] = recipe_class(self, part, options)
return recipes
def _compute_part_signatures(self, parts):
# Compute recipe signature and add to options
base = self['buildout']['eggs-directory'] + os.path.sep
for part in parts:
options = self.get(part)
if options is None:
options = self[part] = {}
recipe, entry = self._recipe(part, options)
req = pkg_resources.Requirement.parse(recipe)
sig = _dists_sig(pkg_resources.working_set.resolve([req]), base)
options['__buildout_signature__'] = ' '.join(sig)
def _recipe(self, part, options):
recipe = options['recipe']
if ':' in recipe:
recipe, entry = recipe.split(':')
else:
entry = 'default'
return recipe, entry
def _read_installed_part_options(self):
old = self._installed_path()
if os.path.isfile(old):
parser = ConfigParser.SafeConfigParser(_spacey_defaults)
parser.optionxform = lambda s: s
parser.read(old)
return dict([
(section,
Options(self, section,
[item for item in parser.items(section)
if item[0] not in _spacey_defaults]
)
)
for section in parser.sections()])
else:
return {'buildout': Options(self, 'buildout', {'parts': ''})}
def _installed_path(self):
return self._buildout_path(self['buildout']['installed'])
def _uninstall(self, installed):
for f in installed.split():
f = self._buildout_path(f)
if os.path.isdir(f):
shutil.rmtree(f)
elif os.path.isfile(f):
os.remove(f)
def _install(self, part):
options = self[part]
recipe, entry = self._recipe(part, options)
recipe_class = pkg_resources.load_entry_point(
recipe, 'zc.buildout', entry)
installed = recipe_class(self, part, options).install()
if installed is None:
installed = []
elif isinstance(installed, basestring):
installed = [installed]
base = self._buildout_path('')
installed = [d.startswith(base) and d[len(base):] or d
for d in installed]
return ' '.join(installed)
def _save_installed_options(self, installed_options):
f = open(self._installed_path(), 'w')
_save_options('buildout', installed_options['buildout'], f)
for part in installed_options['buildout']['parts'].split():
print >>f
_save_options(part, installed_options[part], f)
f.close()
def _error(self, message, *args, **kw):
self._logger.error(message, *args, **kw)
sys.exit(1)
def _setup_logging(self):
root_logger = logging.getLogger()
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(self['buildout']['log-format']))
root_logger.addHandler(handler)
self._logger = logging.getLogger('buildout')
level = self['buildout']['log-level']
if level in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
level = getattr(logging, level)
else:
try:
level = int(level)
except ValueError:
self._error("Invalid logging level %s", level)
verbosity = self['buildout'].get('verbosity', 0)
try:
verbosity = int(verbosity)
except ValueError:
self._error("Invalid verbosity %s", verbosity)
level -= verbosity
root_logger.setLevel(level)
if level <= logging.DEBUG:
sections = list(self)
sections.sort()
print 'Configuration data:'
for section in sections:
_save_options(section, self[section], sys.stdout)
print
_spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*'
'|'
'^[ \t\r\f\v]+'
'|'
'[ \t\r\f\v]+$'
)
def _quote_spacey_nl(match):
match = match.group(0).split('\n', 1)
result = '\n\t'.join(
[(s
.replace(' ', '%(__buildout_space__)s')
.replace('\r', '%(__buildout_space_r__)s')
.replace('\f', '%(__buildout_space_f__)s')
.replace('\v', '%(__buildout_space_v__)s')
.replace('\n', '%(__buildout_space_n__)s')
)
for s in match]
)
return result
_spacey_defaults = dict(
__buildout_space__ = ' ',
__buildout_space_r__ = '\r',
__buildout_space_f__ = '\f',
__buildout_space_v__ = '\v',
__buildout_space_n__ = '\n',
)
def _save_options(section, options, f):
print >>f, '[%s]' % section
items = options.items()
items.sort()
for option, value in items:
value = value.replace('%', '%%')
value = _spacey_nl.sub(_quote_spacey_nl, value)
if value.startswith('\n\t'):
value = '%(__buildout_space_n__)s' + value[2:]
if value.endswith('\n\t'):
value = value[:-2] + '%(__buildout_space_n__)s'
print >>f, option, '=', value
def _open(base, filename, seen):
"""Open a configuration file and return the result as a dictionary,
Recursively open other files based on buildout options found.
"""
filename = os.path.join(base, filename)
if filename in seen:
raise UserError("Recursive file include", seen, filename)
base = os.path.dirname(filename)
seen.append(filename)
result = {}
parser = ConfigParser.SafeConfigParser()
parser.optionxform = lambda s: s
parser.readfp(open(filename))
extends = extended_by = None
for section in parser.sections():
options = dict(parser.items(section))
if section == 'buildout':
extends = options.pop('extends', extends)
extended_by = options.pop('extended-by', extended_by)
result[section] = options
if extends:
extends = extends.split()
extends.reverse()
for fname in extends:
result = _update(_open(base, fname, seen), result)
if extended_by:
for fname in extended_by.split():
result = _update(result, _open(base, fname, seen))
seen.pop()
return result
def _dir_hash(dir):
hash = md5.new()
for (dirpath, dirnames, filenames) in os.walk(dir):
filenames[:] = [f for f in filenames
if not (f.endswith('pyc') or f.endswith('pyo'))
]
hash.update(' '.join(dirnames))
hash.update(' '.join(filenames))
for name in filenames:
hash.update(open(os.path.join(dirpath, name)).read())
return hash.digest().encode('base64').strip()
def _dists_sig(dists, base):
result = []
for dist in dists:
location = dist.location
if dist.precedence == pkg_resources.DEVELOP_DIST:
result.append(dist.project_name + '-' + _dir_hash(location))
else:
if location.startswith(base):
location = location[len(base):]
result.append(location)
return result
def _update(d1, d2):
for section in d2:
if section in d1:
d1[section].update(d2[section])
else:
d1[section] = d2[section]
return d1
def _error(*message):
sys.stderr.write('Error: ' + ' '.join(message) +'\n')
sys.exit(1)
def main(args=None):
if args is None:
args = sys.argv[1:]
config_file = 'buildout.cfg'
verbosity = 0
options = []
while args:
if args[0][0] == '-':
op = orig_op = args.pop(0)
op = op[1:]
while op and op[0] in 'vq':
if op[0] == 'v':
verbosity += 10
else:
verbosity -= 10
op = op[1:]
if op[:1] == 'c':
op = op[1:]
if op:
config_file = op
else:
if args:
config_file = args.pop(0)
else:
_error("No file name specified for option", orig_op)
elif op:
_error("Invalid option", '-'+op[0])
elif '=' in args[0]:
option, value = args.pop(0).split('=', 1)
if len(option.split(':')) != 2:
_error('Invalid option:', option)
section, option = option.split(':')
options.append((section.strip(), option.strip(), value.strip()))
else:
# We've run out of command-line options and option assignnemnts
# The rest should be commands, so we'll stop here
break
if verbosity:
options.append(('buildout', 'verbosity', str(verbosity)))
if args:
command = args.pop(0)
if command not in ('install', 'bootstrap'):
_error('invalid command:', command)
else:
command = 'install'
try:
try:
buildout = Buildout(config_file, options)
getattr(buildout, command)(args)
except UserError, v:
_error(str(v))
finally:
logging.shutdown()
if sys.version_info[:2] < (2, 4):
def reversed(iterable):
result = list(iterable);
result.reverse()
return result
zc.setuptools/zc/buildout/buildout.txt 0 → 100644
View file @ e4f4f913
Buildouts
=========
The word "buildout" refers to a description of a set of parts and the
software to create and assemble them. It is often used informally to
refer to an installed system based on a buildout definition. For
example, if we are creating an application named "Foo", then "the Foo
buildout" is the collection of configuration and application-specific
software that allows an instance of the application to be created. We
may refer to such an instance of the application informally as "a Foo
buildout".
This document describes how to define buildouts using buildout
configuration files and recipes. There are three ways to set up the
buildout software and create a buildout instance:
1. Install the zc.buildout egg with easy_install and use the buildout
script installed in a Python scripts area.
2. Use the buildout bootstrap script to create a buildout that
includes both the setuptools and zc.buildout eggs. This allows you
to use the buildout software without modifying a Python install.
The buildout script is installed into your buildout local scripts
area.
3. Use a buildoput command from an already installed buildout to
bootstrap a new buildout. (See the section on bootstraping later
in this document.)
Often, a software project will be managed in a software repository,
such as a subversion repository, that includes some software source
directories, buildout configuration files, and a copy of the buildout
bootstrap script, To work on the project, one would check out the
project from the repository and run the bootstrap script which
installs setuptools and zc.buildout into the checkout as well as any
parts defined.
We have a sample buildout that we created using the bootstrap command
of an existing buildout (method 3 above). It has the absolute minimum
information. We have bin, develop-eggs, eggs and parts directories,
and a configuration file:
>>> ls(sample_buildout)
d bin
- buildout.cfg
d develop-eggs
d eggs
d parts
The bin directory contains scripts.
>>> ls(sample_buildout, 'bin')
- buildout
- py-zc.buildout
>>> ls(sample_buildout, 'eggs')
- setuptools-0.6-py2.4.egg
- zc.buildout-1.0-py2.4.egg
The develop-eggs and parts directories are initially empty:
>>> ls(sample_buildout, 'develop-eggs')
>>> ls(sample_buildout, 'parts')
The develop-eggs directory holds egg links for software being
developed in the buildout. We separate develop-eggs and other eggs to
allow eggs directories to be shared across multiple buildouts. For
example, a common developer technique is to define a common eggs
directory in their home that all non-develop eggs are stored in. This
allows larger buildouts to be set up much more quickly and saves disk
space.
The parts directory provides an area where recipes can install
part data. For example, if we built a custom Python, we would
install it in the part directory. Part data is stored in a
sub-directory of the parts directory with the same name as the part.
Buildouts are defined using configuration files. These are in the
format defined by the Python ConfigParser module, with extensions
that we'll describe later. By default, when a buildout is run, it
looks for the file buildout.cfg in the directory where the buildout is
run.
The minimal configuration file has a buildout section that defines no
parts:
>>> cat(sample_buildout, 'buildout.cfg')
[buildout]
parts =
A part is simply something to be created by a buildout. It can be
almost anything, such as a Python package, a program, a directory, or
even a configuration file.
A part is created by a recipe. Recipes are always installed as Python
eggs. They can be downloaded from a package server, such as the
Python Package Index, or they can be developed as part of a project.
Let's create a recipe as part of the sample project. We'll create a
recipe for creating directories.
First, we'll create a recipes directory for
our local recipes:
>>> mkdir(sample_buildout, 'recipes')
and then we'll create a source file for our mkdir recipe:
>>> write(sample_buildout, 'recipes', 'mkdir.py',
... """
... import logging, os
...
... class Mkdir:
...
... def __init__(self, buildout, name, options):
... self.buildout = buildout
... self.name = name
... self.options = options
... options['path'] = os.path.join(
... buildout['buildout']['directory'],
... options['path'],
... )
...
... def install(self):
... path = self.options['path']
... if not os.path.isdir(path):
... logging.getLogger(self.name).info(
... 'Creating directory %s', os.path.basename(path))
... os.mkdir(path)
... return path
... """)
The recipe defines a constructor that takes a buildout object, a part
name, and an options dictionary. It saves them in instance attributes.
If the path is relative, we'll interpret it as relative to the
buildout directory. The buildout object passed in is a mapping from
section name to a mapping of options for that section. The buildout
directory is available as the directory option of the buildout
section. We normalize the path and save it back into the options
directory.
Any time we use data from another section, it is important to reflect
that data in the recipe's options when the recipe is constructed.
When buildout is run, it saves configuration data for installed parts
in a file named installed.cfg. In subsequent runs, it compares
part-configuration data stored in the installed.cfg file and the
part-configuration data loaded from the configuration files as
modified by recipe constructors to decide if the configuration of a
part has changed. If the configuration has changed, or if the recipe
has changed, then the part is uninstalled before reinstalling it. The
buildout only looks at the part's options, so any data used to
configure the part needs to be reflected in the part's options. It is
the job of a recipe constructor to make sure that the options include
all rel event data.
Of course, parts are also uninstalled if they are no-longer used.
The install method is responsible for creating the part. In this
case, we need the path of the directory to create. We'll use a
path option from our options dictionary.
The install method logs what it's doing using the Python logging call.
We return the path that we installed. If the part is uninstalled or
reinstalled, then the path returned will be removed by the buildout
machinery. A recipe install method is expected to return None, a
string, or an iterable of strings containing paths to be removed if a
part is uninstalled. For most recipes, this is all of the uninstall
support needed. A recipe can provide custom uninstall support as will
be described later.
We need to provide packaging information so that our recipe can be
installed as an egg. We need to define a setup script for this:
>>> write(sample_buildout, 'recipes', 'setup.py',
... """
... from setuptools import setup
...
... setup(
... name = "recipes",
... entry_points = {'zc.buildout': ['mkdir = mkdir:Mkdir']},
... )
... """)
This setup script is incomplete. It doesn't describe what is to be
included in a distribution. This is fine if we never actually create
a distribution. If recipes are going to be used only internally in a
buildout, then we needn't include distribution information. If we
wanted to use the same recipes in multiple buildouts, then we'd need
to include proper distribution data. To find out more about creating
distributions, see the setuptools documentation.
Our setup script defines an entry point. Entry points provide
a way for an egg to define the services it provides. Here we've said
that we define a zc.buildout entry point named default. Recipe
classes must be exposed as entry points in the zc.buildout group. we
give entry points names within the group. The name "default" is
somewhat special because it allows a recipe to be referenced using a
package name without naming an entry point.
We also need a README.txt for our recipes to avoid an annoying warning
from distutils, on which setuptools and zc.buildout are based:
>>> write(sample_buildout, 'recipes', 'README.txt', " ")
Now let's update our buildout.cfg:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = data-dir
... log-level = INFO
...
... [data-dir]
... recipe = recipes:mkdir
... path = mystuff
... """)
Let's go through the changes one by one::
develop = recipes
This tells the buildout to install a development egg for our recipes.
Any number of paths can be listed. The paths can be relative or
absolute. If relative, they are treated as relative to the buildout
directory. They can be directory or file paths. If a file path is
given, it should point to a Python setup script. If a directory path
is given, it should point to a directory containing a setup.py file.
Development eggs are installed before building any parts, as they may
provide locally-defined recipes needed by the parts.
::
parts = data-dir
Here we've named a part to be "built". We can use any name we want
except that different part names must be unique and recipes will often
use the part name to decide what to do.
::
log-level = INFO
The default level is WARNING, which is fairly quite. In this example,
we set the level to INFO so we can see more details about what the
buildout and recipes are doing.
::
[data-dir]
recipe = recipes:mkdir
path = mystuff
When we name a part, we also create a section of the same
name that contains part data. In this section, we'll define
the recipe to be used to install the part. In this case, we also
specify the path to be created.
Let's run the buildout. We do so by running the build script in the
buildout:
>>> import os
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print system(buildout),
buildout: Running /tmp/sample-buildout/recipes/setup.py -q develop ...
buildout: Installing data-dir
data-dir: Creating directory mystuff
We see that the recipe created the directory, as expected:
>>> ls(sample_buildout)
- .installed.cfg
d bin
- buildout.cfg
d develop-eggs
d eggs
d mystuff
d parts
d recipes
In addition, .installed.cfg has been created containing information
about the part we installed:
>>> cat(sample_buildout, '.installed.cfg')
[buildout]
parts = data-dir
<BLANKLINE>
[data-dir]
__buildout_installed__ = /tmp/sample-buildout/mystuff
__buildout_signature__ = recipes-c7vHV6ekIDUPy/7fjAaYjg==
path = /tmp/sample-buildout/mystuff
recipe = recipes:mkdir
Note that the directory we installed is included in .installed.cfg.
In addition, the path option includes the actual destination
directory.
If we change the name of the directory in the configuration file,
we'll see that the directory gets removed and recreated:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = data-dir
... log-level = INFO
...
... [data-dir]
... recipe = recipes:mkdir
... path = mydata
... """)
>>> print system(buildout),
buildout: Running /tmp/sample-buildout/recipes/setup.py -q develop ...
buildout: Uninstalling data-dir
buildout: Installing data-dir
data-dir: Creating directory mydata
>>> ls(sample_buildout)
- .installed.cfg
d bin
- buildout.cfg
d develop-eggs
d eggs
d mydata
d parts
d recipes
Configuration file syntax
-------------------------
As mentioned earlier, buildout configuration files use the format
defined by the Python ConfigParser module with extensions. The
extensions are:
- option names are case sensitive
- option values can ue a substitution syntax, described below, to
refer to option values in specific sections.
The ConfigParser syntax is very flexible. Section names can contain
any characters other than newlines and right square braces ("]").
Option names can contain any characters other than newlines, colons,
and equal signs, can not start with a space, and don't include
trailing spaces.
It is likely that, in the future, some characters will be given
special buildout-defined meanings. This is already true of the
characters ":", "$", "%", "(", and ")". For now, it is a good idea to
keep section and option names simple, sticking to alphanumeric
characters, hyphens, and periods.
Variable substitutions
----------------------
Buildout configuration files support two kinds of substitutions,
standard ConfigParser substitutions, and string-template
substitutions. To illustrate this, we'll create an debug recipe to
allow us to see interactions with the buildout:
>>> write(sample_buildout, 'recipes', 'debug.py',
... """
... class Debug:
...
... def __init__(self, buildout, name, options):
... self.buildout = buildout
... self.name = name
... self.options = options
...
... def install(self):
... items = self.options.items()
... items.sort()
... for option, value in items:
... print option, value
... """)
In this example, we've used a simple base class that provides a
boilerplate constructor. This recipe doesn't actually create
anything. The install method doesn't return anything, because it
didn't create any files or directories.
We also have to update our setup script:
>>> write(sample_buildout, 'recipes', 'setup.py',
... """
... from setuptools import setup
... entry_points = (
... '''
... [zc.buildout]
... mkdir = mkdir:Mkdir
... debug = debug:Debug
... ''')
... setup(name="recipes", entry_points=entry_points)
... """)
We've rearranged the script a bit to make the entry points easier to
edit. In particular, entry points are now defined as a configuration
string, rather than a dictionary.
Let's update our configuration to provide variable substitution
examples:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = data-dir debug
... log-level = INFO
...
... [debug]
... recipe = recipes:debug
... File 1 = ${data-dir:path}/file
... File 2 = %(File 1)s.out
... File 3 = %(base)s/file3
... File 4 = ${debug:File 3}/log
...
... [data-dir]
... recipe = recipes:mkdir
... path = mydata
...
... [DEFAULT]
... base = var
... """)
In this example, we've used ConfigParser substitutions for file2 and
file3. This type of substitution uses Python string format syntax.
Valid names are options in the same section and options defined in the
DEFAULT section.
We used a string-template substitution for file1. This type of
substitution uses the string.Template syntax. Names substituted are
qualified option names, consisting of a section name and option name
joined by a colon.
Now, if we run the buildout, we'll see the options with the values
substituted.
>>> print system(buildout),
buildout: Running /tmp/sample-buildout/recipes/setup.py -q develop ...
buildout: Uninstalling data-dir
buildout: Installing data-dir
data-dir: Creating directory mydata
buildout: Installing debug
File 1 mydata/file
File 2 mydata/file.out
File 3 var/file3
File 4 var/file3/log
base var
recipe recipes:debug
It might seem surprising that mydata was created again. This is
because we changed our recipes package by adding the debug module.
The buildout system didn't know if this module could effect the mkdir
recipe, so it assumed it could and reinstalled mydata. If we rerun
the buildout:
>>> print system(buildout),
buildout: Running /tmp/sample-buildout/recipes/setup.py -q develop ...
buildout: Installing data-dir
buildout: Installing debug
File 1 mydata/file
File 2 mydata/file.out
File 3 var/file3
File 4 var/file3/log
base var
recipe recipes:debug
We can see that mydata was not recreated.
Note that, in this case, we didn't specify a log level, so
we didn't get output about what the buildout was doing.
Section and option names in variable substitutions are only allowed to
contain alphanumeric characters, hyphens, periods and spaces. This
restriction might be relaxed in future releases.
Multiple configuration files
----------------------------
You can use multiple configuration files. From your main
configuration file, you can include other configuration files in 2
ways:
- Your configuration file can "extend" another configuration file.
Option are read from the other configuration file if they aren't
already defined by your configuration file.
- Your configuration file can be "extended-by" another configuration
file, In this case, the options in the other configuration file
override options in your configuration file.
The configuration files your file extends or is extended by can extend
or be extended by other configuration files. The same file may be
used more than once although, of course, cycles aren't allowed.
To see how this works, we use an example:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... extends = base.cfg
...
... [debug]
... op = buildout
... """)
>>> write(sample_buildout, 'base.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
...
... [debug]
... recipe = recipes:debug
... op = base
... """)
>>> print system(buildout),
op buildout
recipe recipes:debug
The example is pretty trivial, but the pattern it illustrates is
pretty common. In a more practical example, the base buildout might
represent a product and the extending buildout might be a
customization.
Here is a more elaborate example.
>>> extensions = mkdtemp()
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... extends = b1.cfg b2.cfg
... extended-by = e1.cfg %(e2)s
...
... [debug]
... op = %%(name)s
...
... [DEFAULT]
... name = buildout
... """ % dict(e2=os.path.join(extensions, 'e2.cfg')))
>>> write(sample_buildout, 'b1.cfg',
... """
... [buildout]
... extends = base.cfg
...
... [debug]
... op1 = %(name)s 1
... op2 = %(name)s 2
... op3 = %(name)s 3
...
... [DEFAULT]
... name = b1
... """)
>>> write(sample_buildout, 'b2.cfg',
... """
... [buildout]
... extends = base.cfg
...
... [debug]
... op3 = %(name)s 3
... op4 = %(name)s 4
... op5 = %(name)s 5
...
... [DEFAULT]
... name = b2
... """)
>>> write(sample_buildout, 'base.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
...
... [debug]
... recipe = recipes:debug
... name = base
... """)
>>> write(sample_buildout, 'e1.cfg',
... """
... [debug]
... op1 = %(name)s 1
...
... [DEFAULT]
... name = e1
... """)
>>> write(extensions, 'e2.cfg',
... """
... [buildout]
... extends = eb.cfg
... extended-by = ee.cfg
... """)
>>> write(extensions, 'eb.cfg',
... """
... [debug]
... op5 = %(name)s 5
...
... [DEFAULT]
... name = eb
... """)
>>> write(extensions, 'ee.cfg',
... """
... [debug]
... op6 = %(name)s 6
...
... [DEFAULT]
... name = ee
... """)
>>> print system(buildout),
name ee
op buildout
op1 e1 1
op2 b1 2
op3 b2 3
op4 b2 4
op5 eb 5
op6 ee 6
recipe recipes:debug
There are several things to note about this example:
- We can name multiple files in an extends or extended-by option.
- We can reference files recursively.
- DEFAULT sections only directly affect the configuration file they're
used in, but they can have secondary effects. For example, the name
option showed up in the debug section because it was defined in the
debug sections in several of the input files by virtue of being in
their DEFAULT sections.
- Relative file names in extended and extended-by options are
interpreted relative to the directory containing the referencing
configuration file. The files eb.cfg and ee.cfg were found in the
extensions directory because they were referenced from a file in
that directory.
User defaults
-------------
If the file $HOME/.buildout/defaults.cfg, exists, it is read before
reading the configuration file. ($HOME is the value of the HOME
environment variable. The '/' is replaced by the operating system file
delimiter.)
>>> home = mkdtemp()
>>> mkdir(home, '.buildout')
>>> write(home, '.buildout', 'default.cfg',
... """
... [debug]
... op1 = 1
... op7 = 7
... """)
>>> os.environ['HOME'] = home
>>> print system(buildout),
name ee
op buildout
op1 e1 1
op2 b1 2
op3 b2 3
op4 b2 4
op5 eb 5
op6 ee 6
op7 7
recipe recipes:debug
>>> del os.environ['HOME']
Command-line usage
------------------
A number of arguments can be given on the buildout command line. The
command usage is::
buildout [-c file] [-q] [-v] [assignments] [command [command arguments]]
The -c option can be used to specify a configuration file, rather than
buildout.cfg in the current directory.
The -q and -v decrement and increment the verbosity by 10. The
verbosity is used to adjust the logging level. The verbosity is
subtracted from the numeric value of the log-level option specified in
the configuration file.
Assignments are of the form::
section_name:option_name=value
Options and assignments can be given in any order.
Here's an example:
>>> write(sample_buildout, 'other.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
... installed = .other.cfg
...
... [debug]
... name = other
... recipe = recipes:debug
... """)
Note that we used the installed buildout option to specify an
alternate file to store information about installed parts.
>>> print system(buildout+' -c other.cfg debug:op1=foo -v'),
buildout: Running /tmp/sample-buildout/recipes/setup.py -q develop ...
buildout: Installing debug
name other
op1 foo
recipe recipes:debug
Here we used the -c option to specify an alternate configuration file,
and the -v option to increase the level of logging from the default,
WARNING.
Options can also be combined in the usual Unix way, as in:
>>> print system(buildout+' -vcother.cfg debug:op1=foo'),
buildout: Running /tmp/sample-buildout/recipes/setup.py -q develop ...
buildout: Installing debug
name other
op1 foo
recipe recipes:debug
Here we combined the -v and -c options with the configuration file
name. Note that the -c option has to be last, because it takes an
argument.
>>> os.remove(os.path.join(sample_buildout, 'other.cfg'))
>>> os.remove(os.path.join(sample_buildout, '.other.cfg'))
The most commonly used command is 'install' and it takes a
list of parts to install. if any parts are specified, then they must
be listed in the buildout parts option and only those parts are
installed. To illustrate this, we'll update our configuration and run
the buildout in the usual way:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug d1 d2 d3
...
... [d1]
... recipe = recipes:mkdir
... path = d1
...
... [d2]
... recipe = recipes:mkdir
... path = d2
...
... [d3]
... recipe = recipes:mkdir
... path = d3
...
... [debug]
... recipe = recipes:debug
... """)
>>> print system(buildout+' -v'),
buildout: Running /sample-buildout/recipes/setup.py -q develop ...
buildout: Uninstalling debug
buildout: Installing debug
recipe recipes:debug
buildout: Installing d1
d1: Creating directory d1
buildout: Installing d2
d2: Creating directory d2
buildout: Installing d3
d3: Creating directory d3
>>> ls(sample_buildout)
- .installed.cfg
- b1.cfg
- b2.cfg
- base.cfg
d bin
- buildout.cfg
d d1
d d2
d d3
d develop-eggs
- e1.cfg
d eggs
d parts
d recipes
>>> cat(sample_buildout, '.installed.cfg')
[buildout]
parts = debug d1 d2 d3
<BLANKLINE>
[debug]
__buildout_installed__ =
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
recipe = recipes:debug
<BLANKLINE>
[d1]
__buildout_installed__ = /tmp/sample-buildout/d1
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /tmp/sample-buildout/d1
recipe = recipes:mkdir
<BLANKLINE>
[d2]
__buildout_installed__ = /tmp/sample-buildout/d2
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /tmp/sample-buildout/d2
recipe = recipes:mkdir
<BLANKLINE>
[d3]
__buildout_installed__ = /tmp/sample-buildout/d3
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /tmp/sample-buildout/d3
recipe = recipes:mkdir
Now we'll update our configuration file:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug d2 d3 d4
...
... [d2]
... recipe = recipes:mkdir
... path = data2
...
... [d3]
... recipe = recipes:mkdir
... path = data3
...
... [d4]
... recipe = recipes:mkdir
... path = data4
...
... [debug]
... recipe = recipes:debug
... x = 1
... """)
and run the buildout specifying just d3 and d4:
>>> print system(buildout+' -v install d3 d4'),
buildout: Running /sample-buildout/recipes/setup.py -q develop ...
buildout: Uninstalling d3
buildout: Installing d3
d3: Creating directory data3
buildout: Installing d4
d4: Creating directory data4
>>> ls(sample_buildout)
- .installed.cfg
- b1.cfg
- b2.cfg
- base.cfg
d bin
- buildout.cfg
d d1
d d2
d data3
d data4
d develop-eggs
- e1.cfg
d eggs
d parts
d recipes
Only the d3 and d4 recipes ran. d3 was removed and data3 and data4
were created.
The .installed.cfg is only updated for the recipes that ran:
>>> cat(sample_buildout, '.installed.cfg')
[buildout]
parts = debug d2 d3 d4 d1
<BLANKLINE>
[debug]
__buildout_installed__ =
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
recipe = recipes:debug
<BLANKLINE>
[d2]
__buildout_installed__ = /tmp/sample-buildout/d2
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /tmp/sample-buildout/d2
recipe = recipes:mkdir
<BLANKLINE>
[d3]
__buildout_installed__ = /tmp/sample-buildout/data3
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /tmp/sample-buildout/data3
recipe = recipes:mkdir
<BLANKLINE>
[d4]
__buildout_installed__ = /tmp/sample-buildout/data4
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /tmp/sample-buildout/data4
recipe = recipes:mkdir
<BLANKLINE>
[d1]
__buildout_installed__ = /tmp/sample-buildout/d1
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
path = /tmp/sample-buildout/d1
recipe = recipes:mkdir
Note that the installed data for debug, d1, and d2 haven't changed,
because we didn't install those parts and that the d1 and d2
directories are still there.
Now, if we run the buildout without the install command:
>>> print system(buildout+' -v'),
buildout: Running /sample-buildout/recipes/setup.py -q develop ...
buildout: Uninstalling d1
buildout: Uninstalling d2
buildout: Uninstalling debug
buildout: Installing debug
recipe recipes:debug
x 1
buildout: Installing d2
d2: Creating directory data2
buildout: Installing d3
buildout: Installing d4
We see the output of the debug recipe and that data2 was created. We
also see that d1 and d2 have gone away:
>>> ls(sample_buildout)
- .installed.cfg
- b1.cfg
- b2.cfg
- base.cfg
d bin
- buildout.cfg
d data2
d data3
d data4
d develop-eggs
- e1.cfg
d eggs
d parts
d recipes
Alternate directory and file locations
--------------------------------------
The buildout normally puts the bin, eggs, and parts directories in the
directory in the directory containing the configuration file. You can
provide alternate locations, and even names for these directories.
>>> alt = mkdtemp('sample-alt')
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts =
... develop-eggs-directory = %(developbasket)s
... eggs-directory = %(basket)s
... bin-directory = %(scripts)s
... parts-directory = %(work)s
... """ % dict(
... developbasket = os.path.join(alt, 'developbasket'),
... basket = os.path.join(alt, 'basket'),
... scripts = os.path.join(alt, 'scripts'),
... work = os.path.join(alt, 'work'),
... ))
>>> print system(buildout+' -v'),
buildout: Creating directory /tmp/sample-alt/scripts
buildout: Creating directory /tmp/sample-alt/work
buildout: Creating directory /tmp/sample-alt/basket
buildout: Creating directory /sample-alt/developbasket
buildout: Running /tmp/sample-buildout/recipes/setup.py -q develop ...
buildout: Uninstalling d4
buildout: Uninstalling d3
buildout: Uninstalling d2
buildout: Uninstalling debug
>>> ls(alt)
d basket
d developbasket
d scripts
d work
>>> ls(alt, 'developbasket')
- recipes.egg-link
You can also specify an alternate buildout directory:
>>> alt = mkdtemp('sample-alt')
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... directory = %(alt)s
... develop = %(recipes)s
... parts =
... """ % dict(
... alt=alt,
... recipes=os.path.join(sample_buildout, 'recipes'),
... ))
>>> print system(buildout+' -v'),
buildout: Creating directory /tmp/sample-alt/bin
buildout: Creating directory /tmp/sample-alt/parts
buildout: Creating directory /tmp/sample-alt/eggs
buildout: Creating directory /tmp/sample-alt/develop-eggs
buildout: Running /tmp/sample-buildout/recipes/setup.py -q develop ...
>>> ls(alt)
- .installed.cfg
d bin
d develop-eggs
d eggs
d parts
>>> ls(alt, 'develop-eggs')
- recipes.egg-link
Logging control
---------------
Three buildout options are used to control logging:
log-level
specifies the log level
verbosity
adjusts the log level
log-format
allows an alternate logging for mat to be specified
We've already seen the log level and verbosity. Let's look at an example
of changing the format:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts =
... log-level = 25
... verbosity = 5
... log-format = %%(levelname)s %%(message)s
... """)
Here, we've changed the format to include the log-level name, rather
than the logger name. Note that we had to double percent signs,
because configuration options allow ConfigParser variable substitution.
We've also illustrated, with a contrived example, that the log level
can be a numeric value and that the verbosity can be specified in the
configuration file. Because the verbosity is subtracted from the log
level, we get a final log level of 20, which is the INFO level.
>>> print system(buildout),
INFO Running /tmp/sample-buildout/recipes/setup.py -q develop ...
Predefined buildout options
---------------------------
Buildouts have a number of predefined options that recipes can use
and that users can override in their configuration files. To see
these, we'll run a minimal buildout configuration with a debug logging
level. One of the features of debug logging is that the configuration
database is shown.
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... parts =
... """)
>>> print system(buildout+' -vv'),
Configuration data:
[buildout]
bin-directory = /tmp/sample-buildout/bin
develop-eggs-directory = /tmp/sample-buildout/develop-eggs
directory = /tmp/sample-buildout
eggs-directory = /tmp/sample-buildout/eggs
executable = /usr/local/bin/python2.3
installed = /tmp/sample-buildout/.installed.cfg
log-format = %%(name)s: %%(message)s
log-level = WARNING
parts =
parts-directory = /tmp/sample-buildout/parts
python = buildout
verbosity = 20
<BLANKLINE>
All of these options can be overridden by configuration files or by
command-line assignments. We've discussed most of these options
already, but let's review them and touch on some we haven't discussed:
bin-directory
The directory path where scripts are written. This can be a
relative path, which is interpreted relative to the directory
option.
develop-eggs-directory
The directory path where development egg links are created for software
being created in the local project. This can be a relative path,
which is interpreted relative to the directory option.
directory
The buildout directory. This is the base for other buildout file
and directory locations, when relative locations are used.
eggs-directory
The directory path where downloaded eggs are put. It is common to share
this directory across buildouts. Eggs in this directory should
*never* be modified. This can be a relative path, which is
interpreted relative to the directory option.
executable
The Python executable used to run the buildout. See the python
option below.
installed
The file path where information about the results of the previous
buildout run is written. This can be a relative path, which is
interpreted relative to the directory option. This file provides
an inventory of installed parts with information needed to decide
which if any parts need to be uninstalled.
log-format
The format used for logging messages.
log-level
The log level before verbosity adjustment
parts
A white space separated list of parts to be installed.
parts-directory
A working directory that parts can used to store data.
python
The name of a section containing information about the default
Python interpreter. Recipes that need a installation
typically have options to tell them which Python installation to
use. By convention, if a section-specific option isn't used, the
option is looked for in the buildout section. The option must
point to a section with an executable option giving the path to a
Python executable. By default, the buildout section defines the
default Python as the Python used to run the buildout.
verbosity
A log-level adjustment. Typically, this is set via the -q and -v
command-line options.
Bootstrapping
-------------
If zc.buildout is installed, you can use it to create a new buildout
with it's own local copies of zc.buildout and setuptools and with
local buildout scripts.
>>> sample_bootstrapped = mkdtemp('sample-bootstrapped')
>>> print system(buildout
... +' -c'+os.path.join(sample_bootstrapped, 'setup.cfg')
... +' bootstrap'),
Warning: creating /sample-bootstrapped/setup.cfg
Note that a basic setup.cfg was created for us.
>>> ls(sample_bootstrapped)
d bin
d develop-eggs
d eggs
d parts
- setup.cfg
>>> ls(sample_bootstrapped, 'bin')
- buildout
- py-zc.buildout
>>> ls(sample_bootstrapped, 'eggs')
- setuptools-0.6-py2.3.egg
- zc.buildout-1.0-py2.3.egg
Note that the buildout script was installed but not run. To run
the buildout, we'd have to run the installed buildout script.
zc.setuptools/zc/buildout/easy_install.py 0 → 100644
View file @ e4f4f913
##############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Python easy_install API
This module provides a high-level Python API for installing packages.
It doesn't install scripts. It uses setuptools and requires it to be
installed.
$Id$
"""
import logging, os, re, tempfile, sys
import pkg_resources, setuptools.command.setopt
import zc.buildout
# XXX we could potentially speed this up quite a bit by keeping our
# own PackageIndex to analyse whether there are newer dists. A hitch
# is that the package index seems to go out of its way to only handle
# one Python version at a time. :(
logger = logging.getLogger('zc.buildout.easy_install')
# Include buildout and setuptools eggs in paths
buildout_and_setuptools_path = [
(('.egg' in m.__file__)
and m.__file__[:m.__file__.rfind('.egg')+4]
or os.path.dirname(m.__file__)
)
for m in (pkg_resources,)
]
buildout_and_setuptools_path += [
(('.egg' in m.__file__)
and m.__file__[:m.__file__.rfind('.egg')+4]
or os.path.dirname(os.path.dirname(os.path.dirname(m.__file__)))
)
for m in (zc.buildout,)
]
_versions = {sys.executable: '%d.%d' % sys.version_info[:2]}
def _get_version(executable):
try:
return _versions[executable]
except KeyError:
i, o = os.popen4(executable + ' -V')
i.close()
version = o.read().strip()
o.close()
pystring, version = version.split()
assert pystring == 'Python'
version = re.match('(\d[.]\d)[.]\d$', version).group(1)
_versions[executable] = version
return version
def _satisfied(req, env):
dists = env[req.project_name]
best = None
for dist in dists:
if (dist.precedence == pkg_resources.DEVELOP_DIST) and (dist in req):
if best is not None and best.location != dist.location:
raise ValueError('Multiple devel eggs for', req)
best = dist
if best is not None:
return best
specs = [(pkg_resources.parse_version(v), op) for (op, v) in req.specs]
specs.sort()
maxv = None
greater = False
lastv = None
for v, op in specs:
if op == '==' and not greater:
maxv = v
elif op in ('>', '>=', '!='):
maxv = None
greater == True
elif op == '<':
maxv = None
greater == False
elif op == '<=':
maxv = v
greater == False
if v == lastv:
# Repeated versions values are undefined, so
# all bets are off
maxv = None
greater = True
else:
lastv = v
if maxv is not None:
for dist in dists:
if dist.parsed_version == maxv:
return dist
return None
if sys.platform == 'win32':
# work around spawn lamosity on windows
# XXX need safe quoting (see the subproces.list2cmdline) and test
def _safe_arg(arg):
return '"%s"' % arg
else:
_safe_arg = str
_easy_install_cmd = _safe_arg(
'from setuptools.command.easy_install import main; main()'
)
def _call_easy_install(spec, dest, links=(),
index = None,
executable=sys.executable,
always_unzip=False,
):
prefix = sys.exec_prefix + os.path.sep
path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)])
args = ('-c', _easy_install_cmd, '-mUNxd', _safe_arg(dest))
if links:
args += ('-f', _safe_arg(' '.join(links)))
if index:
args += ('-i', index)
if always_unzip:
args += ('-Z', )
level = logger.getEffectiveLevel()
if level > logging.DEBUG:
args += ('-q', )
elif level < logging.DEBUG:
args += ('-v', )
args += (spec, )
if level <= logging.DEBUG:
logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
executable, '" "'.join(args), path)
args += (dict(os.environ, PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first
exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
assert exit_code == 0
# We may overwrite distributions, so clear importer
# cache.
sys.path_importer_cache.clear()
def _get_dist(requirement, env, ws,
dest, links, index, executable, always_unzip):
# Maybe an existing dist is already the best dist that satisfies the
# requirement
dist = _satisfied(requirement, env)
# XXX Special case setuptools because:
# 1. Almost everything depends on it and
# 2. It is expensive to checl for.
# Need to think of a cleaner way to handle this.
# If we already have a satisfactory version, use it.
if dist is None and requirement.project_name == 'setuptools':
dist = env.best_match(requirement, ws)
if dist is None:
if dest is not None:
# May need a new one. Call easy_install
_call_easy_install(str(requirement), dest, links, index,
executable, always_unzip)
# Because we may have added new eggs, we need to rescan
# the destination directory. A possible optimization
# is to get easy_install to recod the files installed
# and either firgure out the distribution added, or
# only rescan if any files have been added.
env.scan([dest])
dist = env.best_match(requirement, ws)
if dist is None:
raise ValueError("Couldn't find", requirement)
# XXX Need test for this
if dist.has_metadata('dependency_links.txt'):
for link in dist.get_metadata_lines('dependency_links.txt'):
link = link.strip()
if link not in links:
links.append(link)
return dist
def install(specs, dest,
links=(), index=None,
executable=sys.executable, always_unzip=False,
path=None):
logger.debug('Installing %r', specs)
path = path and path[:] or []
if dest is not None:
path.insert(0, dest)
path += buildout_and_setuptools_path
links = list(links) # make copy, because we may need to mutate
# For each spec, see if it is already installed. We create a working
# set to keep track of what we've collected and to make sue than the
# distributions assembled are consistent.
env = pkg_resources.Environment(path, python=_get_version(executable))
requirements = [pkg_resources.Requirement.parse(spec) for spec in specs]
ws = pkg_resources.WorkingSet([])
for requirement in requirements:
ws.add(_get_dist(requirement, env, ws,
dest, links, index, executable, always_unzip)
)
# OK, we have the requested distributions and they're in the working
# set, but they may have unmet requirements. We'll simply keep
# trying to resolve requirements, adding missing requirements as they
# are reported.
#
# Note that we don't pass in the environment, because we
# want to look for new eggs unless what we have is the best that matches
# the requirement.
while 1:
try:
ws.resolve(requirements)
except pkg_resources.DistributionNotFound, err:
[requirement] = err
if dest:
logger.debug('Getting required %s', requirement)
ws.add(_get_dist(requirement, env, ws,
dest, links, index, executable, always_unzip)
)
else:
break
return ws
def _editable(spec, dest, links=(), index = None, executable=sys.executable):
prefix = sys.exec_prefix + os.path.sep
path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)])
args = ('-c', _easy_install_cmd, '-eb', _safe_arg(dest))
if links:
args += ('-f', ' '.join(links))
if index:
args += ('-i', index)
level = logger.getEffectiveLevel()
if level > logging.DEBUG:
args += ('-q', )
elif level < logging.DEBUG:
args += ('-v', )
args += (spec, )
if level <= logging.DEBUG:
logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
executable, '" "'.join(args), path)
args += (dict(os.environ, PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first
exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
assert exit_code == 0
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
path=None):
# XXX we're going to download and build the egg every stinking time.
# We need to not do that.
logger.debug('Building %r', spec)
path = path and path[:] or []
if dest is not None:
path.insert(0, dest)
path += buildout_and_setuptools_path
links = list(links) # make copy, because we may need to mutate
# For each spec, see if it is already installed. We create a working
# set to keep track of what we've collected and to make sue than the
# distributions assembled are consistent.
env = pkg_resources.Environment(path, python=_get_version(executable))
requirement = pkg_resources.Requirement.parse(spec)
dist = _satisfied(requirement, env)
if dist is not None:
return dist
# Get an editable version of the package to a temporary directory:
tmp = tempfile.mkdtemp('editable')
_editable(spec, tmp, links, index, executable)
setup_cfg = os.path.join(tmp, requirement.key, 'setup.cfg')
if not os.path.exists(setup_cfg):
f = open(setup_cfg, 'w')
f.close()
setuptools.command.setopt.edit_config(setup_cfg, dict(build_ext=build_ext))
# Now run easy_install for real:
_call_easy_install(
os.path.join(tmp, requirement.key),
dest, links, index, executable, True)
def working_set(specs, executable, path):
return install(specs, None, executable=executable, path=path)
def scripts(reqs, working_set, executable, dest,
scripts=None,
extra_paths=(),
arguments='',
):
reqs = [pkg_resources.Requirement.parse(r) for r in reqs]
projects = [r.project_name for r in reqs]
path = [dist.location for dist in working_set]
path.extend(extra_paths)
path = repr(path)[1:-1].replace(',', ',\n ')
generated = []
for dist in working_set:
if dist.project_name in projects:
for name in pkg_resources.get_entry_map(dist, 'console_scripts'):
if scripts is not None:
sname = scripts.get(name)
if sname is None:
continue
else:
sname = name
sname = os.path.join(dest, sname)
generated.extend(
_script(dist, 'console_scripts', name, path, sname,
executable, arguments)
)
name = 'py-'+dist.project_name
if scripts is not None:
sname = scripts.get(name)
else:
sname = name
if sname is not None:
sname = os.path.join(dest, sname)
generated.extend(
_pyscript(path, sname, executable)
)
return generated
def _script(dist, group, name, path, dest, executable, arguments):
entry_point = dist.get_entry_info(group, name)
generated = []
if sys.platform == 'win32':
# generate exe file and give the script a magic name:
open(dest+'.exe', 'wb').write(
pkg_resources.resource_string('setuptools', 'cli.exe')
)
generated.append(dest+'.exe')
dest += '-script.py'
open(dest, 'w').write(script_template % dict(
python = executable,
path = path,
project = dist.project_name,
name = name,
module_name = entry_point.module_name,
attrs = '.'.join(entry_point.attrs),
arguments = arguments,
))
try:
os.chmod(dest, 0755)
except (AttributeError, os.error):
pass
generated.append(dest)
return generated
script_template = '''\
#!%(python)s
import sys
sys.path[0:0] = [
%(path)s
]
import %(module_name)s
if __name__ == '__main__':
%(module_name)s.%(attrs)s(%(arguments)s)
'''
def _pyscript(path, dest, executable):
generated = []
if sys.platform == 'win32':
# generate exe file and give the script a magic name:
open(dest+'.exe', 'wb').write(
pkg_resources.resource_string('setuptools', 'cli.exe')
)
generated.append(dest+'.exe')
dest += '-script.py'
open(dest, 'w').write(py_script_template % dict(
python = executable,
path = path,
))
try:
os.chmod(dest,0755)
except (AttributeError, os.error):
pass
generated.append(dest)
return generated
py_script_template = '''\
#!%(python)s
import sys
sys.path[0:0] = [
%(path)s
]
_interactive = True
if len(sys.argv) > 1:
import getopt
_options, _args = getopt.getopt(sys.argv[1:], 'ic:')
_interactive = False
for (_opt, _val) in _options:
if _opt == '-i':
_interactive = True
elif _opt == '-c':
exec _val
if _args:
sys.argv[:] = _args
execfile(sys.argv[0])
if _interactive:
import code
code.interact(banner="", local=globals())
'''
zc.setuptools/zc/buildout/easy_install.txt 0 → 100644
View file @ e4f4f913
Minimal Python interface to easy_install
========================================
The easy_install module provides a minimal interface to the setuptools
easy_install command that provides some additional semantics:
- By default, we look for new packages *and* the packages that
they depend on. This is somewhat like (and uses) the --upgrade
option of easy_install, except that we also upgrade required
packages.
- If the highest-revision package satisfying a specification is
already present, then we don't try to get another one. This saves a
lot of search time in the common case that packages are pegged to
specific versions.
- If there is a develop egg that satisfies a requirement, we don't
look for additional distributions. We always give preference to
develop eggs.
- Distutils options for building extensions can be passed.
The easy_install module provides a method, install, for installing one
or more packages and their dependencies. The
install function takes 2 positional arguments:
- An iterable of setuptools requirement strings for the distributions
to be installed, and
- A destination directory to install to and to satisfy
requirements from.
It supports a number of optional keyword arguments:
links
a sequence of URLs, file names, or directories to look for
links to distributions,
index
The URL of an index server, or almost any other valid URL. :)
If not specified, the Python Package Index,
http://cheeseshop.python.org/pypi, is used. You can specify an
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
executable
A path to a Python executable. Distributions will ne installed
using this executable and will be for the matching Python version.
path
A list of additional directories to search for locally-installed
distributions.
always_unzip
A flag indicating that newly-downloaded distributions should be
directories even if they could be installed as zip files.
arguments
Source code to be used to pass arguments when calling a script entry point.
The install method returns a working set containing the distributions
needed to meet the given requirements.
We have a link server that has a number of eggs:
>>> print get(link_server),
<html><body>
<a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br>
<a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
<a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br>
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br>
<a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br>
<a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br>
<a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br>
<a href="extdemo-1.4.tar.gz">extdemo-1.4.tar.gz</a><br>
<a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
</body></html>
Let's make a directory and install the demo egg to it, using the demo:
>>> dest = mkdtemp('sample-install')
>>> import zc.buildout.easy_install
>>> ws = zc.buildout.easy_install.install(
... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/')
We requested version 0.2 of the demo distribution to be installed into
the destination server. We specified that we should search for links
on the link server and that we should use the (empty) link server
index directory as a package index.
The working set contains the distributions we retrieved.
>>> for dist in ws:
... print dist
demo 0.2
demoneeded 1.1
And the actual eggs were added to the eggs directory.
>>> ls(dest)
- demo-0.2-py2.3.egg
- demoneeded-1.1-py2.3.egg
If we ask for the demo distribution without a version restriction,
we'll get the newer version:
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest, links=[link_server], index=link_server+'index/')
>>> ls(dest)
- demo-0.2-py2.3.egg
- demo-0.3-py2.3.egg
- demoneeded-1.1-py2.3.egg
We can supply additional distributions. We can also supply
specifications for distributions that would normally be found via
dependencies. We might do this to specify a sprcific version.
>>> ws = zc.buildout.easy_install.install(
... ['demo', 'other', 'demoneeded==1.0'], dest,
... links=[link_server], index=link_server+'index/')
>>> for dist in ws:
... print dist
demo 0.3
other 1.0
demoneeded 1.0
>>> ls(dest)
- demo-0.2-py2.3.egg
- demo-0.3-py2.3.egg
- demoneeded-1.0-py2.3.egg
- demoneeded-1.1-py2.3.egg
- other-1.0-py2.3.egg
We can specify an alternate Python executable, and we can specify
that, when we retrieve (or create) an egg, it should be unzipped.
>>> dest = mkdtemp('sample-install')
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest, links=[link_server], index=link_server+'index/',
... always_unzip=True, executable= python2_3_executable)
>>> ls(dest)
d demo-0.3-py2.3.egg
d demoneeded-1.1-py2.3.egg
>>> dest = mkdtemp('sample-install')
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest, links=[link_server], index=link_server+'index/',
... always_unzip=True, executable=python2_4_executable)
>>> ls(dest)
d demo-0.3-py2.4.egg
d demoneeded-1.1-py2.4.egg
Script generation
-----------------
The easy_install module provides support for creating scripts from
eggs. It provides a function similar to setuptools except that it
provides facilities for baking a script's path into the script. This
has two advantages:
- The eggs to be used by a script are not chosen at run time, making
startup faster and, more importantly, deterministic.
- The script doesn't have to import pkg_resources because the logic
that pkg_resources would execute at run time is executed at
script-creation time.
The scripts method can be used to generate scripts. Let's create a
destination directory for it to place them in:
>>> import tempfile
>>> bin = mkdtemp()
Now, we'll use the scripts method to generate scripts in this directory
from the demo egg:
>>> scripts = zc.buildout.easy_install.scripts(
... ['demo==0.1'], ws, python2_4_executable, bin)
the four arguments we passed were:
1. A sequence of distribution requirements. These are of the same
form as setuptools requirements. Here we passed a single
requirement, for the version 0.1 demo distribution.
2. A working set,
3. The Python executable to use, and
3. The destination directory.
The bin directory now contains 2 generated scripts:
>>> ls(bin)
- demo
- py-demo
The return value is a list of the scripts generated:
>>> import os, sys
>>> if sys.platform == 'win32':
... scripts == [os.path.join(bin, 'demo.exe'),
... os.path.join(bin, 'demo-script.py'),
... os.path.join(bin, 'py-demo.exe'),
... os.path.join(bin, 'py-demo-script.py')]
... else:
... scripts == [os.path.join(bin, 'demo'),
... os.path.join(bin, 'py-demo')]
True
The demo script run the entry point defined in the demo egg:
>>> cat(bin, 'demo') # doctest: +NORMALIZE_WHITESPACE
#!/usr/local/bin/python2.3
<BLANKLINE>
import sys
sys.path[0:0] = [
'/tmp/xyzsample-install/demo-0.3-py2.3.egg',
'/tmp/xyzsample-install/demoneeded-1.1-py2.3.egg'
]
<BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
eggrecipedemo.main()
Some things to note:
- The demo and demoneeded eggs are added to the beginning of sys.path.
- The module for the script entry point is imported and the entry
point, in this case, 'main', is run.
The py-demo script simply run the Python interactive interpreter with
the path set:
>>> cat(bin, 'py-demo') # doctest: +NORMALIZE_WHITESPACE
#!/usr/local/bin/python2.4
import sys
<BLANKLINE>
sys.path[0:0] = [
'/tmp/tmp5zS2Afsample-install/demo-0.3-py2.4.egg',
'/tmp/tmp5zS2Afsample-install/demoneeded-1.1-py2.4.egg'
]
<BLANKLINE>
_interactive = True
if len(sys.argv) > 1:
import getopt
_options, _args = getopt.getopt(sys.argv[1:], 'ic:')
_interactive = False
for (_opt, _val) in _options:
if _opt == '-i':
_interactive = True
elif _opt == '-c':
exec _val
<BLANKLINE>
if _args:
sys.argv[:] = _args
execfile(sys.argv[0])
<BLANKLINE>
if _interactive:
import code
code.interact(banner="", local=globals())
If invoked with a script name and arguments, it will run that script, instead.
An additional argumnet can be passed to define which scripts to install
and to provie script names. The argument is a dictionary mapping
original script names to new script names.
>>> bin = mkdtemp()
>>> scripts = zc.buildout.easy_install.scripts(
... ['demo==0.1'], ws, python2_4_executable, bin, dict(demo='run'))
>>> if sys.platform == 'win32':
... scripts == [os.path.join(bin, 'run.exe'),
... os.path.join(bin, 'run-script.py')]
... else:
... scripts == [os.path.join(bin, 'run')]
True
>>> ls(bin)
- run
>>> print system(os.path.join(bin, 'run')),
3 1
Including extra paths in scripts
--------------------------------
We can pass a keyword argument, extra paths, to caue additional paths
to be included in the a generated script:
>>> scripts = zc.buildout.easy_install.scripts(
... ['demo==0.1'], ws, python2_4_executable, bin, dict(demo='run'),
... extra_paths=['/foo/bar'])
>>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE
#!/usr/local/bin/python2.3
<BLANKLINE>
import sys
sys.path[0:0] = [
'/tmp/xyzsample-install/demo-0.3-py2.3.egg',
'/tmp/xyzsample-install/demoneeded-1.1-py2.3.egg',
'/foo/bar'
]
<BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
eggrecipedemo.main()
Providing script arguments
--------------------------
A n "argument" keyword argument can be used to pass arguments to an
entry point. The value passed source string to be placed between the
parentheses in the call:
>>> scripts = zc.buildout.easy_install.scripts(
... ['demo==0.1'], ws, python2_4_executable, bin, dict(demo='run'),
... arguments='1, 2')
>>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE
#!/usr/local/bin/python2.3
import sys
sys.path[0:0] = [
'/tmp/xyzsample-install/demo-0.3-py2.3.egg',
'/tmp/xyzsample-install/demoneeded-1.1-py2.3.egg'
]
<BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
eggrecipedemo.main(1, 2)
Handling custom build options for extensions
--------------------------------------------
Sometimes, we need to control how extension modules are built. The
build method provides this level of control. It takes a single
package specification, downloads a source distribution, and builds it
with specified custom build options.
The build method takes 3 positional arguments:
spec
A package specification
dest
A destination directory
build_ext
A dictionary of options to be passed to the distutils build_ext
command when building extensions.
It supports a number of optional keyword arguments:
links
a sequence of URLs, file names, or directories to look for
links to distributions,
index
The URL of an index server, or almost any other valid URL. :)
If not specified, the Python Package Index,
http://cheeseshop.python.org/pypi, is used. You can specify an
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
executable
A path to a Python executable. Distributions will ne installed
using this executable and will be for the matching Python version.
path
A list of additional directories to search for locally-installed
distributions.
always_unzip
A flag indicating that newly-downloaded distributions should be
directories even if they could be installed as zip files.
Our link server included a source distribution that includes a simple
extension, extdemo.c::
#include <Python.h>
#include <extdemo.h>
static PyMethodDef methods[] = {};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
}
The extension depends on a system-dependnt include file, extdemo.h,
that defines a constant, EXTDEMO, that is exposed by the extension.
We'll add an include directory to our sample buildout and add the
needed include file to it:
>>> mkdir(sample_buildout, 'include')
>>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
... "#define EXTDEMO 42\n")
Now, we can use the build function to create an egg from the source
distribution:
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/')
Now if we look in our destination directory, we see we have an extdemo egg:
>>> ls(dest)
d demo-0.3-py2.4.egg
d demoneeded-1.1-py2.4.egg
d extdemo-1.4-py2.3-unix-i686.egg
zc.setuptools/zc/buildout/testing.py 0 → 100644
View file @ e4f4f913
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Various test-support utility functions
$Id$
"""
import BaseHTTPServer, ConfigParser, os, random, re, shutil, socket, sys
import tempfile, threading, time, urllib2, unittest
from zope.testing import doctest, renormalizing
import pkg_resources
import zc.buildout.buildout
def cat(dir, *names):
path = os.path.join(dir, *names)
if (not os.path.exists(path)
and sys.platform == 'win32'
and os.path.exists(path+'-script.py')
):
path = path+'-script.py'
print open(path).read(),
def ls(dir, *subs):
if subs:
dir = os.path.join(dir, *subs)
names = os.listdir(dir)
names.sort()
for name in names:
if os.path.isdir(os.path.join(dir, name)):
print 'd ',
else:
print '- ',
print name
def mkdir(dir, *subs):
if subs:
dir = os.path.join(dir, *subs)
os.mkdir(dir)
def write(dir, *args):
open(os.path.join(dir, *(args[:-1])), 'w').write(args[-1])
def system(command, input=''):
i, o = os.popen4(command)
if input:
i.write(input)
i.close()
return o.read()
def get(url):
return urllib2.urlopen(url).read()
def buildoutSetUp(test):
# we both need to make sure that HOME isn't set and be prepared
# to restore whatever it was after the test.
test.globs['_oldhome'] = os.environ.pop('HOME', None)
temporary_directories = []
def mkdtemp(*args):
d = tempfile.mkdtemp(*args)
temporary_directories.append(d)
return d
sample = mkdtemp('sample-buildout')
# Create a basic buildout.cfg to avoid a warning from buildout:
open(os.path.join(sample, 'buildout.cfg'), 'w').write(
"[buildout]\nparts =\n"
)
# Use the buildout bootstrap command to create a buildout
zc.buildout.buildout.Buildout(os.path.join(sample, 'buildout.cfg'), ()
).bootstrap([])
test.globs.update(dict(
__here = os.getcwd(),
sample_buildout = sample,
ls = ls,
cat = cat,
mkdir = mkdir,
write = write,
system = system,
get = get,
__temporary_directories__ = temporary_directories,
__tearDown__ = [],
mkdtemp = mkdtemp,
))
def buildoutTearDown(test):
os.chdir(test.globs['__here'])
for d in test.globs['__temporary_directories__']:
shutil.rmtree(d)
for f in test.globs['__tearDown__']:
f()
if test.globs.get('_oldhome') is not None:
os.environ['HOME'] = test.globs['_oldhome']
script_template = '''\
#!%(python)s
import sys
sys.path[0:0] = %(path)r
from pkg_resources import load_entry_point
sys.exit(load_entry_point('zc.buildout', 'console_scripts', 'buildout')())
'''
def runsetup(d, executable):
here = os.getcwd()
try:
os.chdir(d)
os.spawnle(
os.P_WAIT, executable, executable,
'setup.py', '-q', 'bdist_egg',
{'PYTHONPATH': os.path.dirname(pkg_resources.__file__)},
)
shutil.rmtree('build')
finally:
os.chdir(here)
def create_sample_eggs(test, executable=sys.executable):
if 'sample_eggs' in test.globs:
sample = os.path.dirname(test.globs['sample_eggs'])
else:
sample = test.globs['mkdtemp']('sample-eggs')
test.globs['sample_eggs'] = os.path.join(sample, 'dist')
write(sample, 'README.txt', '')
for i in (0, 1):
write(sample, 'eggrecipedemobeeded.py', 'y=%s\n' % i)
write(
sample, 'setup.py',
"from setuptools import setup\n"
"setup(name='demoneeded', py_modules=['eggrecipedemobeeded'],"
" zip_safe=True, version='1.%s')\n"
% i
)
runsetup(sample, executable)
write(
sample, 'setup.py',
"from setuptools import setup\n"
"setup(name='other', zip_safe=True, version='1.0', "
"py_modules=['eggrecipedemobeeded'])\n"
)
runsetup(sample, executable)
os.remove(os.path.join(sample, 'eggrecipedemobeeded.py'))
for i in (1, 2, 3):
write(
sample, 'eggrecipedemo.py',
'import eggrecipedemobeeded\n'
'x=%s\n'
'def main(): print x, eggrecipedemobeeded.y\n'
% i)
write(
sample, 'setup.py',
"from setuptools import setup\n"
"setup(name='demo', py_modules=['eggrecipedemo'],"
" install_requires = 'demoneeded',"
" entry_points={'console_scripts': ['demo = eggrecipedemo:main']},"
" zip_safe=True, version='0.%s')\n" % i
)
runsetup(sample, executable)
def find_python(version):
e = os.environ.get('PYTHON%s' % version)
if e is not None:
return e
if sys.platform == 'win32':
e = '\Python%s%s\python.exe' % tuple(version.split('.'))
if os.path.exists(e):
return e
else:
i, o = os.popen4('python%s -c "import sys; print sys.executable"'
% version)
i.close()
e = o.read().strip()
o.close()
if os.path.exists(e):
return e
i, o = os.popen4(
'python -c "import sys; print \'%s.%s\' % sys.version_info[:2]"'
)
i.close()
e = o.read().strip()
o.close()
if e == version:
i, o = os.popen4('python -c "import sys; print sys.executable"')
i.close()
e = o.read().strip()
o.close()
if os.path.exists(e):
return e
raise ValueError(
"Couldn't figure out the exectable for Python %(version)s.\n"
"Set the environment variable PYTHON%(version)s to the location\n"
"of the Python %(version)s executable before running the tests."
)
def multi_python(test):
p23 = find_python('2.3')
p24 = find_python('2.4')
create_sample_eggs(test, executable=p23)
create_sample_eggs(test, executable=p24)
test.globs['python2_3_executable'] = p23
test.globs['python2_4_executable'] = p24
extdemo_c = """
#include <Python.h>
#include <extdemo.h>
static PyMethodDef methods[] = {{NULL}};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
}
"""
extdemo_setup_py = """
from distutils.core import setup, Extension
setup(name = "extdemo", version = "1.4", url="http://www.zope.org",
author="Demo", author_email="demo@demo.com",
ext_modules = [Extension('extdemo', ['extdemo.c'])],
)
"""
def add_source_dist(test):
import tarfile
tmp = tempfile.mkdtemp('test-sdist')
open(os.path.join(tmp, 'extdemo.c'), 'w').write(extdemo_c);
open(os.path.join(tmp, 'setup.py'), 'w').write(extdemo_setup_py);
open(os.path.join(tmp, 'README'), 'w').write("");
open(os.path.join(tmp, 'MANIFEST.in'), 'w').write("include *.c\n");
here = os.getcwd()
os.chdir(tmp)
status = os.spawnl(os.P_WAIT, sys.executable, sys.executable,
os.path.join(tmp, 'setup.py'), '-q', 'sdist')
os.chdir(here)
assert status == 0
if sys.platform == 'win32':
sname = 'extdemo-1.4.zip'
else:
sname = 'extdemo-1.4.tar.gz'
shutil.move(
os.path.join(tmp, 'dist', sname),
os.path.join(test.globs['sample_eggs'], sname),
)
def make_tree(test):
sample_eggs = test.globs['sample_eggs']
tree = dict(
[(n, open(os.path.join(sample_eggs, n), 'rb').read())
for n in os.listdir(sample_eggs)
])
tree['index'] = {}
return tree
class Server(BaseHTTPServer.HTTPServer):
def __init__(self, tree, *args):
BaseHTTPServer.HTTPServer.__init__(self, *args)
self.tree = tree
__run = True
def serve_forever(self):
while self.__run:
self.handle_request()
def handle_error(self, *_):
self.__run = False
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def __init__(self, request, address, server):
self.tree = server.tree
BaseHTTPServer.BaseHTTPRequestHandler.__init__(
self, request, address, server)
def do_GET(self):
if '__stop__' in self.path:
raise SystemExit
tree = self.tree
for name in self.path.split('/'):
if not name:
continue
tree = tree.get(name)
if tree is None:
self.send_response(404, 'Not Found')
out = '<html><body>Not Found</body></html>'
self.send_header('Content-Length', str(len(out)))
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write(out)
return
self.send_response(200)
if isinstance(tree, dict):
out = ['<html><body>\n']
items = tree.items()
items.sort()
for name, v in items:
if isinstance(v, dict):
name += '/'
out.append('<a href="%s">%s</a><br>\n' % (name, name))
out.append('</body></html>\n')
out = ''.join(out)
self.send_header('Content-Length', str(len(out)))
self.send_header('Content-Type', 'text/html')
else:
out = tree
self.send_header('Content-Length', len(out))
if name.endswith('.egg'):
self.send_header('Content-Type', 'application/zip')
elif name.endswith('.gz'):
self.send_header('Content-Type', 'application/x-gzip')
elif name.endswith('.zip'):
self.send_header('Content-Type', 'application/x-gzip')
else:
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write(out)
def log_request(*s):
pass
def _run(tree, port):
server_address = ('localhost', port)
httpd = Server(tree, server_address, Handler)
httpd.serve_forever()
def get_port():
for i in range(10):
port = random.randrange(20000, 30000)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
try:
s.connect(('localhost', port))
except socket.error:
return port
finally:
s.close()
raise RuntimeError, "Can't find port"
def _start_server(tree, name=''):
port = get_port()
thread = threading.Thread(target=_run, args=(tree, port), name=name)
thread.setDaemon(True)
thread.start()
wait(port, up=True)
return port, thread
def start_server(tree):
return _start_server(tree)[0]
def stop_server(url, thread=None):
try:
urllib2.urlopen(url+'__stop__')
except Exception:
pass
if thread is not None:
thread.join() # wait for thread to stop
def setUpServer(test, tree):
port, thread = _start_server(tree, name=test.name)
link_server = 'http://localhost:%s/' % port
test.globs['link_server'] = link_server
test.globs['__tearDown__'].append(lambda: stop_server(link_server, thread))
def wait(port, up):
addr = 'localhost', port
for i in range(120):
time.sleep(0.25)
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(addr)
s.close()
if up:
break
except socket.error, e:
if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
raise
s.close()
if not up:
break
else:
if up:
raise
else:
raise SystemError("Couln't stop server")
zc.setuptools/zc/buildout/tests.py 0 → 100644
View file @ e4f4f913
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""XXX short summary goes here.
$Id$
"""
import os, re, shutil, sys, unittest
from zope.testing import doctest, renormalizing
import zc.buildout.testing
os_path_sep = os.path.sep
if os_path_sep == '\\':
os_path_sep *= 2
def buildout_error_handling():
r"""Buildout error handling
Asking for a section that doesn't exist, yields a key error:
>>> import os
>>> os.chdir(sample_buildout)
>>> import zc.buildout.buildout
>>> buildout = zc.buildout.buildout.Buildout('buildout.cfg', [])
>>> buildout['eek']
Traceback (most recent call last):
...
KeyError: 'eek'
Asking for an option that doesn't exist, a MissingOption error is raised:
>>> buildout['buildout']['eek']
Traceback (most recent call last):
...
MissingOption: Missing option: buildout:eek
It is an error to create a variable-reference cycle:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = data_dir debug
... x = ${buildout:y}
... y = ${buildout:z}
... z = ${buildout:x}
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Error: Circular reference in substitutions.
We're evaluating buildout:y, buildout:z, buildout:x
and are referencing: buildout:y.
It is an error to use funny characters in variable refereces:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = data_dir debug
... x = ${bui$ldout:y}
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
Error: The section name in substitution, ${bui$ldout:y},
has invalid characters.
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = data_dir debug
... x = ${buildout:y{z}
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
Error: The option name in substitution, ${buildout:y{z},
has invalid characters.
and too have too many or too few colons:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = data_dir debug
... x = ${parts}
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
Error: The substitution, ${parts},
doesn't contain a colon.
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = data_dir debug
... x = ${buildout:y:z}
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
Error: The substitution, ${buildout:y:z},
has too many colons.
Al parts have to have a section:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... parts = x
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
Error: No section was specified for part x
and all parts have to have a specified recipe:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... parts = x
...
... [x]
... foo = 1
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
Error: Missing option: x:recipe
"""
def test_comparing_saved_options_with_funny_characters():
"""
If an option has newlines, extra/odd spaces or a %, we need to make
sure the comparison with the saved value works correctly.
>>> mkdir(sample_buildout, 'recipes')
>>> write(sample_buildout, 'recipes', 'debug.py',
... '''
... class Debug:
... def __init__(self, buildout, name, options):
... options['debug'] = \"\"\" <zodb>
...
... <filestorage>
... path foo
... </filestorage>
...
... </zodb>
... \"\"\"
... options['debug1'] = \"\"\"
... <zodb>
...
... <filestorage>
... path foo
... </filestorage>
...
... </zodb>
... \"\"\"
... options['debug2'] = ' x '
... options['debug3'] = '42'
... options['format'] = '%3d'
...
... def install(self):
... open('t', 'w').write('t')
... return 't'
... ''')
>>> write(sample_buildout, 'recipes', 'setup.py',
... '''
... from setuptools import setup
... setup(
... name = "recipes",
... entry_points = {'zc.buildout': ['default = debug:Debug']},
... )
... ''')
>>> write(sample_buildout, 'recipes', 'README.txt', " ")
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = debug
...
... [debug]
... recipe = recipes
... ''')
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print system(buildout+' -v'), # doctest: +ELLIPSIS
buildout: Running ...setup.py -q develop ...
buildout: Installing debug
If we run the buildout again, we shoudn't get a message about
uninstalling anything because the configuration hasn't changed.
>>> print system(buildout+' -v'),
buildout: Running setup.py -q develop ...
buildout: Installing debug
"""
def linkerSetUp(test):
zc.buildout.testing.buildoutSetUp(test, clear_home=False)
zc.buildout.testing.multi_python(test)
zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
def easy_install_SetUp(test):
zc.buildout.testing.buildoutSetUp(test)
zc.buildout.testing.multi_python(test)
zc.buildout.testing.add_source_dist(test)
zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
class PythonNormalizing(renormalizing.RENormalizing):
def _transform(self, want, got):
if '/xyzsample-install/' in want:
got = got.replace('-py2.4.egg', '-py2.3.egg')
firstg = got.split('\n')[0]
firstw = want.split('\n')[0]
if firstg.startswith('#!') and firstw.startswith('#!'):
firstg = ' '.join(firstg.split()[1:])
got = firstg + '\n' + '\n'.join(got.split('\n')[1:])
firstw = ' '.join(firstw.split()[1:])
want = firstw + '\n' + '\n'.join(want.split('\n')[1:])
for pattern, repl in self.patterns:
want = pattern.sub(repl, want)
got = pattern.sub(repl, got)
return want, got
def check_output(self, want, got, optionflags):
if got == want:
return True
want, got = self._transform(want, got)
if got == want:
return True
return doctest.OutputChecker.check_output(self, want, got, optionflags)
def output_difference(self, example, got, optionflags):
want = example.want
# If want is empty, use original outputter. This is useful
# when setting up tests for the first time. In that case, we
# generally use the differencer to display output, which we evaluate
# by hand.
if not want.strip():
return doctest.OutputChecker.output_difference(
self, example, got, optionflags)
# Dang, this isn't as easy to override as we might wish
original = want
want, got = self._transform(want, got)
# temporarily hack example with normalized want:
example.want = want
result = doctest.OutputChecker.output_difference(
self, example, got, optionflags)
example.want = original
return result
def test_suite():
return unittest.TestSuite((
doctest.DocFileSuite(
'buildout.txt',
setUp=zc.buildout.testing.buildoutSetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=renormalizing.RENormalizing([
(re.compile('__buildout_signature__ = recipes-\S+'),
'__buildout_signature__ = recipes-SSSSSSSSSSS'),
(re.compile('\S+sample-(\w+)%s(\S+)' % os_path_sep),
r'/sample-\1/\2'),
(re.compile('\S+sample-(\w+)'), r'/sample-\1'),
(re.compile('executable = \S+python\S*'),
'executable = python'),
(re.compile('setuptools-\S+[.]egg'), 'setuptools.egg'),
(re.compile('zc.buildout(-\S+)?[.]egg(-link)?'),
'zc.buildout.egg'),
(re.compile('creating \S*setup.cfg'), 'creating setup.cfg'),
(re.compile('(\n?)- ([a-zA-Z_.-]+)-script.py\n- \\2.exe\n'),
'\\1- \\2\n'),
(re.compile("(\w)%s(\w)" % os_path_sep), r"\1/\2"),
])
),
doctest.DocFileSuite(
'easy_install.txt',
setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=PythonNormalizing([
(re.compile("'"
"(\w:)?"
"[%(sep)s/]\S+sample-install[%(sep)s/]"
"[%(sep)s/]?(dist"
"[%(sep)s/])?"
% dict(sep=os_path_sep)),
'/sample-eggs/'),
(re.compile("([d-] ((ext)?demo(needed)?|other)"
"-\d[.]\d-py)\d[.]\d(-[^. \t\n]+)?[.]egg"),
'\\1V.V.egg'),
(re.compile('(\n?)- ([a-zA-Z_.-]+)-script.py\n- \\2.exe\n'),
'\\1- \\2\n'),
(re.compile('extdemo-1[.]4[.]tar[.]gz'), 'extdemo-1.4.zip'),
(re.compile('#!\S+python\S+'), '#!python'),
]),
),
doctest.DocTestSuite(
setUp=zc.buildout.testing.buildoutSetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=PythonNormalizing([
(re.compile("buildout: Running \S*setup.py"),
'buildout: Running setup.py'),
]),
)
))
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7