Commit c39cdf2a authored by Jim Fulton's avatar Jim Fulton

Merge pull request #87 from buildout/jim-meta-recipes

Jim meta recipes
parents 4810b6ff 816245bd
Change History Change History
************** **************
2.1.0 (2013-??-??)
==================
- Added `meta-recipe support`_.
2.0.1 (2013-02-16) 2.0.1 (2013-02-16)
================== ==================
......
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
# #
############################################################################## ##############################################################################
name = "zc.buildout" name = "zc.buildout"
version = "2.0.0b2" version = "2.0.1"
import os import os
from setuptools import setup from setuptools import setup
...@@ -53,6 +53,8 @@ long_description=( ...@@ -53,6 +53,8 @@ long_description=(
+ '\n' + + '\n' +
read('src', 'zc', 'buildout', 'debugging.txt') read('src', 'zc', 'buildout', 'debugging.txt')
+ '\n' + + '\n' +
read('src', 'zc', 'buildout', 'meta-recipes.txt')
+ '\n' +
read('src', 'zc', 'buildout', 'testing.txt') read('src', 'zc', 'buildout', 'testing.txt')
+ '\n' + + '\n' +
read('src', 'zc', 'buildout', 'easy_install.txt') read('src', 'zc', 'buildout', 'easy_install.txt')
......
...@@ -157,7 +157,7 @@ class Buildout(DictMixin): ...@@ -157,7 +157,7 @@ class Buildout(DictMixin):
data = dict(buildout=_buildout_default_options.copy()) data = dict(buildout=_buildout_default_options.copy())
self._buildout_dir = os.getcwd() self._buildout_dir = os.getcwd()
if not _isurl(config_file): if config_file and not _isurl(config_file):
config_file = os.path.abspath(config_file) config_file = os.path.abspath(config_file)
base = os.path.dirname(config_file) base = os.path.dirname(config_file)
if not os.path.exists(config_file): if not os.path.exists(config_file):
...@@ -762,11 +762,11 @@ class Buildout(DictMixin): ...@@ -762,11 +762,11 @@ class Buildout(DictMixin):
for k, v in _spacey_defaults: for k, v in _spacey_defaults:
value = value.replace(k, v) value = value.replace(k, v)
options[option] = value options[option] = value
result[section] = Options(self, section, options) result[section] = self.Options(self, section, options)
return result, True return result, True
else: else:
return ({'buildout': Options(self, 'buildout', {'parts': ''})}, return ({'buildout': self.Options(self, 'buildout', {'parts': ''})},
False, False,
) )
...@@ -829,7 +829,8 @@ class Buildout(DictMixin): ...@@ -829,7 +829,8 @@ class Buildout(DictMixin):
try: try:
timeout = int(timeout) timeout = int(timeout)
import socket import socket
self._logger.info('Setting socket time out to %d seconds.', timeout) self._logger.info(
'Setting socket time out to %d seconds.', timeout)
socket.setdefaulttimeout(timeout) socket.setdefaulttimeout(timeout)
except ValueError: except ValueError:
self._logger.warning("Default socket timeout is used !\n" self._logger.warning("Default socket timeout is used !\n"
...@@ -1063,9 +1064,21 @@ class Buildout(DictMixin): ...@@ -1063,9 +1064,21 @@ class Buildout(DictMixin):
runsetup = setup # backward compat. runsetup = setup # backward compat.
def annotate(self, args): def annotate(self, args=None):
_print_annotate(self._annotated) _print_annotate(self._annotated)
def print_options(self):
for section in sorted(self._data):
if section == 'buildout' or section == self['buildout']['versions']:
continue
print_('['+section+']')
for k, v in sorted(self._data[section].items()):
if '\n' in v:
v = '\n ' + v.replace('\n', '\n ')
else:
v = ' '+v
print_("%s =%s" % (k, v))
def __getitem__(self, section): def __getitem__(self, section):
__doing__ = 'Getting section %s.', section __doing__ = 'Getting section %s.', section
try: try:
...@@ -1078,13 +1091,34 @@ class Buildout(DictMixin): ...@@ -1078,13 +1091,34 @@ class Buildout(DictMixin):
except KeyError: except KeyError:
raise MissingSection(section) raise MissingSection(section)
options = Options(self, section, data) options = self.Options(self, section, data)
self._data[section] = options self._data[section] = options
options._initialize() options._initialize()
return options return options
def __setitem__(self, key, value): def __setitem__(self, name, data):
raise NotImplementedError('__setitem__') if name in self._raw:
raise KeyError("Section already exists", name)
self._raw[name] = dict((k, str(v)) for (k, v) in data.items())
self[name] # Add to parts
def parse(self, data):
try:
from cStringIO import StringIO
except ImportError:
from io import StringIO
import textwrap
sections = zc.buildout.configparser.parse(
StringIO(textwrap.dedent(data)), '')
for name in sections:
if name in self._raw:
raise KeyError("Section already exists", name)
self._raw[name] = dict((k, str(v))
for (k, v) in sections[name].items())
for name in sections:
self[name] # Add to parts
def __delitem__(self, key): def __delitem__(self, key):
raise NotImplementedError('__delitem__') raise NotImplementedError('__delitem__')
...@@ -1159,20 +1193,20 @@ class Options(DictMixin): ...@@ -1159,20 +1193,20 @@ class Options(DictMixin):
if '${' in v: if '${' in v:
self._dosub(k, v) self._dosub(k, v)
if self.name == 'buildout': if name == 'buildout':
return # buildout section can never be a part return # buildout section can never be a part
recipe = self.get('recipe') if self.get('recipe'):
if not recipe: self.initialize()
return self.buildout._parts.append(name)
def initialize(self):
reqs, entry = _recipe(self._data) reqs, entry = _recipe(self._data)
buildout = self.buildout buildout = self.buildout
recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout) recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout)
__doing__ = 'Initializing part %s.', name name = self.name
self.recipe = recipe_class(buildout, name, self) self.recipe = recipe_class(buildout, name, self)
buildout._parts.append(name)
def _do_extend_raw(self, name, data, doing): def _do_extend_raw(self, name, data, doing):
if name == 'buildout': if name == 'buildout':
...@@ -1357,6 +1391,8 @@ class Options(DictMixin): ...@@ -1357,6 +1391,8 @@ class Options(DictMixin):
def __repr__(self): def __repr__(self):
return repr(dict(self)) return repr(dict(self))
Buildout.Options = Options
_spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*' _spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*'
'|' '|'
'^[ \t\r\f\v]+' '^[ \t\r\f\v]+'
......
...@@ -390,7 +390,7 @@ We'll get a user error, not a traceback. ...@@ -390,7 +390,7 @@ We'll get a user error, not a traceback.
While: While:
Installing. Installing.
Getting section data-dir. Getting section data-dir.
Initializing part data-dir. Initializing section data-dir.
Error: Invalid Path Error: Invalid Path
......
Meta-recipe support
===================
Buildout recipes provide reusable Python modules for common
configuration tasks. The most widely used recipes tend to provide
low-level functions, like installing eggs or software distributions,
creating configuration files, and so on. The normal recipe framework
is fairly well suited to building these general components.
Full-blown applications may require many, often tens, of parts.
Defining the many parts that make up an application can be tedious and
often entails a lot of repetition. Buildout provides a number of
mechanisms to avoid repetition, including merging of configuration
files and macros, but these, while useful to an extent, don't scale
very well. Buildout isn't and shouldn't be a programming language.
Meta-recipes allow us to bring Python to bear to provide higher-level
abstractions for buildouts.
A meta-recipe is a regular Python recipe that primarily operates by
creating parts. A meta recipe isn't merely a high level recipe. It's
a recipe that defers most or all of it's work to lower-level recipes by
manipulating the buildout database.
A `presentation at PyCon 2011
<http://blip.tv/pycon-us-videos-2009-2010-2011/pycon-2011-deploying-applications-with-zc-buildout-4897770>`_
described early work with meta recipes.
A simple meta-recipe example
============================
Let's look at a fairly simple meta-recipe example. First, consider a
buildout configuration that builds a database deployment::
[buildout]
parts = ctl pack
[deployment]
recipe = zc.recipe.deployment
name = ample
user = zope
[ctl]
recipe = zc.recipe.rhrc
deployment = deployment
chkconfig = 345 99 10
parts = main
[main]
recipe = zc.zodbrecipes:server
deployment = deployment
address = 8100
path = /var/databases/ample/main.fs
zeo.conf =
<zeo>
address ${:address}
</zeo>
%import zc.zlibstorage
<zlibstorage>
<filestorage>
path ${:path}
</filestorage>
</zlibstorage>
[pack]
recipe = zc.recipe.deployment:crontab
deployment = deployment
times = 1 2 * * 6
command = ${buildout:bin-directory}/zeopack -d3 -t00 ${main:address}
.. -> low_level
This buildout doesn't build software. Rather it builds configuration
for deploying a database configuration using already-deployed
software. For the purpose of this document, however, the details are
totally unimportant.
Rather than crafting the configuration above every time, we can write
a meta-recipe that crafts it for us. We'll use our meta-recipe as
follows::
[buildout]
parts = ample
[ample]
recipe = com.example.ample:db
path = /var/databases/ample/main.fs
The idea here is that the meta recipe allows us to specify the minimal
information necessary. A meta-recipe often automates policies and
assumptions that are application and organization dependent. The
example above assumes, for example, that we want to pack to 3
days in the past on Saturdays.
So now, let's see the meta recipe that automates this::
class Recipe:
def __init__(self, buildout, name, options):
buildout.parse('''
[deployment]
recipe = zc.recipe.deployment
name = %s
user = zope
''' % name)
buildout['main'] = dict(
recipe = 'zc.zodbrecipes:server',
deployment = 'deployment',
address = 8100,
path = options['path'],
**{
'zeo.conf': '''
<zeo>
address ${:address}
</zeo>
%import zc.zlibstorage
<zlibstorage>
<filestorage>
path ${:path}
</filestorage>
</zlibstorage>
'''}
)
buildout.parse('''
[pack]
recipe = zc.recipe.deployment:crontab
deployment = deployment
times = 1 2 * * 6
command =
${buildout:bin-directory}/zeopack -d3 -t00 ${main:address}
[ctl]
recipe = zc.recipe.rhrc
deployment = deployment
chkconfig = 345 99 10
parts = main
''')
def install(self):
pass
update = install
.. -> source
>>> exec(source)
The meta recipe just adds parts to the buildout. It does this by
setting items and and calling the ``parse`` method. The ``parse``
method just takes a string in buildout configuration syntax. It's
useful when we want to add static, or nearly static part data. The
setting items syntax is useful when we have non-trivial computation
for part data.
The order that we add parts is important. When adding a part, any
string substitutions and other dependencies are evaluated, so the
referenced parts must be defined first. This is why, for example, the
``pack`` part is added after the ``main`` part.
Note that the meta recipe supplied an integer for one of the
options. In addition to strings, it's legal to supply integer values.
There are a few things to note about this example:
- The install and update methods are empty.
While not required, this is a very common pattern for meta
recipes. Most meta recipes, simply invoke other recipes.
- Setting a buildout item or calling parse, adds any sections with
recipes as parts.
- An exception will be raised if a section already exists.
Testing
-------
Now, let's test our meta recipe. We'll test it without actually
running buildout. Rather, we'll use a specialized buildout provided by
the zc.buildout.testing module.
>>> import zc.buildout.testing
>>> buildout = zc.buildout.testing.Buildout()
The testing buildout is intended to be passed to recipes being
tested:
>>> _ = Recipe(buildout, 'ample', dict(path='/var/databases/ample/main.fs'))
After running the recipe, we should see the buildout database
populated by the recipe:
>>> buildout.print_options()
[ctl]
chkconfig = 345 99 10
deployment = deployment
parts = main
recipe = zc.recipe.rhrc
[deployment]
name = ample
recipe = zc.recipe.deployment
user = zope
[main]
address = 8100
deployment = deployment
path = /var/databases/ample/main.fs
recipe = zc.zodbrecipes:server
zeo.conf =
<BLANKLINE>
<zeo>
address 8100
</zeo>
<BLANKLINE>
%import zc.zlibstorage
<BLANKLINE>
<zlibstorage>
<filestorage>
path /var/databases/ample/main.fs
</filestorage>
</zlibstorage>
<BLANKLINE>
[pack]
command = /sample-buildout/bin/zeopack -d3 -t00 8100
deployment = deployment
recipe = zc.recipe.deployment:crontab
times = 1 2 * * 6
...@@ -171,6 +171,19 @@ def wait_until(label, func, *args, **kw): ...@@ -171,6 +171,19 @@ def wait_until(label, func, *args, **kw):
time.sleep(0.01) time.sleep(0.01)
raise ValueError('Timed out waiting for: '+label) raise ValueError('Timed out waiting for: '+label)
class TestOptions(zc.buildout.buildout.Options):
def initialize(self):
pass
class Buildout(zc.buildout.buildout.Buildout):
def __init__(self):
zc.buildout.buildout.Buildout.__init__(
self, '', [('buildout', 'directory', os.getcwd())])
Options = TestOptions
def buildoutSetUp(test): def buildoutSetUp(test):
test.globs['__tear_downs'] = __tear_downs = [] test.globs['__tear_downs'] = __tear_downs = []
......
...@@ -1390,7 +1390,7 @@ def internal_errors(): ...@@ -1390,7 +1390,7 @@ def internal_errors():
While: While:
Installing. Installing.
Getting section data-dir. Getting section data-dir.
Initializing part data-dir. Initializing section data-dir.
<BLANKLINE> <BLANKLINE>
An internal error occured due to a bug in either zc.buildout or in a An internal error occured due to a bug in either zc.buildout or in a
recipe being used: recipe being used:
...@@ -3236,7 +3236,7 @@ def test_suite(): ...@@ -3236,7 +3236,7 @@ def test_suite():
'zc.\1 = >=1.99'), 'zc.\1 = >=1.99'),
]) ])
) + manuel.capture.Manuel(), ) + manuel.capture.Manuel(),
'buildout.txt', 'buildout.txt', 'meta-recipes.txt',
setUp=buildout_txt_setup, setUp=buildout_txt_setup,
tearDown=zc.buildout.testing.buildoutTearDown, tearDown=zc.buildout.testing.buildoutTearDown,
), ),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment