From 816245bdcedaf1ce96c525691972990b0222a057 Mon Sep 17 00:00:00 2001 From: Jim Fulton <jim@zope.com> Date: Mon, 25 Feb 2013 07:20:27 -0500 Subject: [PATCH] added meta-recipe support --- CHANGES.rst | 5 + setup.py | 2 + src/zc/buildout/buildout.py | 64 +++++++-- src/zc/buildout/buildout.txt | 2 +- src/zc/buildout/meta-recipes.txt | 232 +++++++++++++++++++++++++++++++ src/zc/buildout/testing.py | 13 ++ src/zc/buildout/tests.py | 4 +- 7 files changed, 305 insertions(+), 17 deletions(-) create mode 100644 src/zc/buildout/meta-recipes.txt diff --git a/CHANGES.rst b/CHANGES.rst index eff74f45..4de887d9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Change History ************** +2.1.0 (2013-??-??) +================== + +- Added `meta-recipe support`_. + 2.0.1 (2013-02-16) ================== diff --git a/setup.py b/setup.py index 6a09292c..b1872ccb 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,8 @@ long_description=( + '\n' + read('src', 'zc', 'buildout', 'debugging.txt') + '\n' + + read('src', 'zc', 'buildout', 'meta-recipes.txt') + + '\n' + read('src', 'zc', 'buildout', 'testing.txt') + '\n' + read('src', 'zc', 'buildout', 'easy_install.txt') diff --git a/src/zc/buildout/buildout.py b/src/zc/buildout/buildout.py index 0ab2896e..dc987373 100644 --- a/src/zc/buildout/buildout.py +++ b/src/zc/buildout/buildout.py @@ -157,7 +157,7 @@ class Buildout(DictMixin): data = dict(buildout=_buildout_default_options.copy()) self._buildout_dir = os.getcwd() - if not _isurl(config_file): + if config_file and not _isurl(config_file): config_file = os.path.abspath(config_file) base = os.path.dirname(config_file) if not os.path.exists(config_file): @@ -762,11 +762,11 @@ class Buildout(DictMixin): for k, v in _spacey_defaults: value = value.replace(k, v) options[option] = value - result[section] = Options(self, section, options) + result[section] = self.Options(self, section, options) return result, True else: - return ({'buildout': Options(self, 'buildout', {'parts': ''})}, + return ({'buildout': self.Options(self, 'buildout', {'parts': ''})}, False, ) @@ -829,7 +829,8 @@ class Buildout(DictMixin): try: timeout = int(timeout) import socket - self._logger.info('Setting socket time out to %d seconds.', timeout) + self._logger.info( + 'Setting socket time out to %d seconds.', timeout) socket.setdefaulttimeout(timeout) except ValueError: self._logger.warning("Default socket timeout is used !\n" @@ -1062,9 +1063,21 @@ class Buildout(DictMixin): runsetup = setup # backward compat. - def annotate(self, args): + def annotate(self, args=None): _print_annotate(self._annotated) + def print_options(self): + for section in sorted(self._data): + if section == 'buildout' or section == self['buildout']['versions']: + continue + print_('['+section+']') + for k, v in sorted(self._data[section].items()): + if '\n' in v: + v = '\n ' + v.replace('\n', '\n ') + else: + v = ' '+v + print_("%s =%s" % (k, v)) + def __getitem__(self, section): __doing__ = 'Getting section %s.', section try: @@ -1077,13 +1090,34 @@ class Buildout(DictMixin): except KeyError: raise MissingSection(section) - options = Options(self, section, data) + options = self.Options(self, section, data) self._data[section] = options options._initialize() return options - def __setitem__(self, key, value): - raise NotImplementedError('__setitem__') + def __setitem__(self, name, data): + if name in self._raw: + raise KeyError("Section already exists", name) + self._raw[name] = dict((k, str(v)) for (k, v) in data.items()) + self[name] # Add to parts + + def parse(self, data): + try: + from cStringIO import StringIO + except ImportError: + from io import StringIO + import textwrap + + sections = zc.buildout.configparser.parse( + StringIO(textwrap.dedent(data)), '') + for name in sections: + if name in self._raw: + raise KeyError("Section already exists", name) + self._raw[name] = dict((k, str(v)) + for (k, v) in sections[name].items()) + + for name in sections: + self[name] # Add to parts def __delitem__(self, key): raise NotImplementedError('__delitem__') @@ -1158,20 +1192,20 @@ class Options(DictMixin): if '${' in v: self._dosub(k, v) - if self.name == 'buildout': + if name == 'buildout': return # buildout section can never be a part - recipe = self.get('recipe') - if not recipe: - return + if self.get('recipe'): + self.initialize() + self.buildout._parts.append(name) + def initialize(self): reqs, entry = _recipe(self._data) buildout = self.buildout recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout) - __doing__ = 'Initializing part %s.', name + name = self.name self.recipe = recipe_class(buildout, name, self) - buildout._parts.append(name) def _do_extend_raw(self, name, data, doing): if name == 'buildout': @@ -1356,6 +1390,8 @@ class Options(DictMixin): def __repr__(self): return repr(dict(self)) +Buildout.Options = Options + _spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*' '|' '^[ \t\r\f\v]+' diff --git a/src/zc/buildout/buildout.txt b/src/zc/buildout/buildout.txt index 6b48a8ea..73f893c3 100644 --- a/src/zc/buildout/buildout.txt +++ b/src/zc/buildout/buildout.txt @@ -390,7 +390,7 @@ We'll get a user error, not a traceback. While: Installing. Getting section data-dir. - Initializing part data-dir. + Initializing section data-dir. Error: Invalid Path diff --git a/src/zc/buildout/meta-recipes.txt b/src/zc/buildout/meta-recipes.txt new file mode 100644 index 00000000..948dc144 --- /dev/null +++ b/src/zc/buildout/meta-recipes.txt @@ -0,0 +1,232 @@ +Meta-recipe support +=================== + + +Buildout recipes provide reusable Python modules for common +configuration tasks. The most widely used recipes tend to provide +low-level functions, like installing eggs or software distributions, +creating configuration files, and so on. The normal recipe framework +is fairly well suited to building these general components. + +Full-blown applications may require many, often tens, of parts. +Defining the many parts that make up an application can be tedious and +often entails a lot of repetition. Buildout provides a number of +mechanisms to avoid repetition, including merging of configuration +files and macros, but these, while useful to an extent, don't scale +very well. Buildout isn't and shouldn't be a programming language. + +Meta-recipes allow us to bring Python to bear to provide higher-level +abstractions for buildouts. + +A meta-recipe is a regular Python recipe that primarily operates by +creating parts. A meta recipe isn't merely a high level recipe. It's +a recipe that defers most or all of it's work to lower-level recipes by +manipulating the buildout database. + +A `presentation at PyCon 2011 +<http://blip.tv/pycon-us-videos-2009-2010-2011/pycon-2011-deploying-applications-with-zc-buildout-4897770>`_ +described early work with meta recipes. + +A simple meta-recipe example +============================ + +Let's look at a fairly simple meta-recipe example. First, consider a +buildout configuration that builds a database deployment:: + + [buildout] + parts = ctl pack + + [deployment] + recipe = zc.recipe.deployment + name = ample + user = zope + + [ctl] + recipe = zc.recipe.rhrc + deployment = deployment + chkconfig = 345 99 10 + parts = main + + [main] + recipe = zc.zodbrecipes:server + deployment = deployment + address = 8100 + path = /var/databases/ample/main.fs + zeo.conf = + <zeo> + address ${:address} + </zeo> + %import zc.zlibstorage + <zlibstorage> + <filestorage> + path ${:path} + </filestorage> + </zlibstorage> + + [pack] + recipe = zc.recipe.deployment:crontab + deployment = deployment + times = 1 2 * * 6 + command = ${buildout:bin-directory}/zeopack -d3 -t00 ${main:address} + +.. -> low_level + +This buildout doesn't build software. Rather it builds configuration +for deploying a database configuration using already-deployed +software. For the purpose of this document, however, the details are +totally unimportant. + +Rather than crafting the configuration above every time, we can write +a meta-recipe that crafts it for us. We'll use our meta-recipe as +follows:: + + [buildout] + parts = ample + + [ample] + recipe = com.example.ample:db + path = /var/databases/ample/main.fs + +The idea here is that the meta recipe allows us to specify the minimal +information necessary. A meta-recipe often automates policies and +assumptions that are application and organization dependent. The +example above assumes, for example, that we want to pack to 3 +days in the past on Saturdays. + +So now, let's see the meta recipe that automates this:: + + class Recipe: + + def __init__(self, buildout, name, options): + + buildout.parse(''' + [deployment] + recipe = zc.recipe.deployment + name = %s + user = zope + ''' % name) + + buildout['main'] = dict( + recipe = 'zc.zodbrecipes:server', + deployment = 'deployment', + address = 8100, + path = options['path'], + **{ + 'zeo.conf': ''' + <zeo> + address ${:address} + </zeo> + + %import zc.zlibstorage + + <zlibstorage> + <filestorage> + path ${:path} + </filestorage> + </zlibstorage> + '''} + ) + + buildout.parse(''' + [pack] + recipe = zc.recipe.deployment:crontab + deployment = deployment + times = 1 2 * * 6 + command = + ${buildout:bin-directory}/zeopack -d3 -t00 ${main:address} + + [ctl] + recipe = zc.recipe.rhrc + deployment = deployment + chkconfig = 345 99 10 + parts = main + ''') + + def install(self): + pass + + update = install + +.. -> source + + >>> exec(source) + +The meta recipe just adds parts to the buildout. It does this by +setting items and and calling the ``parse`` method. The ``parse`` +method just takes a string in buildout configuration syntax. It's +useful when we want to add static, or nearly static part data. The +setting items syntax is useful when we have non-trivial computation +for part data. + +The order that we add parts is important. When adding a part, any +string substitutions and other dependencies are evaluated, so the +referenced parts must be defined first. This is why, for example, the +``pack`` part is added after the ``main`` part. + +Note that the meta recipe supplied an integer for one of the +options. In addition to strings, it's legal to supply integer values. + +There are a few things to note about this example: + +- The install and update methods are empty. + + While not required, this is a very common pattern for meta + recipes. Most meta recipes, simply invoke other recipes. + +- Setting a buildout item or calling parse, adds any sections with + recipes as parts. + +- An exception will be raised if a section already exists. + +Testing +------- + +Now, let's test our meta recipe. We'll test it without actually +running buildout. Rather, we'll use a specialized buildout provided by +the zc.buildout.testing module. + + >>> import zc.buildout.testing + >>> buildout = zc.buildout.testing.Buildout() + +The testing buildout is intended to be passed to recipes being +tested: + + >>> _ = Recipe(buildout, 'ample', dict(path='/var/databases/ample/main.fs')) + +After running the recipe, we should see the buildout database +populated by the recipe: + + >>> buildout.print_options() + [ctl] + chkconfig = 345 99 10 + deployment = deployment + parts = main + recipe = zc.recipe.rhrc + [deployment] + name = ample + recipe = zc.recipe.deployment + user = zope + [main] + address = 8100 + deployment = deployment + path = /var/databases/ample/main.fs + recipe = zc.zodbrecipes:server + zeo.conf = + <BLANKLINE> + <zeo> + address 8100 + </zeo> + <BLANKLINE> + %import zc.zlibstorage + <BLANKLINE> + <zlibstorage> + <filestorage> + path /var/databases/ample/main.fs + </filestorage> + </zlibstorage> + <BLANKLINE> + [pack] + command = /sample-buildout/bin/zeopack -d3 -t00 8100 + deployment = deployment + recipe = zc.recipe.deployment:crontab + times = 1 2 * * 6 diff --git a/src/zc/buildout/testing.py b/src/zc/buildout/testing.py index 41c1ae8a..a1067757 100644 --- a/src/zc/buildout/testing.py +++ b/src/zc/buildout/testing.py @@ -166,6 +166,19 @@ def wait_until(label, func, *args, **kw): time.sleep(0.01) raise ValueError('Timed out waiting for: '+label) +class TestOptions(zc.buildout.buildout.Options): + + def initialize(self): + pass + +class Buildout(zc.buildout.buildout.Buildout): + + def __init__(self): + zc.buildout.buildout.Buildout.__init__( + self, '', [('buildout', 'directory', os.getcwd())]) + + Options = TestOptions + def buildoutSetUp(test): test.globs['__tear_downs'] = __tear_downs = [] diff --git a/src/zc/buildout/tests.py b/src/zc/buildout/tests.py index 321998d5..a3145998 100644 --- a/src/zc/buildout/tests.py +++ b/src/zc/buildout/tests.py @@ -1381,7 +1381,7 @@ def internal_errors(): While: Installing. Getting section data-dir. - Initializing part data-dir. + Initializing section data-dir. <BLANKLINE> An internal error occured due to a bug in either zc.buildout or in a recipe being used: @@ -3227,7 +3227,7 @@ def test_suite(): 'zc.\1 = >=1.99'), ]) ) + manuel.capture.Manuel(), - 'buildout.txt', + 'buildout.txt', 'meta-recipes.txt', setUp=buildout_txt_setup, tearDown=zc.buildout.testing.buildoutTearDown, ), -- 2.30.9