Import base recipes.

slapos/slapos.lib.recipe/CHANGES.txt

+Changes
+=======
+
+1.0 (unreleased)
+----------------

slapos/slapos.lib.recipe/README.txt

+Library for creating SlapOS recipes for software instantiation
+==============================================================
+
+Thanks to using slapos.lib.recipe it is easier to create zc.buildout recipes in SlapOS environment.
+
+How to use?
+-----------
+
+In setup.py of recipe add only one install requires to slap.lib.recipe.
+
+In code itself subclass from slap.lib.recipe.BaseSlapRecipe.BaseSlapRecipe.
+
+Use _install hook:
+
+::
+
+  from slap.lib.recipe.BaseSlapRecipe import BaseSlapRecipe
+
+  class Recipe(BaseSlapRecipe):
+    ...
+    def _install(self):
+      # refer below for list of available objects
+      specific code
+      of recipe
+
+Available variables self.:
+
+ * name and options passed by zc.buildout during init
+ * work_directory -- buildout's directory
+ * bin_directory -- places for generated binaries
+ * running_wrapper_location -- filename of wrapper to create
+ * data_root_directory -- directory container for data -- inside this
+   directory it is advised to create named directories for provided servers
+   which needs data
+ * backup_directory -- directory container for backups -- inside this
+   directory it is advised o created named directories for backups, with same
+   structure as in data_root_directory
+ * var_directory -- container for various, unix following things:
+
+   * log_directory -- container for logs
+   * run_directory -- container for pidfiles and sockets
+
+ * etc_directory -- place to put named files and directories of configuration
+   for provided servers
+ * computer_id -- id of computer
+ * computer_partition_id -- if of computer partition
+ * server_url - url of Vifib server
+ * software_release_url -- url of software release being instantiated
+ * slap -- initialised connection to Vifib server
+ * computer_partition -- initialised connection to computer partition
+ * request -- shortcut to computer partition request method
+
+By default all directories are created before calling _install hook.
+
+_install method shall return list of paths which are safe to be removed by
+buildout during part uninstallation.
+
+Important assumptions
+---------------------
+
+Because in SlapOS environment zc.buildout does not know when data are changed,
+recipes shall be always uninstalled/installed. This is done during constructing
+recipe instance which subclasses from BaseSlapRecipe.

slapos/slapos.lib.recipe/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.lib.recipe/setup.py

+from setuptools import setup, find_packages
+
+version = '1.0'
+name = 'slapos.lib.recipe'
+long_description = open("README.txt").read() + "\n" + \
+    open("CHANGES.txt").read()
+
+setup(name=name,
+      version=version,
+      description="Library, helpers and superclass for SlapOS zc.buildout recipes",
+      long_description=long_description,
+      classifiers=[
+          "Framework :: Buildout",
+          "Programming Language :: Python",
+        ],
+      keywords='slap librecipe',
+      license='GPLv3',
+      namespace_packages=['slapos'],
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      include_package_data=True,
+      install_requires=[
+        'netaddr', # to manipulate on IP addresses
+        'setuptools', # namespaces
+        'slapos.slap', # uses internally
+        'zc.buildout', # plays with buildout
+        'zc.recipe.egg', # for scripts generation
+        ],
+      zip_safe=True,
+    )

slapos/slapos.lib.recipe/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

slapos/slapos.lib.recipe/src/slapos/lib/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

slapos/slapos.lib.recipe/src/slapos/lib/recipe/BaseSlapRecipe.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import logging
+from slapos import slap
+import os
+import zc.buildout
+import zc.recipe.egg
+from hashlib import md5
+import stat
+import netaddr
+import time
+
+class BaseSlapRecipe:
+  """Base class for all slap.recipe.*"""
+  def __init__(self, buildout, name, options):
+    """Default initialisation"""
+    self.name = name
+    self.options = options
+    self.logger = logging.getLogger(self.name)
+    self.slap = slap.slap()
+    self.work_directory = os.path.abspath(buildout['buildout'][
+      'directory'])
+    self.bin_directory = os.path.join(buildout['buildout'][
+      'directory'], 'bin')
+    self.data_root_directory = os.path.join(self.work_directory, 'srv')
+    self.backup_directory = os.path.join(self.data_root_directory, 'backup')
+    self.var_directory = os.path.join(self.work_directory, 'var')
+    self.log_directory = os.path.join(self.var_directory, 'log')
+    self.run_directory = os.path.join(self.var_directory, 'run')
+    self.etc_directory = os.path.join(self.work_directory, 'etc')
+    self.tmp_directory = os.path.join(self.work_directory, 'tmp')
+    self.wrapper_directory = os.path.join(self.etc_directory, 'run')
+    self.wrapper_report_directory = os.path.join(self.etc_directory, 'report')
+    self.wrapper_xml_report_directory = os.path.join(self.var_directory,
+        'xml_report')
+    self.destroy_script_location = os.path.join(self, self.work_directory,
+        'sbin', 'destroy')
+
+    # default directory structure information
+    self.default_directory_list = [
+      self.bin_directory, # CP/bin - instance own binaries
+      os.path.join(self, self.work_directory, 'sbin'), # CP/sbin - system
+       # binaries, not exposed, only CP/sbin/destroy
+      self.data_root_directory, # CP/srv - data container
+      self.backup_directory, # CP/srv/backup - backup container
+      self.etc_directory, # CP/etc - configuration container
+      self.wrapper_directory, # CP/etc/run - for wrappers
+      self.wrapper_report_directory, # CP/etc/report - for report wrappers
+      self.var_directory, # CP/var - partition "internal" container for logs,
+                          # and another metadata
+      self.wrapper_xml_report_directory, # CP/var/xml_report - for xml_report wrappers
+      self.log_directory, # CP/var/log - log container
+      self.run_directory, # CP/var/run - working container - pids, sockets
+      self.tmp_directory, # CP/tmp - temporary files
+    ]
+
+    # SLAP related information
+    slap_connection = buildout['slap_connection']
+    self.computer_id=slap_connection['computer_id']
+    self.computer_partition_id=slap_connection['partition_id']
+    self.server_url=slap_connection['server_url']
+    self.software_release_url=slap_connection['software_release_url']
+    self.key_file=slap_connection.get('key_file')
+    self.cert_file=slap_connection.get('cert_file')
+
+    # setup egg to give possibility to generate scripts
+    self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
+
+    # setup auto uninstall/install
+    self._setupAutoInstallUninstall()
+
+  def _setupAutoInstallUninstall(self):
+    """By default SlapOS recipes are reinstalled each time"""
+    # Note: It is possible to create in future subclass which will do no-op in
+    # this method
+    self.options['slapos_timestamp'] = str(time.time())
+
+  def _getIpAddress(self, test_method):
+    """Internal helper method to fetch ip address"""
+    if not 'ip_list' in self.parameter_dict:
+      raise AttributeError
+    for name, ip in self.parameter_dict['ip_list']:
+      if test_method(ip):
+        return ip
+    raise AttributeError
+
+  def getLocalIPv4Address(self):
+    """Returns local IPv4 address available on partition"""
+    # XXX: Lack checking for locality of address
+    return self._getIpAddress(netaddr.valid_ipv4)
+
+  def getGlobalIPv6Address(self):
+    """Returns global IPv6 address available on partition"""
+    # XXX: Lack checking for globality of address
+    return self._getIpAddress(netaddr.valid_ipv6)
+
+  def createConfigurationFile(self, name, content):
+    """Creates named configuration file and returns its path"""
+    file_path = os.path.join(self.etc_directory, name)
+    self._writeFile(file_path, content)
+    self.logger.debug('Created configuration file: %r' % file_path)
+    return file_path
+
+  def createRunningWrapper(self, wrapper_name, file_content):
+    """Creates named running wrapper and returns its path"""
+    wrapper_path = os.path.join(self.wrapper_directory, wrapper_name)
+    self._writeExecutable(wrapper_path, file_content)
+    return wrapper_path
+
+  def createReportRunningWrapper(self, file_content):
+    """Creates report runnig wrapper and returns its path"""
+    report_wrapper_path = os.path.join(self.wrapper_report_directory,
+        'slapreport')
+    self._writeExecutable(report_wrapper_path, file_content)
+    return report_wrapper_path
+
+  def substituteTemplate(self, template_location, mapping_dict):
+    """Returns template content after substitution"""
+    return open(template_location, 'r').read() % mapping_dict
+
+  def _writeExecutable(self, path, content, mode='0700'):
+    """Creates file in path with content and sets mode
+
+    If file was created or altered returns true
+    Otherwise returns false
+
+    To be used to create executables
+
+    Raises os related errors"""
+    return self._writeFile(path, content, mode)
+
+  def _writeFile(self, path, content, mode='0600'):
+    """Creates file in path with content and sets mode
+
+    If file was created or altered returns true
+    Otherwise returns false
+
+    Raises os related errors"""
+
+    file_altered = False
+    if not os.path.exists(path):
+      open(path, 'w').write(content)
+      file_altered = True
+    else:
+      new_sum = md5()
+      current_sum = md5()
+      new_sum.update(content)
+      current_sum.update(open(path, 'r').read())
+      if new_sum.digest() != current_sum.digest():
+        file_altered = True
+        open(path, 'w').write(content)
+
+    if oct(stat.S_IMODE(os.stat(path).st_mode)) != mode:
+      os.chmod(path, int(mode, 8))
+      file_altered = True
+
+    return file_altered
+
+  def createBackupDirectory(self, name, mode='0700'):
+    """Creates named directory in self.backup_directory and returns its path"""
+    path = os.path.join(self.backup_directory, name)
+    self._createDirectory(path, mode)
+    return path
+
+  def createDataDirectory(self, name, mode='0700'):
+    """Creates named directory in self.data_root_directory and returns its path"""
+    path = os.path.join(self.data_root_directory, name)
+    self._createDirectory(path, mode)
+    return path
+
+  def _createDirectory(self, path, mode='0700'):
+    """Creates path directory and sets mode
+
+    If directory was created or its mode was altered returns true
+    Otherwise returns false
+
+    Raises os related errors"""
+    directory_altered = False
+    if not os.path.exists(path):
+      os.mkdir(path, int(mode, 8))
+      directory_altered = True
+    if not os.path.isdir(path):
+      raise zc.buildout.UserError('Path %r exists, but it is not directory'
+          % path)
+    if oct(stat.S_IMODE(os.stat(path).st_mode)) != mode:
+      os.chmod(path, int(mode, 8))
+      directory_altered = True
+    if directory_altered:
+      self.logger.debug('Created directory %r with permission %r' % (path, mode))
+    return directory_altered
+
+  def _createDefaultDirectoryStructure(self):
+    for directory in self.default_directory_list:
+      self._createDirectory(directory)
+
+  def generatePassword(self, len=32):
+    """Generates password. Shall be secured, until then all are insecure"""
+    return 'insecure'
+
+  def install(self):
+    self.slap.initializeConnection(self.server_url, self.key_file,
+        self.cert_file)
+    self.computer_partition = self.slap.registerComputerPartition(
+      self.computer_id,
+      self.computer_partition_id)
+    self.request = self.computer_partition.request
+    self.setConnectionDict = self.computer_partition.setConnectionDict
+    self._createDefaultDirectoryStructure()
+    self.parameter_dict = self.computer_partition.getInstanceParameterDict()
+
+    # call children part of install
+    path_list = self._install()
+
+    return path_list
+
+  update = install
+
+  def _install(self):
+    """Hook which shall be implemented in children class"""
+    raise NotImplementedError('Shall be implemented by subclass')

slapos/slapos.lib.recipe/src/slapos/lib/recipe/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################

slapos/slapos.recipe.build/CHANGES.txt

+1.0 (unreleased)
+----------------
+
+ - initial release [Łukasz Nowak]

slapos/slapos.recipe.build/MANIFEST.in

+include CHANGES.txt

slapos/slapos.recipe.build/README.txt

+slapos.recipe.build
+===================
+
+Recipe to build the software.
+
+Example buildout::
+
+[buildout]
+parts =
+  file
+
+develop = slapos.recipe.build
+
+[zlib]
+# Use standard configure, make, make install way
+recipe = slapos.recipe.build:cmmi
+url = http://prdownloads.sourceforge.net/libpng/zlib-1.2.5.tar.gz?download
+md5sum = c735eab2d659a96e5a594c9e8541ad63
+slapos_promisee =
+  directory:include
+  file:include/zconf.h
+  file:include/zlib.h
+  directory:lib
+  statlib:lib/libz.a
+  dynlib:lib/libz.so linked:libc.so.6 rpath:
+  dynlib:lib/libz.so.1 linked:libc.so.6 rpath:
+  dynlib:lib/libz.so.1.2.5 linked:libc.so.6
+  directory:lib/pkgconfig
+  file:lib/pkgconfig/zlib.pc
+  directory:share
+  directory:share/man
+  directory:share/man/man3
+  file:share/man/man3/zlib.3
+
+[file]
+recipe = slapos.recipe.build:cmmi
+url = ftp://ftp.astron.com/pub/file/file-5.04.tar.gz
+md5sum = accade81ff1cc774904b47c72c8aeea0
+environment =
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib
+slapos_promisee =
+  directory:bin
+  dynlib:bin/file linked:libz.so.1,libc.so.6,libmagic.so.1 rpath:${zlib:location}/lib,!/lib
+  directory:include
+  file:include/magic.h
+  directory:lib
+  statlib:lib/libmagic.a
+  statlib:lib/libmagic.la
+  dynlib:lib/libmagic.so linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
+  dynlib:lib/libmagic.so.1 linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
+  dynlib:lib/libmagic.so.1.0.0 linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
+  directory:share
+  directory:share/man
+  directory:share/man/man1
+  file:share/man/man1/file.1
+  directory:share/man/man3
+  file:share/man/man3/libmagic.3
+  directory:share/man/man4
+  file:share/man/man4/magic.4
+  directory:share/man/man5
+  directory:share/misc
+  file:share/misc/magic.mgc
+
+[somethingelse]
+# default way with using script
+recipe = slapos.recipe.build
+url_0 = http://host/path/file.tar.gz
+md5sum = 9631070eac74f92a812d4785a84d1b4e
+script =
+  import os
+  os.chdir(%(work_directory)s)
+  unpack(%(url_0), strip_path=True)
+  execute('make')
+  execute('make install DEST=%(location)s')
+slapos_promisee =
+  ...
+
+[anythingelse]
+# reusing different recipe?
+recipe = slapos.recipe.build:backend
+backend = hexagonit.recipe.cmmi
+slapos_promisee =
+  ...
+# parameters to build
+
+TODO:
+
+ * add linking suport, buildout definition:
+
+slapos_link = <relative/path> [optional-path
+
+can be used as:
+
+[file]
+slapos_link =
+  bin/file
+  bin/file ${buildout:bin-directory}/bin/anotherfile
+
+Which will link ${file:location}/bin/file to ${buildout:bin-directory}/bin/file
+and ${file:location}/bin/file to ${buildout:bin-directory}/bin/anotherfile

slapos/slapos.recipe.build/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.recipe.build/setup.py

+from setuptools import setup, find_packages
+
+version = '1.0'
+name = 'slapos.recipe.build'
+long_description = open("README.txt").read() + "\n" + \
+    open("CHANGES.txt").read()
+
+setup(name=name,
+      version=version,
+      description="Simple download recipe",
+      long_description=long_description,
+      classifiers=[
+          "Framework :: Buildout :: Recipe",
+          "Programming Language :: Python",
+        ],
+      keywords='slapos recipe build',
+      license='GPLv3',
+      namespace_packages=['slapos'],
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      include_package_data=True,
+      install_requires=[
+        'setuptools', # for namespace and internal usage
+        'zc.buildout', # needed to play internally
+        ],
+      entry_points={'zc.buildout': [
+        'default = %s:Script' % name,
+        'cmmi = %s:Cmmi' % name,
+        ]},
+      zip_safe=True,
+    )

slapos/slapos.recipe.build/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.build/src/slapos/recipe/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.build/src/slapos/recipe/build/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import logging
+import os
+import setuptools
+import shutil
+import subprocess
+import tempfile
+import zc.buildout
+
+def readElfAsDict(f):
+  """Reads ELF information from file"""
+  popen = subprocess.Popen(['readelf', '-d', f],
+      stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  result = popen.communicate()[0]
+  if popen.returncode != 0:
+    raise AssertionError(result)
+  library_list = []
+  rpath_list = []
+  runpath_list = []
+  for l in result.split('\n'):
+    if '(NEEDED)' in l:
+      library_list.append(l.split(':')[1].strip(' []'))
+    elif '(RPATH)' in l:
+      rpath_list = [q.rstrip('/') for q in l.split(':',1)[1].strip(' []').split(':')]
+    elif '(RUNPATH)' in l:
+      runpath_list = [q.rstrip('/') for q in l.split(':',1)[1].strip(' []').split(':')]
+  if len(runpath_list) == 0:
+    runpath_list = rpath_list
+  elif len(rpath_list) != 0 and runpath_list != rpath_list:
+    raise ValueError('RPATH and RUNPATH are different.')
+  return dict(
+    library_list=sorted(library_list),
+    runpath_list=sorted(runpath_list)
+  )
+
+def call(*args, **kwargs):
+  """Subprocess call with closed file descriptors and stdin"""
+  kwargs.update(
+    stdin=subprocess.PIPE,
+    close_fds=True)
+  popen = subprocess.Popen(*args, **kwargs)
+  popen.stdin.flush()
+  popen.stdin.close()
+  popen.stdin = None
+  popen.communicate()
+  if popen.returncode != 0:
+    raise subprocess.CalledProcessError(popen.returncode, ' '.join(args[0]))
+
+def calls(call_string, **kwargs):
+  """Subprocesser caller which allows to pass arguments as string"""
+  call(call_string.split(), **kwargs)
+
+def guessworkdir(path):
+  if len(os.listdir(path)) == 1:
+    return os.path.join(path, os.listdir(path)[0])
+  return path
+
+class Script:
+  """Free script building system"""
+  def _checkPromisee(self, location):
+    promisee_problem_list = []
+    a = promisee_problem_list.append
+    for promisee in self.options['slapos_promisee'].split('\n'):
+      promisee = promisee.strip()
+      if not promisee:
+        continue
+      if promisee.startswith('file:') or promisee.startswith('statlib'):
+        s, path = promisee.split(':')
+        if not os.path.exists(os.path.join(location, path)):
+          a('File promisee not met for %r' % path)
+      elif promisee.startswith('directory'):
+        s, path = promisee.split(':')
+        if not os.path.isdir(os.path.join(location, path)):
+          a('Directory promisee not met for %r' %
+              path)
+      elif promisee.startswith('dynlib:'):
+        if 'linked:' not in promisee:
+          raise zc.buildout.UserError('dynlib promisee requires \'linked:\' '
+            'parameter.')
+        if 'rpath:' not in promisee:
+          rpath_list = []
+        for promisee_part in promisee.split():
+          if promisee_part.startswith('dynlib:'):
+            s, path = promisee_part.split(':')
+          elif promisee_part.startswith('linked:'):
+            s, link_list = promisee_part.split(':')
+            link_list = link_list.split(',')
+          elif promisee_part.startswith('rpath:'):
+            s, rpath_list = promisee_part.split(':')
+            if rpath_list:
+              r = rpath_list
+              rpath_list = []
+              for q in r.split(','):
+                if q.startswith('!'):
+                  q = q.replace('!', location)
+                rpath_list.append(q)
+            else:
+              rpath_list = []
+        if not os.path.exists(os.path.join(location, path)):
+          a('Dynlib promisee file not met %r' % promisee)
+        else:
+          elf_dict = readElfAsDict(os.path.join(location, path))
+          if sorted(link_list) != sorted(elf_dict['library_list']):
+            a('Promisee library list not met (wanted: %r, found: %r)'%(
+              link_list, elf_dict['library_list']))
+          if sorted(rpath_list) != sorted(elf_dict['runpath_list']):
+            a('Promisee rpath list not met (wanted: %r, found: %r)'%(
+              rpath_list, elf_dict['runpath_list']))
+      else:
+        raise zc.buildout.UserError('Unknown promisee %r' % promisee)
+    if len(promisee_problem_list):
+      raise zc.buildout.UserError('Promisee not met, found issues:\n  %s' %
+          '  '.join([q+'\n' for q in promisee_problem_list]))
+
+  def download(self, url, md5sum):
+    download = zc.buildout.download.Download(self.buildout['buildout'],
+        hash_name=True)
+    path, is_temp = download(url, md5sum=self.options.get('md5sum'))
+    return path
+
+  def extract(self, path):
+    extract_dir = tempfile.mkdtemp(self.name)
+    self.logger.debug('Created working directory %r' % extract_dir)
+    setuptools.archive_util.unpack_archive(path, extract_dir)
+    self.cleanup_dir_list.append(extract_dir)
+    return extract_dir
+
+  script = 'raise NotImplementedError'
+  def __init__(self, buildout, name, options):
+    self.cleanup_dir_list = []
+    self.options = options
+    self.buildout = buildout
+    self.name = name
+    self.logger = logging.getLogger('SlapOS build of %s' % self.name)
+
+    self.options.setdefault('location',
+        os.path.join(buildout['buildout']['parts-directory'], self.name))
+
+    # cleanup some variables
+    for k in ['location', 'url', 'md5sum']:
+      self.options[k] = self.options.get(k, '').strip()
+    self.options['script'] = self.options.get('script', self.script) % self.options
+
+  def getEnvironment(self):
+    # prepare cool dictionary
+    wanted_env = {}
+    for line in self.options.get('environment', '').splitlines():
+      line = line.strip()
+      if not line:
+        continue
+      if not '=' in line:
+        raise zc.buildout.UserError('Line %r in environment is incorrect' % line)
+
+      key, value = line.split('=')
+      key = key.strip()
+      value = value.strip()
+      if key in wanted_env:
+        raise zc.buildout.UserError('Key %r is repeated' % key)
+      wanted_env[key] = value
+    env = {}
+    for k,v in os.environ.iteritems():
+      change = wanted_env.pop(k, None)
+      if change is not None:
+        env[k] = change % os.environ
+        self.logger.info('Environment %r setup to %r' % (k, env[k]))
+      else:
+        env[k] =v
+    for k,v in wanted_env.iteritems():
+      self.logger.info('Environment %r added with %r' % (k, v))
+      env[k] = v
+    return env
+
+  def install(self):
+    try:
+      env = self.getEnvironment()
+      exec self.options['script']
+      try:
+        self._checkPromisee(self.options['location'])
+      except Exception:
+        if os.path.exists(self.options['location']):
+          self.logger.info('Removing location %r because of error' % self.options['location'])
+          shutil.rmtree(self.options['location'])
+        raise
+    finally:
+      for d in self.cleanup_dir_list:
+        if os.path.exists(d):
+          self.logger.debug('Cleanup directory %r' % d)
+          shutil.rmtree(d)
+
+    return [self.options['location']]
+
+  def update(self):
+    pass
+
+class Cmmi(Script):
+  """Simple configure-make-make-insall compatible with hexagonit.recipe.cmmi
+
+  Compatibility on parameter level, without bug-to-bug, hack-to-hack"""
+
+  script = """
+extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
+workdir = guessworkdir(extract_dir)
+configure_command = ["./configure", "--prefix=%(location)s"]
+configure_command.extend(%(configure-options)r.split())
+self.logger.info('Configuring with: %%s' %% configure_command)
+call(configure_command, cwd=workdir, env=env)
+self.logger.info('Building')
+call("make", cwd=workdir, env=env)
+self.logger.info('Installing')
+call(["make", "install"], cwd=workdir, env=env)
+"""
+
+  def __init__(self, buildout, name, options):
+    options['configure-options'] = ' '.join(options.get('configure-options', '').strip().splitlines())
+    Script.__init__(self, buildout, name, options)

slapos/slapos.recipe.download/CHANGES.txt

+1.0 (unreleased)
+----------------
+
+ - initial release [Łukasz Nowak]

slapos/slapos.recipe.download/MANIFEST.in

+include CHANGES.txt

slapos/slapos.recipe.download/README.txt

+slapos.recipe.download
+======================
+
+Extremely simple recipe to download using zc.buildout download utility.
+
+Usage
+-----
+
+  [buildout]
+  parts =
+    download
+
+  [download]
+  recipe = slapos.recipe.download
+  url = https://some.url/file
+
+Such profile will download https://some.url/file and put it in
+buildout:parts-directory/download/download
+
+filename parameter can be used to change destination named filename.
+
+destination parameter allows to put explicit destination.
+
+md5sum parameter allows pass md5sum.
+
+mode (octal, so for rw-r--r-- use 0644) allows to set mode
+
+Exposes target attribute which is path to downloaded file.
+
+Notes
+-----
+
+This recipe suffers from buildout download utility issue, which will do not
+try to redownload resource with wrong md5sum.

slapos/slapos.recipe.download/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.recipe.download/setup.py

+from setuptools import setup, find_packages
+
+version = '1.0'
+name = 'slapos.recipe.download'
+long_description = open("README.txt").read() + "\n" + \
+    open("CHANGES.txt").read()
+
+setup(name=name,
+      version=version,
+      description="Simple download recipe",
+      long_description=long_description,
+      classifiers=[
+          "Framework :: Buildout :: Recipe",
+          "Programming Language :: Python",
+        ],
+      keywords='slapos download',
+      license='GPLv3',
+      namespace_packages=['slapos'],
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      include_package_data=True,
+      install_requires=[
+        'setuptools', # for namespace
+        'zc.buildout', # needed to play internally
+        ],
+      entry_points={'zc.buildout': ['default = %s:Recipe' % name]},
+      zip_safe=True,
+    )

slapos/slapos.recipe.download/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.download/src/slapos/recipe/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.download/src/slapos/recipe/download/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import os
+import logging
+import shutil
+import zc.buildout
+class Recipe:
+  def __init__(self, buildout, name, options):
+    self.buildout = buildout
+    self.name = name
+    self.options = options
+    self.logger = logging.getLogger(self.name)
+    if 'filename' in self.options and 'destination' in self.options:
+      raise zc.buildout.UserError('Parameters filename and destination are '
+          'exclusive.')
+    self.parts = None
+    self.destination = self.options.get('destination', None)
+    if self.destination is None:
+      self.parts = os.path.join(self.buildout['buildout']['parts-directory'],
+          self.name)
+      self.destination = os.path.join(self.parts, self.options.get('filename',
+        self.name))
+    options['target'] = self.destination
+
+  def install(self):
+    if self.parts is not None:
+      if not os.path.isdir(self.parts):
+        os.mkdir(self.parts)
+    download = zc.buildout.download.Download(self.buildout['buildout'],
+        hash_name=True)
+    path, is_temp = download(self.options['url'],
+        md5sum=self.options.get('md5sum'))
+    if os.path.exists(self.destination):
+      os.unlink(self.destination)
+    shutil.copy(path, self.destination)
+    mode = self.options.get('mode')
+    if mode is not None:
+      mode = int(mode, 8)
+      os.chmod(self.destination, mode)
+      self.logger.debug('Mode of %r set to 0%o.' % (self.destination, mode))
+    self.logger.debug('Downloaded %r and saved to %r.' % (self.options['url'],
+      self.destination))
+    if self.parts is not None:
+      return [self.parts]
+    else:
+      return []
+
+  update = install

slapos/slapos.recipe.libcloud/CHANGES.txt

+0.0.3 (unreleased)
+------------------
+
+0.0.2 (2010-10-22)
+------------------
+ - Working version
+
+0.0.1 (2010-09-28)
+------------------
+ - Initial release
+   [Lukasz Nowak]

slapos/slapos.recipe.libcloud/README.txt

+libcloud recipe
+*********************
+
+Slapified recipe to interact with any libcloud supported IaaS system
\ No newline at end of file

slapos/slapos.recipe.libcloud/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.recipe.libcloud/setup.py

+from setuptools import setup, find_packages
+
+version = '0.0.3'
+name = 'slapos.recipe.libcloud'
+long_description = open("README.txt").read() + "\n" + open("CHANGES.txt").read()
+
+setup(name=name,
+      version=version,
+      description="Slapified buildout recipe to interact with libcloud",
+      long_description=long_description,
+      classifiers=[
+          "Framework :: Buildout :: Recipe",
+          "Programming Language :: Python",
+        ],
+      keywords='slap recipe libcloud',
+      license='undecided',
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      namespace_packages=['slapos', 'slapos.recipe'],
+      include_package_data=True,
+      install_requires=[
+        'slapos.tool.cloudmgr',
+        'setuptools',
+        'slapos.lib.recipe',
+        'zc.buildout',
+        'zc.recipe.egg',
+        ],
+      zip_safe=True,
+      entry_points={'zc.buildout': ['default = %s:Recipe' % name]}
+    )

slapos/slapos.recipe.libcloud/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

slapos/slapos.recipe.libcloud/src/slapos/recipe/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

slapos/slapos.recipe.libcloud/src/slapos/recipe/libcloud/__init__.py

+from slapos.lib.recipe.BaseSlapRecipe import BaseSlapRecipe
+import os
+import sys
+import zc.buildout
+import zc.recipe.egg
+from slapos.slap.slap import ServerError
+from slapos.tool.cloudmgr.cloudinterface import NodeInterface
+from pprint import pformat
+class SlavePartitionError(Exception):
+  pass
+
+class Recipe(BaseSlapRecipe):
+  SECURITY_GROUP_NAME = 'VifibEC2Security'
+  def __init__(self, buildout, name, options):
+    BaseSlapRecipe.__init__(self, buildout, name, options)
+    self.destroy_wrapper_location = os.path.join(self.bin_directory, 'destroy')
+    self.running_wrapper_location = os.path.join(self.bin_directory, 'run')
+
+    # wrapper parts
+    options['scripts'] = '''
+      run
+      destroy
+    '''
+    options['entry-points'] = '''
+      run=slapos.recipe.libcloud.run:run
+      destroy=slapos.recipe.libcloud.destroy:destroy
+    '''
+    self.egg = zc.recipe.egg.Egg(buildout, '', options)
+
+    self.configuration_file = os.path.join(self.etc_directory, 'cloudmgr.cnf')
+
+  def _loadSecretAndKey(self):
+    """Loads security parameters for connection"""
+    self.secret = open(os.path.join(self.work_directory, 'secret.txt')
+        ).read().strip()
+    self.key = open(os.path.join(self.work_directory, 'key.txt')
+        ).read().strip()
+
+  def _updateConfigurationFile(self):
+    configuration_dict = dict(
+      key=self.key,
+      secret=self.secret,
+      node_list=self.slave_partition_configuration_dict_list
+    )
+    self._writeFile(self.configuration_file, pformat(configuration_dict))
+
+  def _install(self):
+    """libcloud compatible machine is installed by creating wrapper, which
+    will run succesfully as long as the machine is available.
+    """
+    self._loadSecretAndKey()
+    self.slave_partition_configuration_dict_list = []
+    self.egg.extra_paths = sys.path
+    for slave_partition in [self.slap.registerComputerPartition(
+      self.computer_id, slave_id) for slave_id in self.computer_partition\
+          .getInstanceParameterDict()['slave_id_list']]:
+      try:
+        self.slave_partition_configuration_dict_list.append(
+            self._installSlavePartition(slave_partition))
+      except SlavePartitionError, e:
+        self.logger.warning('Slave Parttion %r not installed, issue: %r'%(
+          slave_partition.getId(), e))
+    # Installs wrappers
+    self._updateConfigurationFile()
+    self.options['arguments'] = "server_binary = %r, configuration_file = %r"%(
+        self.options['server_binary'], self.configuration_file)
+    self.egg.install()
+    os.chmod(os.path.join(
+      self.running_wrapper_location), int('0700', 8))
+    os.chmod(os.path.join(
+      self.destroy_wrapper_location), int('0700', 8))
+    return []
+
+  def _installSlavePartition(self, slave_partition):
+    requested_dict = slave_partition.getInstanceParameterDict()
+    requested_dict.setdefault('service', 'EC2_EU_WEST')
+    requested_dict.setdefault('location', '0')
+    requested_dict.setdefault('image', 'ami-05cae171')
+    requested_dict.setdefault('size', 'm1.smal')
+    requested_dict.setdefault('security_group', 'VifibEC2Security')
+    connection_dict = slave_partition.getConnectionDict()
+    node_kw = dict(
+      key = self.key,
+      secret = self.secret,
+      service = requested_dict['service'],
+      location = requested_dict['location'],
+      node_uuid = connection_dict.get('node_uuid', None),
+      ssh_key = connection_dict.get('ssh_key', None)
+    )
+    node = NodeInterface(**node_kw)
+    update_kw = dict(
+      image = requested_dict['image'],
+      size = requested_dict['size'],
+      security_group = requested_dict['security_group'],
+    )
+    self.logger.info('Updating %r' % slave_partition.getId())
+    connection_dict.update(node.update(**update_kw))
+    self.logger.info('Fetching public ip of %r' % slave_partition.getId())
+    connection_dict.update(node.getPublicIpList())
+    slave_partition.available()
+    connection_dict.setdefault('username', 'root')
+    slave_partition.setConnectionDict(connection_dict)
+    requested_dict.update(connection_dict)
+    slave_partition_state = slave_partition.getState()
+    # as cloudmgr is not related with slap and runs as async process to recipe
+    # assume that whatever came from slave is correctly done
+    if slave_partition_state in ['started', 'stopped']:
+      # stopped cannot be supported, because in case of libcloud it is equal
+      # to destroyed
+      # even worse: stopped is the first state for installed partition
+      try:
+        getattr(slave_partition, slave_partition_state)()
+      except ServerError:
+        # Recipe is becoming responsible for system state, so it have to
+        # not die in case of slap server error
+        pass
+      requested_dict['requested_state'] = 'started'
+    elif slave_partition_state == 'destroyed':
+      requested_dict['requested_state'] = slave_partition_state
+      try:
+        slave_partition.destroyed()
+      except ServerError:
+        # Recipe is becoming responsible for system state, so it have to
+        # not die in case of slap server error
+        pass
+    return requested_dict
+

slapos/slapos.recipe.libcloud/src/slapos/recipe/libcloud/run.py

+from os import execl
+import sys
+
+def run(server_binary, configuration_file):
+  sys.stdout.flush()
+  execl(server_binary, server_binary, configuration_file)

slapos/slapos.recipe.libcloudrequest/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.recipe.libcloudrequest/setup.py

+from setuptools import setup, find_packages
+
+version = '0.0.3'
+name = 'slapos.recipe.libcloudrequest'
+long_description = open("README.txt").read() + "\n" + open("CHANGES.txt").read()
+
+setup(name=name,
+      version=version,
+      description="Slapified buildout recipe to request libcloud instance",
+      long_description=long_description,
+      classifiers=[
+          "Framework :: Buildout :: Recipe",
+          "Programming Language :: Python",
+        ],
+      keywords='slap recipe libcloud reques',
+      license='GPLv3',
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      namespace_packages=['slapos', 'slapos.recipe'],
+      include_package_data=True,
+      install_requires=[
+        'setuptools',
+        'slapos.lib.recipe',
+        'zc.buildout',
+        ],
+      zip_safe=True,
+      entry_points={'zc.buildout': ['default = %s:Recipe' % name]}
+    )

slapos/slapos.recipe.libcloudrequest/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.libcloudrequest/src/slapos/recipe/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.libcloudrequest/src/slapos/recipe/libcloudrequest/__init__.py

+from slapos.lib.recipe.BaseSlapRecipe import BaseSlapRecipe
+import zc.buildout
+class Recipe(BaseSlapRecipe):
+  def _install(self):
+    amazon = self.request(
+        'https://svn.erp5.org/repos/public/slapos/trunk/software_release/libcloud/software.cfg',
+        'Amazon EC Image',
+        'Amazon EC Connector Partition',
+        True)
+    amazon_dict = amazon.getConnectionDict()
+    if amazon_dict == {}:
+      raise zc.buildout.UserError('Slave not installed yet')
+    self.computer_partition.setConnectionDict(amazon.getConnectionDict())
+    return []

slapos/slapos.recipe.nosqltestbed/CHANGES.txt

+Changelog
+=========
+
+0.0.9 (2011/04/06)
+------------------
+
+ - resetting the default software type to kumo_cloud
+   [Guillaume Bottex]
+
+0.0.8 (2011/03/29)
+------------------
+
+ - Adding user input support to set testing options (number of testers and servers, number of requests, ...)
+   [Guillaume Bottex]
+
+0.0.7 (2011/03/03)
+------------------
+
+ - stripping connection_xml data for each software type in a better way
+   [Guillaume Bottex]
+
+0.0.6 (2011/03/02)
+------------------
+
+ - stripping connection_xml data for each software type
+   [Guillaume Bottex]
+
+0.0.5 (2011/03/01)
+------------------
+
+ - Fixing mistake
+   [Guillaume Bottex]
+
+0.0.4 (2011/02/28)
+------------------
+
+ - Adapting the __init__ method to the modification made to slapgrid
+   [Guillaume Bottex]
+
+0.0.3 (2011/02/24)
+------------------
+
+ - Fixing mistakes
+   [Guillaume Bottex]
+
+0.0.2 (2011/02/24)
+------------------
+
+ - Fixing mistakes
+   [Guillaume Bottex]
+
+0.0.1dev (2011/02/24)
+---------------------
+
+ - Initial version
+   [Guillaume Bottex]

slapos/slapos.recipe.nosqltestbed/MANIFEST.in

+include CHANGES.txt
+recursive-include src/slapos/recipe/nosqltestbed *.in

slapos/slapos.recipe.nosqltestbed/README.txt

+slapos.recipe.nosqltestbed
+==========================

slapos/slapos.recipe.nosqltestbed/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.recipe.nosqltestbed/setup.py

+from setuptools import setup, find_packages
+import os
+
+name = "slapos.recipe.nosqltestbed"
+version = '0.0.9'
+
+def read(*rnames):
+  return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+
+long_description=( read('README.txt')
+                   + '\n' +
+                   read('CHANGES.txt')
+                 )
+
+setup(
+    name = name,
+    version = version,
+    description = "ZC Buildout recipe for the No SQL test bed",
+    long_description=long_description,
+    license = "GPLv3",
+    keywords = "buildout nosqltestbed",
+    classifiers=[
+        "Framework :: Buildout :: Recipe",
+        "Programming Language :: Python",
+    ],
+    packages = find_packages('src'),
+    package_dir = {'': 'src'},
+    include_package_data=True,
+    install_requires = [
+      'zc.recipe.egg',
+      'setuptools',
+      'slapos.lib.recipe',
+      ],
+    namespace_packages = ['slapos', 'slapos.recipe'],
+    entry_points = {'zc.buildout': ['default = %s:NoSQLTestBed' % name]},
+    )

slapos/slapos.recipe.nosqltestbed/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__) 

slapos/slapos.recipe.nosqltestbed/src/slapos/recipe/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__) 

slapos/slapos.recipe.nosqltestbed/src/slapos/recipe/nosqltestbed/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import urllib
+import urllib2
+import pkg_resources
+from slapos.lib.recipe.BaseSlapRecipe import BaseSlapRecipe
+
+class NoSQLTestBed(BaseSlapRecipe):
+
+  def _install(self):
+    self.parameter_dict = self.computer_partition.getInstanceParameterDict()
+    software_type = self.parameter_dict.get('slap_software_type', 'default')
+    if software_type is None or software_type == 'RootSoftwareInstance':
+      software_type = 'kumo_cloud'
+    if "run_%s" % software_type in dir(self) and \
+       callable(getattr(self, "run_%s" % software_type)):
+      return getattr(self, "run_%s" % software_type)()
+    else:
+      raise NotImplementedError("Do not support %s" % software_type)
+
+  def run_kumo_cloud(self):
+    """ Deploy kumofs systeom on a cloud. """
+
+    kumo_cloud_config = {}
+    kumo_cloud_config.update(self.options)
+    kumo_cloud_config.update(self.parameter_dict)
+
+    kumo_cloud_config['address'] = self.getGlobalIPv6Address()
+    kumo_cloud_config['report_path'] = self.log_directory
+    
+    if 'nb_server_max' not in kumo_cloud_config:
+      kumo_cloud_config['nb_server_max'] = 3
+    
+    if 'nb_tester_max' not in kumo_cloud_config:
+      kumo_cloud_config['nb_tester_max'] = 3
+    
+    if 'nb_thread' not in kumo_cloud_config:
+      kumo_cloud_config['nb_thread'] = 1
+    
+    if 'nb_request' not in kumo_cloud_config:
+      kumo_cloud_config['nb_request'] = 1000
+    
+    kumo_cloud_config['software_release_url'] = self.software_release_url
+    kumo_cloud_config['server_url'] = self.server_url
+    kumo_cloud_config['key_file'] = self.key_file
+    kumo_cloud_config['cert_file'] = self.cert_file
+    kumo_cloud_config['computer_id'] = self.computer_id
+    kumo_cloud_config['computer_partition_id'] = self.computer_partition_id
+    kumo_cloud_config['plugin_name'] = 'kumo'
+
+    kumo_cloud_connection = {}
+    kumo_cloud_connection['url'] = "http://["+kumo_cloud_config['address']+"]:5000/"
+    self.computer_partition.setConnectionDict(kumo_cloud_connection)
+
+    nosqltester_manager_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'kumotester_manager_run.in'))
+    nosqltester_manager_runner_path = self.createRunningWrapper("kumotester_manager",
+          self.substituteTemplate(nosqltester_manager_wrapper_template_location, kumo_cloud_config))
+
+    return [nosqltester_manager_runner_path]
+
+  def run_all(self):
+    """ Runs all services on one machine. """
+    all_config = {}
+    all_config.update(self.options)
+
+    ipaddress = "[%s]" % self.getGlobalIPv6Address()
+
+    all_config['manager_address'] = ipaddress
+    all_config['manager_port'] = 19700
+    all_config['server_address'] = ipaddress
+    all_config['server_port'] = 19800
+    all_config['server_listen_port'] = 19900
+    all_config['server_storage'] = os.path.join(self.data_root_directory, "kumodb.tch")
+    all_config['gateway_address'] = ipaddress
+    all_config['gateway_port'] = 11411
+    all_config['manager_log'] = os.path.join(self.log_directory, "kumo-manager.log")
+    all_config['server_log'] = os.path.join(self.log_directory, "kumo-server.log")
+    all_config['gateway_log'] = os.path.join(self.log_directory, "kumo-gateway.log")
+
+    manager_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'kumo_manager_run.in'))
+    manager_runner_path = self.createRunningWrapper("kumo-manager",
+          self.substituteTemplate(manager_wrapper_template_location, all_config))
+    server_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'kumo_server_run.in'))
+    server_runner_path = self.createRunningWrapper("kumo-server",
+          self.substituteTemplate(server_wrapper_template_location, all_config))
+    gateway_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'kumo_gateway_run.in'))
+    gateway_runner_path = self.createRunningWrapper("kumo-gateway",
+          self.substituteTemplate(gateway_wrapper_template_location, all_config))
+
+    return [manager_runner_path, server_runner_path, gateway_runner_path]
+
+  def run_kumo_manager(self, ipaddress=None):
+    """ Runs the kumofs manager. """
+    manager_config = {}
+    manager_config.update(self.options)
+
+    if ipaddress is None:
+      manager_config['manager_address'] = "[%s]" % self.getGlobalIPv6Address()
+    else:
+      manager_config['manager_address'] = ipaddress
+    
+    manager_config['manager_port'] = 19700
+    manager_config['manager_log'] = os.path.join(self.log_directory, "kumo-manager.log")
+
+    manager_connection = {}
+    manager_connection['address'] = manager_config['manager_address']
+    manager_connection['port'] = manager_config['manager_port']
+    self.computer_partition.setConnectionDict(manager_connection)
+
+    manager_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'kumo_manager_run.in'))
+    manager_runner_path = self.createRunningWrapper("kumo-manager",
+          self.substituteTemplate(manager_wrapper_template_location, manager_config))
+
+    return [manager_runner_path]
+
+  def run_kumo_server(self):
+    """ Runs the kumofs server. """
+    server_config = {}
+    server_config.update(self.options)
+    server_config.update(self.parameter_dict)
+
+    server_config['server_address'] = "[%s]" % self.getGlobalIPv6Address()
+    server_config['server_port'] = 19800
+    server_config['server_listen_port'] = 19900
+    server_config['server_storage'] = os.path.join(self.var_directory,"kumodb.tch")
+    server_config['server_log'] = os.path.join(self.log_directory, "kumo-server.log")
+
+    server_connection = {}
+    server_connection['address'] = server_config['server_address']
+    self.computer_partition.setConnectionDict(server_connection)
+
+    server_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'kumo_server_run.in'))
+    server_runner_path = self.createRunningWrapper("kumo-server",
+          self.substituteTemplate(server_wrapper_template_location, server_config))
+
+    return [server_runner_path]
+
+  def run_kumo_gateway(self):
+    """ Runs the kumofs gateway. """
+    gateway_config = {}
+    gateway_config.update(self.options)
+    gateway_config.update(self.parameter_dict)
+
+    gateway_config['gateway_address'] = "[%s]" % self.getGlobalIPv6Address()
+    gateway_config['gateway_port'] = 11411
+    gateway_config['gateway_log'] = os.path.join(self.log_directory, "kumo-gateway.log")
+
+    gateway_connection = {}
+    gateway_connection['address'] = gateway_config['gateway_address']
+    gateway_connection['port'] = gateway_config['gateway_port']
+    self.computer_partition.setConnectionDict(gateway_connection)
+
+    gateway_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'kumo_gateway_run.in'))
+    gateway_runner_path = self.createRunningWrapper("kumo-gateway",
+          self.substituteTemplate(gateway_wrapper_template_location, gateway_config))
+
+    return [gateway_runner_path]
+
+  def run_kumo_tester(self):
+    """ Runs the kumofs tester. """
+    tester_config = {}
+    tester_config.update(self.options)
+    tester_config.update(self.parameter_dict)
+
+    tester_config['tester_address'] = self.getGlobalIPv6Address()
+    # tester_config['url'] = "http://%s:5000/" % tester_config['tester_address']
+    # tester_config['start_url'] = "http://%s:5000/start" % tester_config['tester_address']
+    tester_config['report_path'] = self.log_directory
+    tester_config['binary'] = tester_config['memstrike_binary'] + " -l " + \
+                              tester_config['gateway_address'].strip("[]") + " -p " + \
+                              tester_config['gateway_port'] + " -t " + \
+                              tester_config['nb_thread'] + " " + \
+                              tester_config['nb_request'] #" 1000" " -t 32 1024000"
+
+    tester_connection = {}
+    tester_connection['start_url'] = "http://%s:5000/start" % tester_config['tester_address']
+    self.computer_partition.setConnectionDict(tester_connection)
+
+    tester_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'nosqltester_run.in'))
+    tester_runner_path = self.createRunningWrapper("nosqltester",
+          self.substituteTemplate(tester_wrapper_template_location, tester_config))
+
+    return [tester_runner_path]
+

slapos/slapos.recipe.nosqltestbed/src/slapos/recipe/nosqltestbed/template/kumo_gateway_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(gateway_binary)s -F -E -m %(manager_address)s:%(manager_port)s -t %(gateway_address)s:%(gateway_port)s --verbose -o %(gateway_log)s

slapos/slapos.recipe.nosqltestbed/src/slapos/recipe/nosqltestbed/template/kumo_manager_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(manager_binary)s -a -l %(manager_address)s:%(manager_port)s --verbose -o %(manager_log)s

slapos/slapos.recipe.nosqltestbed/src/slapos/recipe/nosqltestbed/template/kumo_server_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(server_binary)s -l %(server_address)s:%(server_port)s -L %(server_listen_port)s -m %(manager_address)s:%(manager_port)s -s %(server_storage)s  --verbose -o %(server_log)s

slapos/slapos.recipe.nosqltestbed/src/slapos/recipe/nosqltestbed/template/kumotester_manager_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(nosqltester_manager_binary)s -a %(address)s -r %(report_path)s -s %(nb_server_max)s -t %(nb_tester_max)s %(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s %(plugin_name)s %(nb_thread)s %(nb_request)s

slapos/slapos.recipe.nosqltestbed/src/slapos/recipe/nosqltestbed/template/nosqltester_manager_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(nosqltester_manager_binary)s -a %(address)s -r %(report_path)s -s %(nb_server_max)s -t %(nb_tester_max)s %(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s %(plugin_name)s

slapos/slapos.recipe.nosqltestbed/src/slapos/recipe/nosqltestbed/template/nosqltester_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(nosqltester_binary)s -h %(host_address)s -a %(tester_address)s -r %(report_path)s -b "%(binary)s"

slapos/slapos.recipe.sheepdogtestbed/CHANGES.txt

+Changelog
+=========
+
+0.0.1dev (2011/05/16)
+---------------------
+
+ - Initial version
+   [Guillaume Bottex]

slapos/slapos.recipe.sheepdogtestbed/MANIFEST.in

+include CHANGES.txt
+recursive-include src/slapos/recipe/sheepdogtestbed *.in

slapos/slapos.recipe.sheepdogtestbed/README.txt

+slapos.recipe.sheepdogtestbed
+=============================

slapos/slapos.recipe.sheepdogtestbed/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.recipe.sheepdogtestbed/setup.py

+from setuptools import setup, find_packages
+import os
+
+name = "slapos.recipe.sheepdogtestbed"
+version = '0.0.1'
+
+def read(*rnames):
+  return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+
+long_description=( read('README.txt')
+                   + '\n' +
+                   read('CHANGES.txt')
+                 )
+
+setup(
+    name = name,
+    version = version,
+    description = "ZC Buildout recipe for the sheepdog test bed",
+    long_description=long_description,
+    license = "GPLv3",
+    keywords = "buildout sheepdogtestbed",
+    classifiers=[
+        "Framework :: Buildout :: Recipe",
+        "Programming Language :: Python",
+    ],
+    packages = find_packages('src'),
+    package_dir = {'': 'src'},
+    include_package_data=True,
+    install_requires = [
+      'zc.recipe.egg',
+      'setuptools',
+      'slapos.lib.recipe',
+      ],
+    namespace_packages = ['slapos', 'slapos.recipe'],
+    entry_points = {'zc.buildout': ['default = %s:SheepDogTestBed' % name]},
+    )

slapos/slapos.recipe.sheepdogtestbed/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__) 

slapos/slapos.recipe.sheepdogtestbed/src/slapos/recipe/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__) 

slapos/slapos.recipe.sheepdogtestbed/src/slapos/recipe/sheepdogtestbed/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import urllib
+import urllib2
+import pkg_resources
+from slapos.lib.recipe.BaseSlapRecipe import BaseSlapRecipe
+
+class SheepDogTestBed(BaseSlapRecipe):
+
+  def _install(self):
+    self.parameter_dict = self.computer_partition.getInstanceParameterDict()
+    software_type = self.parameter_dict.get('slap_software_type', 'default')
+    if software_type is None or software_type == 'RootSoftwareInstance':
+      software_type = 'default'
+    if "run_%s" % software_type in dir(self) and \
+       callable(getattr(self, "run_%s" % software_type)):
+      return getattr(self, "run_%s" % software_type)()
+    else:
+      raise NotImplementedError("Do not support %s" % software_type)
+
+  def run_default(self):
+    return self.run_sheepdog_test()
+
+  def run_sheepdog_test(self):
+    """ Launch sheepdog test process. """
+    
+    sheepdog_test_config = {}
+    sheepdog_test_config.update(self.options)
+    sheepdog_test_config.update(self.parameter_dict)
+
+    sheepdog_test_config['address'] = self.getGlobalIPv6Address()
+    sheepdog_test_config['report_path'] = self.log_directory
+    
+    if 'nb_server_max' not in sheepdog_test_config:
+      sheepdog_test_config['nb_server_max'] = 0
+    
+    if 'nb_tester_max' not in sheepdog_test_config:
+      sheepdog_test_config['nb_tester_max'] = 3
+    
+    if 'nb_thread' not in sheepdog_test_config:
+      sheepdog_test_config['nb_thread'] = 1
+    
+    if 'nb_request' not in sheepdog_test_config:
+      sheepdog_test_config['nb_request'] = 1000
+
+
+    sheepdog_test_config['software_release_url'] = self.software_release_url
+    sheepdog_test_config['server_url'] = self.server_url
+    sheepdog_test_config['key_file'] = self.key_file
+    sheepdog_test_config['cert_file'] = self.cert_file
+    sheepdog_test_config['computer_id'] = self.computer_id
+    sheepdog_test_config['computer_partition_id'] = self.computer_partition_id
+    sheepdog_test_config['plugin_name'] = 'sheepdog'
+
+    sheepdog_test_connection = {}
+    sheepdog_test_connection['url'] = "http://["+sheepdog_test_config['address']+"]:5000/"
+    self.computer_partition.setConnectionDict(sheepdog_test_connection)
+    
+    nosqltester_manager_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'nosqltester_manager_run.in'))
+    nosqltester_manager_runner_path = self.createRunningWrapper("sheepdog_test_manager",
+          self.substituteTemplate(nosqltester_manager_wrapper_template_location, sheepdog_test_config))
+
+    return [nosqltester_manager_runner_path]
+
+  def run_sheepdog_tester(self):
+    """ Runs the sheepdog tester. """
+    tester_config = {}
+    tester_config.update(self.options)
+    tester_config.update(self.parameter_dict)
+
+    tester_config['tester_address'] = self.getGlobalIPv6Address()
+    tester_config['report_path'] = self.log_directory
+    tester_config['binary'] = tester_config['sheepstrike_binary'] + \
+                              " -t " + \
+                              tester_config['nb_thread'] + " " + \
+                              tester_config['nb_request'] #" 1000" " -t 32 1024000"
+
+    tester_connection = {}
+    tester_connection['start_url'] = "http://%s:5000/start" % tester_config['tester_address']
+    self.computer_partition.setConnectionDict(tester_connection)
+
+    tester_wrapper_template_location = pkg_resources.resource_filename(
+                                             __name__, os.path.join(
+                                             'template', 'nosqltester_run.in'))
+    tester_runner_path = self.createRunningWrapper("nosqltester",
+          self.substituteTemplate(tester_wrapper_template_location, tester_config))
+
+    return [tester_runner_path]
+

slapos/slapos.recipe.sheepdogtestbed/src/slapos/recipe/sheepdogtestbed/template/nosqltester_manager_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(nosqltester_manager_binary)s -a %(address)s -r %(report_path)s -s %(nb_server_max)s -t %(nb_tester_max)s %(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s %(plugin_name)s

slapos/slapos.recipe.sheepdogtestbed/src/slapos/recipe/sheepdogtestbed/template/nosqltester_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(nosqltester_binary)s -h %(host_address)s -a %(tester_address)s -r %(report_path)s -b "%(binary)s"

slapos/slapos.recipe.template/CHANGES.txt

+1.0 (unreleased)
+----------------
+
+ - initial release [Łukasz Nowak]

slapos/slapos.recipe.template/MANIFEST.in

+include CHANGES.txt

slapos/slapos.recipe.template/README.txt

+slapos.recipe.template
+======================
+
+Fully networked template recipe, reusing collective.recipe.template with
+ability to download template over the network
+
+Usage
+-----
+
+  [buildout]
+  parts = template
+
+  [template]
+  recipe = slapos.recipe.template
+  url = http://server/with/template
+  # optional md5sum
+  md5sum = 1234567890
+  output = ${buildout:directory}/result
+
+All parameters except url and md5sum will be passed to
+collective.recipe.template, so please visit
+http://pypi.python.org/pypi/collective.recipe.template for full information.

slapos/slapos.recipe.template/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.recipe.template/setup.py

+from setuptools import setup, find_packages
+
+version = '1.0'
+name = 'slapos.recipe.template'
+long_description = open("README.txt").read() + "\n" + \
+    open("CHANGES.txt").read()
+
+setup(name=name,
+      version=version,
+      description="collective.recipe.template with network input support",
+      long_description=long_description,
+      classifiers=[
+          "Framework :: Buildout :: Recipe",
+          "Programming Language :: Python",
+        ],
+      keywords='slapos download',
+      license='GPLv3',
+      namespace_packages=['slapos'],
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      include_package_data=True,
+      install_requires=[
+        'collective.recipe.template', # used for real run
+        'setuptools', # for namespace
+        'zc.buildout', # needed to play internally
+        ],
+      entry_points={'zc.buildout': ['default = %s:Recipe' % name]},
+      zip_safe=True,
+    )

slapos/slapos.recipe.template/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.template/src/slapos/recipe/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.template/src/slapos/recipe/template/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import collective.recipe.template
+import zc.buildout
+class Recipe(collective.recipe.template.Recipe):
+  def __init__(self, buildout, name, options):
+    download = zc.buildout.download.Download(buildout['buildout'],
+                hash_name=True)
+    path, is_temp = download(options.pop('url'),
+        md5sum=options.get('md5sum'))
+    options['input'] = path
+    collective.recipe.template.Recipe.__init__(self, buildout, name, options)