Import base recipes.

slapos/slapos.lib.recipe/CHANGES.txt

+Changes
+=======
+
+1.0 (unreleased)
+----------------

slapos/slapos.lib.recipe/README.txt

+Library for creating SlapOS recipes for software instantiation
+==============================================================
+
+Thanks to using slapos.lib.recipe it is easier to create zc.buildout recipes in SlapOS environment.
+
+How to use?
+-----------
+
+In setup.py of recipe add only one install requires to slap.lib.recipe.
+
+In code itself subclass from slap.lib.recipe.BaseSlapRecipe.BaseSlapRecipe.
+
+Use _install hook:
+
+::
+
+  from slap.lib.recipe.BaseSlapRecipe import BaseSlapRecipe
+
+  class Recipe(BaseSlapRecipe):
+    ...
+    def _install(self):
+      # refer below for list of available objects
+      specific code
+      of recipe
+
+Available variables self.:
+
+ * name and options passed by zc.buildout during init
+ * work_directory -- buildout's directory
+ * bin_directory -- places for generated binaries
+ * running_wrapper_location -- filename of wrapper to create
+ * data_root_directory -- directory container for data -- inside this
+   directory it is advised to create named directories for provided servers
+   which needs data
+ * backup_directory -- directory container for backups -- inside this
+   directory it is advised o created named directories for backups, with same
+   structure as in data_root_directory
+ * var_directory -- container for various, unix following things:
+
+   * log_directory -- container for logs
+   * run_directory -- container for pidfiles and sockets
+
+ * etc_directory -- place to put named files and directories of configuration
+   for provided servers
+ * computer_id -- id of computer
+ * computer_partition_id -- if of computer partition
+ * server_url - url of Vifib server
+ * software_release_url -- url of software release being instantiated
+ * slap -- initialised connection to Vifib server
+ * computer_partition -- initialised connection to computer partition
+ * request -- shortcut to computer partition request method
+
+By default all directories are created before calling _install hook.
+
+_install method shall return list of paths which are safe to be removed by
+buildout during part uninstallation.
+
+Important assumptions
+---------------------
+
+Because in SlapOS environment zc.buildout does not know when data are changed,
+recipes shall be always uninstalled/installed. This is done during constructing
+recipe instance which subclasses from BaseSlapRecipe.

slapos/slapos.lib.recipe/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.lib.recipe/setup.py

+from setuptools import setup, find_packages
+
+version = '1.0'
+name = 'slapos.lib.recipe'
+long_description = open("README.txt").read() + "\n" + \
+    open("CHANGES.txt").read()
+
+setup(name=name,
+      version=version,
+      description="Library, helpers and superclass for SlapOS zc.buildout recipes",
+      long_description=long_description,
+      classifiers=[
+          "Framework :: Buildout",
+          "Programming Language :: Python",
+        ],
+      keywords='slap librecipe',
+      license='GPLv3',
+      namespace_packages=['slapos'],
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      include_package_data=True,
+      install_requires=[
+        'netaddr', # to manipulate on IP addresses
+        'setuptools', # namespaces
+        'slapos.slap', # uses internally
+        'zc.buildout', # plays with buildout
+        'zc.recipe.egg', # for scripts generation
+        ],
+      zip_safe=True,
+    )

slapos/slapos.lib.recipe/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

slapos/slapos.lib.recipe/src/slapos/lib/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

slapos/slapos.lib.recipe/src/slapos/lib/recipe/BaseSlapRecipe.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import logging
+from slapos import slap
+import os
+import zc.buildout
+import zc.recipe.egg
+from hashlib import md5
+import stat
+import netaddr
+import time
+
+class BaseSlapRecipe:
+  """Base class for all slap.recipe.*"""
+  def __init__(self, buildout, name, options):
+    """Default initialisation"""
+    self.name = name
+    self.options = options
+    self.logger = logging.getLogger(self.name)
+    self.slap = slap.slap()
+    self.work_directory = os.path.abspath(buildout['buildout'][
+      'directory'])
+    self.bin_directory = os.path.join(buildout['buildout'][
+      'directory'], 'bin')
+    self.data_root_directory = os.path.join(self.work_directory, 'srv')
+    self.backup_directory = os.path.join(self.data_root_directory, 'backup')
+    self.var_directory = os.path.join(self.work_directory, 'var')
+    self.log_directory = os.path.join(self.var_directory, 'log')
+    self.run_directory = os.path.join(self.var_directory, 'run')
+    self.etc_directory = os.path.join(self.work_directory, 'etc')
+    self.tmp_directory = os.path.join(self.work_directory, 'tmp')
+    self.wrapper_directory = os.path.join(self.etc_directory, 'run')
+    self.wrapper_report_directory = os.path.join(self.etc_directory, 'report')
+    self.wrapper_xml_report_directory = os.path.join(self.var_directory,
+        'xml_report')
+    self.destroy_script_location = os.path.join(self, self.work_directory,
+        'sbin', 'destroy')
+
+    # default directory structure information
+    self.default_directory_list = [
+      self.bin_directory, # CP/bin - instance own binaries
+      os.path.join(self, self.work_directory, 'sbin'), # CP/sbin - system
+       # binaries, not exposed, only CP/sbin/destroy
+      self.data_root_directory, # CP/srv - data container
+      self.backup_directory, # CP/srv/backup - backup container
+      self.etc_directory, # CP/etc - configuration container
+      self.wrapper_directory, # CP/etc/run - for wrappers
+      self.wrapper_report_directory, # CP/etc/report - for report wrappers
+      self.var_directory, # CP/var - partition "internal" container for logs,
+                          # and another metadata
+      self.wrapper_xml_report_directory, # CP/var/xml_report - for xml_report wrappers
+      self.log_directory, # CP/var/log - log container
+      self.run_directory, # CP/var/run - working container - pids, sockets
+      self.tmp_directory, # CP/tmp - temporary files
+    ]
+
+    # SLAP related information
+    slap_connection = buildout['slap_connection']
+    self.computer_id=slap_connection['computer_id']
+    self.computer_partition_id=slap_connection['partition_id']
+    self.server_url=slap_connection['server_url']
+    self.software_release_url=slap_connection['software_release_url']
+    self.key_file=slap_connection.get('key_file')
+    self.cert_file=slap_connection.get('cert_file')
+
+    # setup egg to give possibility to generate scripts
+    self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
+
+    # setup auto uninstall/install
+    self._setupAutoInstallUninstall()
+
+  def _setupAutoInstallUninstall(self):
+    """By default SlapOS recipes are reinstalled each time"""
+    # Note: It is possible to create in future subclass which will do no-op in
+    # this method
+    self.options['slapos_timestamp'] = str(time.time())
+
+  def _getIpAddress(self, test_method):
+    """Internal helper method to fetch ip address"""
+    if not 'ip_list' in self.parameter_dict:
+      raise AttributeError
+    for name, ip in self.parameter_dict['ip_list']:
+      if test_method(ip):
+        return ip
+    raise AttributeError
+
+  def getLocalIPv4Address(self):
+    """Returns local IPv4 address available on partition"""
+    # XXX: Lack checking for locality of address
+    return self._getIpAddress(netaddr.valid_ipv4)
+
+  def getGlobalIPv6Address(self):
+    """Returns global IPv6 address available on partition"""
+    # XXX: Lack checking for globality of address
+    return self._getIpAddress(netaddr.valid_ipv6)
+
+  def createConfigurationFile(self, name, content):
+    """Creates named configuration file and returns its path"""
+    file_path = os.path.join(self.etc_directory, name)
+    self._writeFile(file_path, content)
+    self.logger.debug('Created configuration file: %r' % file_path)
+    return file_path
+
+  def createRunningWrapper(self, wrapper_name, file_content):
+    """Creates named running wrapper and returns its path"""
+    wrapper_path = os.path.join(self.wrapper_directory, wrapper_name)
+    self._writeExecutable(wrapper_path, file_content)
+    return wrapper_path
+
+  def createReportRunningWrapper(self, file_content):
+    """Creates report runnig wrapper and returns its path"""
+    report_wrapper_path = os.path.join(self.wrapper_report_directory,
+        'slapreport')
+    self._writeExecutable(report_wrapper_path, file_content)
+    return report_wrapper_path
+
+  def substituteTemplate(self, template_location, mapping_dict):
+    """Returns template content after substitution"""
+    return open(template_location, 'r').read() % mapping_dict
+
+  def _writeExecutable(self, path, content, mode='0700'):
+    """Creates file in path with content and sets mode
+
+    If file was created or altered returns true
+    Otherwise returns false
+
+    To be used to create executables
+
+    Raises os related errors"""
+    return self._writeFile(path, content, mode)
+
+  def _writeFile(self, path, content, mode='0600'):
+    """Creates file in path with content and sets mode
+
+    If file was created or altered returns true
+    Otherwise returns false
+
+    Raises os related errors"""
+
+    file_altered = False
+    if not os.path.exists(path):
+      open(path, 'w').write(content)
+      file_altered = True
+    else:
+      new_sum = md5()
+      current_sum = md5()
+      new_sum.update(content)
+      current_sum.update(open(path, 'r').read())
+      if new_sum.digest() != current_sum.digest():
+        file_altered = True
+        open(path, 'w').write(content)
+
+    if oct(stat.S_IMODE(os.stat(path).st_mode)) != mode:
+      os.chmod(path, int(mode, 8))
+      file_altered = True
+
+    return file_altered
+
+  def createBackupDirectory(self, name, mode='0700'):
+    """Creates named directory in self.backup_directory and returns its path"""
+    path = os.path.join(self.backup_directory, name)
+    self._createDirectory(path, mode)
+    return path
+
+  def createDataDirectory(self, name, mode='0700'):
+    """Creates named directory in self.data_root_directory and returns its path"""
+    path = os.path.join(self.data_root_directory, name)
+    self._createDirectory(path, mode)
+    return path
+
+  def _createDirectory(self, path, mode='0700'):
+    """Creates path directory and sets mode
+
+    If directory was created or its mode was altered returns true
+    Otherwise returns false
+
+    Raises os related errors"""
+    directory_altered = False
+    if not os.path.exists(path):
+      os.mkdir(path, int(mode, 8))
+      directory_altered = True
+    if not os.path.isdir(path):
+      raise zc.buildout.UserError('Path %r exists, but it is not directory'
+          % path)
+    if oct(stat.S_IMODE(os.stat(path).st_mode)) != mode:
+      os.chmod(path, int(mode, 8))
+      directory_altered = True
+    if directory_altered:
+      self.logger.debug('Created directory %r with permission %r' % (path, mode))
+    return directory_altered
+
+  def _createDefaultDirectoryStructure(self):
+    for directory in self.default_directory_list:
+      self._createDirectory(directory)
+
+  def generatePassword(self, len=32):
+    """Generates password. Shall be secured, until then all are insecure"""
+    return 'insecure'
+
+  def install(self):
+    self.slap.initializeConnection(self.server_url, self.key_file,
+        self.cert_file)
+    self.computer_partition = self.slap.registerComputerPartition(
+      self.computer_id,
+      self.computer_partition_id)
+    self.request = self.computer_partition.request
+    self.setConnectionDict = self.computer_partition.setConnectionDict
+    self._createDefaultDirectoryStructure()
+    self.parameter_dict = self.computer_partition.getInstanceParameterDict()
+
+    # call children part of install
+    path_list = self._install()
+
+    return path_list
+
+  update = install
+
+  def _install(self):
+    """Hook which shall be implemented in children class"""
+    raise NotImplementedError('Shall be implemented by subclass')

slapos/slapos.lib.recipe/src/slapos/lib/recipe/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################

slapos/slapos.recipe.build/CHANGES.txt

+1.0 (unreleased)
+----------------
+
+ - initial release [Łukasz Nowak]

slapos/slapos.recipe.build/MANIFEST.in

+include CHANGES.txt

slapos/slapos.recipe.build/README.txt

+slapos.recipe.build
+===================
+
+Recipe to build the software.
+
+Example buildout::
+
+[buildout]
+parts =
+  file
+
+develop = slapos.recipe.build
+
+[zlib]
+# Use standard configure, make, make install way
+recipe = slapos.recipe.build:cmmi
+url = http://prdownloads.sourceforge.net/libpng/zlib-1.2.5.tar.gz?download
+md5sum = c735eab2d659a96e5a594c9e8541ad63
+slapos_promisee =
+  directory:include
+  file:include/zconf.h
+  file:include/zlib.h
+  directory:lib
+  statlib:lib/libz.a
+  dynlib:lib/libz.so linked:libc.so.6 rpath:
+  dynlib:lib/libz.so.1 linked:libc.so.6 rpath:
+  dynlib:lib/libz.so.1.2.5 linked:libc.so.6
+  directory:lib/pkgconfig
+  file:lib/pkgconfig/zlib.pc
+  directory:share
+  directory:share/man
+  directory:share/man/man3
+  file:share/man/man3/zlib.3
+
+[file]
+recipe = slapos.recipe.build:cmmi
+url = ftp://ftp.astron.com/pub/file/file-5.04.tar.gz
+md5sum = accade81ff1cc774904b47c72c8aeea0
+environment =
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib
+slapos_promisee =
+  directory:bin
+  dynlib:bin/file linked:libz.so.1,libc.so.6,libmagic.so.1 rpath:${zlib:location}/lib,!/lib
+  directory:include
+  file:include/magic.h
+  directory:lib
+  statlib:lib/libmagic.a
+  statlib:lib/libmagic.la
+  dynlib:lib/libmagic.so linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
+  dynlib:lib/libmagic.so.1 linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
+  dynlib:lib/libmagic.so.1.0.0 linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
+  directory:share
+  directory:share/man
+  directory:share/man/man1
+  file:share/man/man1/file.1
+  directory:share/man/man3
+  file:share/man/man3/libmagic.3
+  directory:share/man/man4
+  file:share/man/man4/magic.4
+  directory:share/man/man5
+  directory:share/misc
+  file:share/misc/magic.mgc
+
+[somethingelse]
+# default way with using script
+recipe = slapos.recipe.build
+url_0 = http://host/path/file.tar.gz
+md5sum = 9631070eac74f92a812d4785a84d1b4e
+script =
+  import os
+  os.chdir(%(work_directory)s)
+  unpack(%(url_0), strip_path=True)
+  execute('make')
+  execute('make install DEST=%(location)s')
+slapos_promisee =
+  ...
+
+[anythingelse]
+# reusing different recipe?
+recipe = slapos.recipe.build:backend
+backend = hexagonit.recipe.cmmi
+slapos_promisee =
+  ...
+# parameters to build
+
+TODO:
+
+ * add linking suport, buildout definition:
+
+slapos_link = <relative/path> [optional-path
+
+can be used as:
+
+[file]
+slapos_link =
+  bin/file
+  bin/file ${buildout:bin-directory}/bin/anotherfile
+
+Which will link ${file:location}/bin/file to ${buildout:bin-directory}/bin/file
+and ${file:location}/bin/file to ${buildout:bin-directory}/bin/anotherfile

slapos/slapos.recipe.build/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.recipe.build/setup.py

+from setuptools import setup, find_packages
+
+version = '1.0'
+name = 'slapos.recipe.build'
+long_description = open("README.txt").read() + "\n" + \
+    open("CHANGES.txt").read()
+
+setup(name=name,
+      version=version,
+      description="Simple download recipe",
+      long_description=long_description,
+      classifiers=[
+          "Framework :: Buildout :: Recipe",
+          "Programming Language :: Python",
+        ],
+      keywords='slapos recipe build',
+      license='GPLv3',
+      namespace_packages=['slapos'],
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      include_package_data=True,
+      install_requires=[
+        'setuptools', # for namespace and internal usage
+        'zc.buildout', # needed to play internally
+        ],
+      entry_points={'zc.buildout': [
+        'default = %s:Script' % name,
+        'cmmi = %s:Cmmi' % name,
+        ]},
+      zip_safe=True,
+    )

slapos/slapos.recipe.build/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.build/src/slapos/recipe/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.recipe.build/src/slapos/recipe/build/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import logging
+import os
+import setuptools
+import shutil
+import subprocess
+import tempfile
+import zc.buildout
+
+def readElfAsDict(f):
+  """Reads ELF information from file"""
+  popen = subprocess.Popen(['readelf', '-d', f],
+      stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  result = popen.communicate()[0]
+  if popen.returncode != 0:
+    raise AssertionError(result)
+  library_list = []
+  rpath_list = []
+  runpath_list = []
+  for l in result.split('\n'):
+    if '(NEEDED)' in l:
+      library_list.append(l.split(':')[1].strip(' []'))
+    elif '(RPATH)' in l:
+      rpath_list = [q.rstrip('/') for q in l.split(':',1)[1].strip(' []').split(':')]
+    elif '(RUNPATH)' in l:
+      runpath_list = [q.rstrip('/') for q in l.split(':',1)[1].strip(' []').split(':')]
+  if len(runpath_list) == 0:
+    runpath_list = rpath_list
+  elif len(rpath_list) != 0 and runpath_list != rpath_list:
+    raise ValueError('RPATH and RUNPATH are different.')
+  return dict(
+    library_list=sorted(library_list),
+    runpath_list=sorted(runpath_list)
+  )
+
+def call(*args, **kwargs):
+  """Subprocess call with closed file descriptors and stdin"""
+  kwargs.update(
+    stdin=subprocess.PIPE,
+    close_fds=True)
+  popen = subprocess.Popen(*args, **kwargs)
+  popen.stdin.flush()
+  popen.stdin.close()
+  popen.stdin = None
+  popen.communicate()
+  if popen.returncode != 0:
+    raise subprocess.CalledProcessError(popen.returncode, ' '.join(args[0]))
+
+def calls(call_string, **kwargs):
+  """Subprocesser caller which allows to pass arguments as string"""
+  call(call_string.split(), **kwargs)
+
+def guessworkdir(path):
+  if len(os.listdir(path)) == 1:
+    return os.path.join(path, os.listdir(path)[0])
+  return path
+
+class Script:
+  """Free script building system"""
+  def _checkPromisee(self, location):
+    promisee_problem_list = []
+    a = promisee_problem_list.append
+    for promisee in self.options['slapos_promisee'].split('\n'):
+      promisee = promisee.strip()
+      if not promisee:
+        continue
+      if promisee.startswith('file:') or promisee.startswith('statlib'):
+        s, path = promisee.split(':')
+        if not os.path.exists(os.path.join(location, path)):
+          a('File promisee not met for %r' % path)
+      elif promisee.startswith('directory'):
+        s, path = promisee.split(':')
+        if not os.path.isdir(os.path.join(location, path)):
+          a('Directory promisee not met for %r' %
+              path)
+      elif promisee.startswith('dynlib:'):
+        if 'linked:' not in promisee:
+          raise zc.buildout.UserError('dynlib promisee requires \'linked:\' '
+            'parameter.')
+        if 'rpath:' not in promisee:
+          rpath_list = []
+        for promisee_part in promisee.split():
+          if promisee_part.startswith('dynlib:'):
+            s, path = promisee_part.split(':')
+          elif promisee_part.startswith('linked:'):
+            s, link_list = promisee_part.split(':')
+            link_list = link_list.split(',')
+          elif promisee_part.startswith('rpath:'):
+            s, rpath_list = promisee_part.split(':')
+            if rpath_list:
+              r = rpath_list
+              rpath_list = []
+              for q in r.split(','):
+                if q.startswith('!'):
+                  q = q.replace('!', location)
+                rpath_list.append(q)
+            else:
+              rpath_list = []
+        if not os.path.exists(os.