Commit ff17fae0 authored by Łukasz Nowak's avatar Łukasz Nowak

Import base recipes.

parent 31529364
Changes
=======
1.0 (unreleased)
----------------
Library for creating SlapOS recipes for software instantiation
==============================================================
Thanks to using slapos.lib.recipe it is easier to create zc.buildout recipes in SlapOS environment.
How to use?
-----------
In setup.py of recipe add only one install requires to slap.lib.recipe.
In code itself subclass from slap.lib.recipe.BaseSlapRecipe.BaseSlapRecipe.
Use _install hook:
::
from slap.lib.recipe.BaseSlapRecipe import BaseSlapRecipe
class Recipe(BaseSlapRecipe):
...
def _install(self):
# refer below for list of available objects
specific code
of recipe
Available variables self.:
* name and options passed by zc.buildout during init
* work_directory -- buildout's directory
* bin_directory -- places for generated binaries
* running_wrapper_location -- filename of wrapper to create
* data_root_directory -- directory container for data -- inside this
directory it is advised to create named directories for provided servers
which needs data
* backup_directory -- directory container for backups -- inside this
directory it is advised o created named directories for backups, with same
structure as in data_root_directory
* var_directory -- container for various, unix following things:
* log_directory -- container for logs
* run_directory -- container for pidfiles and sockets
* etc_directory -- place to put named files and directories of configuration
for provided servers
* computer_id -- id of computer
* computer_partition_id -- if of computer partition
* server_url - url of Vifib server
* software_release_url -- url of software release being instantiated
* slap -- initialised connection to Vifib server
* computer_partition -- initialised connection to computer partition
* request -- shortcut to computer partition request method
By default all directories are created before calling _install hook.
_install method shall return list of paths which are safe to be removed by
buildout during part uninstallation.
Important assumptions
---------------------
Because in SlapOS environment zc.buildout does not know when data are changed,
recipes shall be always uninstalled/installed. This is done during constructing
recipe instance which subclasses from BaseSlapRecipe.
[egg_info]
tag_build = .dev
tag_svn_revision = 1
from setuptools import setup, find_packages
version = '1.0'
name = 'slapos.lib.recipe'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read()
setup(name=name,
version=version,
description="Library, helpers and superclass for SlapOS zc.buildout recipes",
long_description=long_description,
classifiers=[
"Framework :: Buildout",
"Programming Language :: Python",
],
keywords='slap librecipe',
license='GPLv3',
namespace_packages=['slapos'],
packages=find_packages('src'),
package_dir={'': 'src'},
include_package_data=True,
install_requires=[
'netaddr', # to manipulate on IP addresses
'setuptools', # namespaces
'slapos.slap', # uses internally
'zc.buildout', # plays with buildout
'zc.recipe.egg', # for scripts generation
],
zip_safe=True,
)
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import logging
from slapos import slap
import os
import zc.buildout
import zc.recipe.egg
from hashlib import md5
import stat
import netaddr
import time
class BaseSlapRecipe:
"""Base class for all slap.recipe.*"""
def __init__(self, buildout, name, options):
"""Default initialisation"""
self.name = name
self.options = options
self.logger = logging.getLogger(self.name)
self.slap = slap.slap()
self.work_directory = os.path.abspath(buildout['buildout'][
'directory'])
self.bin_directory = os.path.join(buildout['buildout'][
'directory'], 'bin')
self.data_root_directory = os.path.join(self.work_directory, 'srv')
self.backup_directory = os.path.join(self.data_root_directory, 'backup')
self.var_directory = os.path.join(self.work_directory, 'var')
self.log_directory = os.path.join(self.var_directory, 'log')
self.run_directory = os.path.join(self.var_directory, 'run')
self.etc_directory = os.path.join(self.work_directory, 'etc')
self.tmp_directory = os.path.join(self.work_directory, 'tmp')
self.wrapper_directory = os.path.join(self.etc_directory, 'run')
self.wrapper_report_directory = os.path.join(self.etc_directory, 'report')
self.wrapper_xml_report_directory = os.path.join(self.var_directory,
'xml_report')
self.destroy_script_location = os.path.join(self, self.work_directory,
'sbin', 'destroy')
# default directory structure information
self.default_directory_list = [
self.bin_directory, # CP/bin - instance own binaries
os.path.join(self, self.work_directory, 'sbin'), # CP/sbin - system
# binaries, not exposed, only CP/sbin/destroy
self.data_root_directory, # CP/srv - data container
self.backup_directory, # CP/srv/backup - backup container
self.etc_directory, # CP/etc - configuration container
self.wrapper_directory, # CP/etc/run - for wrappers
self.wrapper_report_directory, # CP/etc/report - for report wrappers
self.var_directory, # CP/var - partition "internal" container for logs,
# and another metadata
self.wrapper_xml_report_directory, # CP/var/xml_report - for xml_report wrappers
self.log_directory, # CP/var/log - log container
self.run_directory, # CP/var/run - working container - pids, sockets
self.tmp_directory, # CP/tmp - temporary files
]
# SLAP related information
slap_connection = buildout['slap_connection']
self.computer_id=slap_connection['computer_id']
self.computer_partition_id=slap_connection['partition_id']
self.server_url=slap_connection['server_url']
self.software_release_url=slap_connection['software_release_url']
self.key_file=slap_connection.get('key_file')
self.cert_file=slap_connection.get('cert_file')
# setup egg to give possibility to generate scripts
self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
# setup auto uninstall/install
self._setupAutoInstallUninstall()
def _setupAutoInstallUninstall(self):
"""By default SlapOS recipes are reinstalled each time"""
# Note: It is possible to create in future subclass which will do no-op in
# this method
self.options['slapos_timestamp'] = str(time.time())
def _getIpAddress(self, test_method):
"""Internal helper method to fetch ip address"""
if not 'ip_list' in self.parameter_dict:
raise AttributeError
for name, ip in self.parameter_dict['ip_list']:
if test_method(ip):
return ip
raise AttributeError
def getLocalIPv4Address(self):
"""Returns local IPv4 address available on partition"""
# XXX: Lack checking for locality of address
return self._getIpAddress(netaddr.valid_ipv4)
def getGlobalIPv6Address(self):
"""Returns global IPv6 address available on partition"""
# XXX: Lack checking for globality of address
return self._getIpAddress(netaddr.valid_ipv6)
def createConfigurationFile(self, name, content):
"""Creates named configuration file and returns its path"""
file_path = os.path.join(self.etc_directory, name)
self._writeFile(file_path, content)
self.logger.debug('Created configuration file: %r' % file_path)
return file_path
def createRunningWrapper(self, wrapper_name, file_content):
"""Creates named running wrapper and returns its path"""
wrapper_path = os.path.join(self.wrapper_directory, wrapper_name)
self._writeExecutable(wrapper_path, file_content)
return wrapper_path
def createReportRunningWrapper(self, file_content):
"""Creates report runnig wrapper and returns its path"""
report_wrapper_path = os.path.join(self.wrapper_report_directory,
'slapreport')
self._writeExecutable(report_wrapper_path, file_content)
return report_wrapper_path
def substituteTemplate(self, template_location, mapping_dict):
"""Returns template content after substitution"""
return open(template_location, 'r').read() % mapping_dict
def _writeExecutable(self, path, content, mode='0700'):
"""Creates file in path with content and sets mode
If file was created or altered returns true
Otherwise returns false
To be used to create executables
Raises os related errors"""
return self._writeFile(path, content, mode)
def _writeFile(self, path, content, mode='0600'):
"""Creates file in path with content and sets mode
If file was created or altered returns true
Otherwise returns false
Raises os related errors"""
file_altered = False
if not os.path.exists(path):
open(path, 'w').write(content)
file_altered = True
else:
new_sum = md5()
current_sum = md5()
new_sum.update(content)
current_sum.update(open(path, 'r').read())
if new_sum.digest() != current_sum.digest():
file_altered = True
open(path, 'w').write(content)
if oct(stat.S_IMODE(os.stat(path).st_mode)) != mode:
os.chmod(path, int(mode, 8))
file_altered = True
return file_altered
def createBackupDirectory(self, name, mode='0700'):
"""Creates named directory in self.backup_directory and returns its path"""
path = os.path.join(self.backup_directory, name)
self._createDirectory(path, mode)
return path
def createDataDirectory(self, name, mode='0700'):
"""Creates named directory in self.data_root_directory and returns its path"""
path = os.path.join(self.data_root_directory, name)
self._createDirectory(path, mode)
return path
def _createDirectory(self, path, mode='0700'):
"""Creates path directory and sets mode
If directory was created or its mode was altered returns true
Otherwise returns false
Raises os related errors"""
directory_altered = False
if not os.path.exists(path):
os.mkdir(path, int(mode, 8))
directory_altered = True
if not os.path.isdir(path):
raise zc.buildout.UserError('Path %r exists, but it is not directory'
% path)
if oct(stat.S_IMODE(os.stat(path).st_mode)) != mode:
os.chmod(path, int(mode, 8))
directory_altered = True
if directory_altered:
self.logger.debug('Created directory %r with permission %r' % (path, mode))
return directory_altered
def _createDefaultDirectoryStructure(self):
for directory in self.default_directory_list:
self._createDirectory(directory)
def generatePassword(self, len=32):
"""Generates password. Shall be secured, until then all are insecure"""
return 'insecure'
def install(self):
self.slap.initializeConnection(self.server_url, self.key_file,
self.cert_file)
self.computer_partition = self.slap.registerComputerPartition(
self.computer_id,
self.computer_partition_id)
self.request = self.computer_partition.request
self.setConnectionDict = self.computer_partition.setConnectionDict
self._createDefaultDirectoryStructure()
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
# call children part of install
path_list = self._install()
return path_list
update = install
def _install(self):
"""Hook which shall be implemented in children class"""
raise NotImplementedError('Shall be implemented by subclass')
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
1.0 (unreleased)
----------------
- initial release [Łukasz Nowak]
slapos.recipe.build
===================
Recipe to build the software.
Example buildout::
[buildout]
parts =
file
develop = slapos.recipe.build
[zlib]
# Use standard configure, make, make install way
recipe = slapos.recipe.build:cmmi
url = http://prdownloads.sourceforge.net/libpng/zlib-1.2.5.tar.gz?download
md5sum = c735eab2d659a96e5a594c9e8541ad63
slapos_promisee =
directory:include
file:include/zconf.h
file:include/zlib.h
directory:lib
statlib:lib/libz.a
dynlib:lib/libz.so linked:libc.so.6 rpath:
dynlib:lib/libz.so.1 linked:libc.so.6 rpath:
dynlib:lib/libz.so.1.2.5 linked:libc.so.6
directory:lib/pkgconfig
file:lib/pkgconfig/zlib.pc
directory:share
directory:share/man
directory:share/man/man3
file:share/man/man3/zlib.3
[file]
recipe = slapos.recipe.build:cmmi
url = ftp://ftp.astron.com/pub/file/file-5.04.tar.gz
md5sum = accade81ff1cc774904b47c72c8aeea0
environment =
CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib
slapos_promisee =
directory:bin
dynlib:bin/file linked:libz.so.1,libc.so.6,libmagic.so.1 rpath:${zlib:location}/lib,!/lib
directory:include
file:include/magic.h
directory:lib
statlib:lib/libmagic.a
statlib:lib/libmagic.la
dynlib:lib/libmagic.so linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
dynlib:lib/libmagic.so.1 linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
dynlib:lib/libmagic.so.1.0.0 linked:libz.so.1,libc.so.6 rpath:${zlib:location}/lib
directory:share
directory:share/man
directory:share/man/man1
file:share/man/man1/file.1
directory:share/man/man3
file:share/man/man3/libmagic.3
directory:share/man/man4
file:share/man/man4/magic.4
directory:share/man/man5
directory:share/misc
file:share/misc/magic.mgc
[somethingelse]
# default way with using script
recipe = slapos.recipe.build
url_0 = http://host/path/file.tar.gz
md5sum = 9631070eac74f92a812d4785a84d1b4e
script =
import os
os.chdir(%(work_directory)s)
unpack(%(url_0), strip_path=True)
execute('make')
execute('make install DEST=%(location)s')
slapos_promisee =
...
[anythingelse]
# reusing different recipe?
recipe = slapos.recipe.build:backend
backend = hexagonit.recipe.cmmi
slapos_promisee =
...
# parameters to build
TODO:
* add linking suport, buildout definition:
slapos_link = <relative/path> [optional-path
can be used as:
[file]
slapos_link =
bin/file
bin/file ${buildout:bin-directory}/bin/anotherfile
Which will link ${file:location}/bin/file to ${buildout:bin-directory}/bin/file
and ${file:location}/bin/file to ${buildout:bin-directory}/bin/anotherfile
[egg_info]
tag_build = .dev
tag_svn_revision = 1
from setuptools import setup, find_packages
version = '1.0'
name = 'slapos.recipe.build'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read()
setup(name=name,
version=version,
description="Simple download recipe",
long_description=long_description,
classifiers=[
"Framework :: Buildout :: Recipe",
"Programming Language :: Python",
],
keywords='slapos recipe build',
license='GPLv3',
namespace_packages=['slapos'],
packages=find_packages('src'),
package_dir={'': 'src'},
include_package_data=True,
install_requires=[
'setuptools', # for namespace and internal usage
'zc.buildout', # needed to play internally
],
entry_points={'zc.buildout': [
'default = %s:Script' % name,
'cmmi = %s:Cmmi' % name,
]},
zip_safe=True,
)
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import logging
import os
import setuptools
import shutil
import subprocess
import tempfile
import zc.buildout
def readElfAsDict(f):
"""Reads ELF information from file"""
popen = subprocess.Popen(['readelf', '-d', f],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = popen.communicate()[0]
if popen.returncode != 0:
raise AssertionError(result)
library_list = []
rpath_list = []
runpath_list = []
for l in result.split('\n'):
if '(NEEDED)' in l:
library_list.append(l.split(':')[1].strip(' []'))
elif '(RPATH)' in l:
rpath_list = [q.rstrip('/') for q in l.split(':',1)[1].strip(' []').split(':')]
elif '(RUNPATH)' in l:
runpath_list = [q.rstrip('/') for q in l.split(':',1)[1].strip(' []').split(':')]
if len(runpath_list) == 0:
runpath_list = rpath_list
elif len(rpath_list) != 0 and runpath_list != rpath_list:
raise ValueError('RPATH and RUNPATH are different.')
return dict(
library_list=sorted(library_list),
runpath_list=sorted(runpath_list)
)
def call(*args, **kwargs):
"""Subprocess call with closed file descriptors and stdin"""
kwargs.update(
stdin=subprocess.PIPE,
close_fds=True)
popen = subprocess.Popen(*args, **kwargs)
popen.stdin.flush()
popen.stdin.close()
popen.stdin = None
popen.communicate()
if popen.returncode != 0:
raise subprocess.CalledProcessError(popen.returncode, ' '.join(args[0]))
def calls(call_string, **kwargs):
"""Subprocesser caller which allows to pass arguments as string"""
call(call_string.split(), **kwargs)
def guessworkdir(path):
if len(os.listdir(path)) == 1:
return os.path.join(path, os.listdir(path)[0])
return path
class Script:
"""Free script building system"""
def _checkPromisee(self, location):
promisee_problem_list = []
a = promisee_problem_list.append
for promisee in self.options['slapos_promisee'].split('\n'):
promisee = promisee.strip()
if not promisee:
continue
if promisee.startswith('file:') or promisee.startswith('statlib'):
s, path = promisee.split(':')
if not os.path.exists(os.path.join(location, path)):
a('File promisee not met for %r' % path)
elif promisee.startswith('directory'):
s, path = promisee.split(':')
if not os.path.isdir(os.path.join(location, path)):
a('Directory promisee not met for %r' %
path)
elif promisee.startswith('dynlib:'):
if 'linked:' not in promisee:
raise zc.buildout.UserError('dynlib promisee requires \'linked:\' '
'parameter.')
if 'rpath:' not in promisee:
rpath_list = []
for promisee_part in promisee.split():
if promisee_part.startswith('dynlib:'):
s, path = promisee_part.split(':')
elif promisee_part.startswith('linked:'):
s, link_list = promisee_part.split(':')
link_list = link_list.split(',')
elif promisee_part.startswith('rpath:'):
s, rpath_list = promisee_part.split(':')
if rpath_list:
r = rpath_list
rpath_list = []
for q in r.split(','):
if q.startswith('!'):
q = q.replace('!', location)
rpath_list.append(q)
else:
rpath_list = []
if not os.path.exists(os.path.join(location, path)):