diff --git a/component/apache-perl/buildout.cfg b/component/apache-perl/buildout.cfg new file mode 100644 index 0000000000000000000000000000000000000000..3165218eb55a44a7c8456a2f3dab04b8314f80a1 --- /dev/null +++ b/component/apache-perl/buildout.cfg @@ -0,0 +1,21 @@ +[buildout] +parts = apache-perl + +extends = + ../apache/buildout.cfg + ../perl/buildout.cfg + ../libuuid/buildout.cfg + +[apache-perl] +# Note: Shall react on each build of apache and reinstall itself +recipe = hexagonit.recipe.cmmi +url = http://perl.apache.org/dist/mod_perl-2.0.5.tar.gz +md5sum = 03d01d135a122bd8cebd0cd5b185d674 +configure-command = + ${perl:location}/bin/perl Makefile.PL +configure-options = + MP_AP_PREFIX=${apache-2.2:location} + +environment = + # CPPFLAGS=-I${libuuid:location}/include + MP_CCOPTS=-L${libuuid:location}/lib diff --git a/component/apache-php/buildout.cfg b/component/apache-php/buildout.cfg index b9d146958fe012faa658b868c5c959eeb51ff509..b7bb2d0c88d87dadc3de48006c62947c209e4a12 100644 --- a/component/apache-php/buildout.cfg +++ b/component/apache-php/buildout.cfg @@ -55,10 +55,50 @@ configure-options = --enable-bz2 --enable-ftp + +# Changing TMPDIR is required for PEAR installation. +# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp. +# XXX we could mkdir tmp there + environment = PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid + TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_} + + + + +[apache-php-postgres] +<=apache-php +configure-options = + --with-apxs2=${apache:location}/bin/apxs + --with-libxml-dir=${libxml2:location} + --with-zlib-dir=${zlib:location} + --with-bz2-dir=${bzip2:location} + --with-mcrypt=${libmcrypt:location} + --with-gd + --with-jpeg-dir=${libjpeg:location} + --with-png-dir=${libpng:location} + --enable-gd-native-ttf + --with-ttf + --with-freetype-dir=${freetype:location} + --with-curl=${curl:location} + --with-zip-dir=${zip:location} + --with-imap=${cclient:location} + --with-iconv-dir=${libiconv:location} + --with-gettext=${gettext:location} + --with-ldap=${openldap:location} + --with-imap-ssl + --with-openssl=${openssl:location} + --enable-libxml + --enable-mbstring + --enable-session + --enable-exif + --enable-zip + --enable-bz2 + --enable-ftp + --with-pgsql=${postgresql:location} [libmcrypt] diff --git a/component/postgresql/buildout.cfg b/component/postgresql/buildout.cfg new file mode 100644 index 0000000000000000000000000000000000000000..e1fb112c1a0502371d8bb1ec1635feac32e929fd --- /dev/null +++ b/component/postgresql/buildout.cfg @@ -0,0 +1,16 @@ +[buildout] +extends = + ../openssl/buildout.cfg + ../readline/buildout.cfg + ../zlib/buildout.cfg + ../ncurses/buildout.cfg +parts = postgresql + +[postgresql] +recipe = hexagonit.recipe.cmmi +url = http://ftp.postgresql.org/pub/source/v9.1.6/postgresql-9.1.6.tar.bz2 +md5sum = 000755f66c0de58bbd4cd2b89b45b8e2 +configure-options = --with-openssl +environment = + CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib + LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib diff --git a/setup.py b/setup.py index 19deaea4e05c54b4d2534277f4e8d37022c6861a..90816e1bfaea8b988785787af04a981614faea89 100755 --- a/setup.py +++ b/setup.py @@ -110,6 +110,9 @@ setup(name=name, 'novnc = slapos.recipe.novnc:Recipe', 'onetimeupload = slapos.recipe.onetimeupload:Recipe', 'pbs = slapos.recipe.pbs:Recipe', + 'postgres = slapos.recipe.postgres:Recipe', + 'postgres.import = slapos.recipe.postgres:ImportRecipe', + 'postgres.export = slapos.recipe.postgres:ExportRecipe', 'proactive = slapos.recipe.proactive:Recipe', 'publish = slapos.recipe.publish:Recipe', 'publishurl = slapos.recipe.publishurl:Recipe', @@ -126,7 +129,8 @@ setup(name=name, 'siptester = slapos.recipe.siptester:SipTesterRecipe', 'slapconfiguration = slapos.recipe.slapconfiguration:Recipe', 'slapcontainer = slapos.recipe.container:Recipe', - 'slapmonitor = slapos.recipe.slapmonitor:Recipe', + 'slapmonitor = slapos.recipe.slapmonitor:MonitorRecipe', + 'slapmonitor-xml = slapos.recipe.slapmonitor:MonitorXMLRecipe', 'slapreport = slapos.recipe.slapreport:Recipe', 'slaprunner = slapos.recipe.slaprunner:Recipe', 'slaprunner.test = slapos.recipe.slaprunner:Test', diff --git a/slapos/recipe/addresiliency/__init__.py b/slapos/recipe/addresiliency/__init__.py index 69872e21453d6b943f49835f3b744509e88add8b..45ecd8162ab9716c398364594722442ea33af2d3 100644 --- a/slapos/recipe/addresiliency/__init__.py +++ b/slapos/recipe/addresiliency/__init__.py @@ -26,56 +26,73 @@ ############################################################################## from slapos.recipe.librecipe import GenericSlapRecipe -import sys import os class Recipe(GenericSlapRecipe): - """ This class provides the installation of the resilience - script on the partition. - """ - - def _install(self): - path_list = [] - self_id = int(self.parameter_dict['number']) - ip = self.parameter_dict['ip-list'].split(' ') - print 'Creating bully script with ips : %s\n' % ip - slap_connection = self.buildout['slap-connection'] - - path_conf = os.path.join(self.options['script'], 'conf.in') - path_bully = os.path.join(self.options['script'], self.parameter_dict['script']) - path_bully_new = os.path.join(self.options['script'], 'new.py') - path_run = os.path.join(self.options['run'], self.parameter_dict['wrapper']) - print 'paths: %s\n%s\n' % (path_run, path_bully) - bully_conf = dict(self_id=self_id, - ip_list=ip, - executable=sys.executable, - syspath=sys.path, - server_url=slap_connection['server-url'], - key_file=slap_connection.get('key-file'), - cert_file=slap_connection.get('cert-file'), - computer_id=slap_connection['computer-id'], - partition_id=slap_connection['partition-id'], - software=slap_connection['software-release-url'], - namebase=self.parameter_dict['namebase'], - confpath=path_conf) - try: - conf = self.createFile(path_conf, - self.substituteTemplate( - self.getTemplateFilename('conf.in.in'), - bully_conf)) - path_list.append(conf) - script = self.createExecutable(path_bully, - self.substituteTemplate( - self.getTemplateFilename('bully.py.in'), - bully_conf)) - path_list.append(script) - - wrapper = self.createPythonScript( - path_run, - 'slapos.recipe.librecipe.execute.execute', - [path_bully]) - path_list.append(wrapper) - except IOError: - pass - return path_list + """ This class provides the installation of the resilience + scripts on the partition. + + bin/takeover will perform a rename (must be run manually). + bin/bully will monitor, run elections and perform renames when needed. + """ + + def _install(self): + path_list = [] + + confpath = os.path.join(self.options['etc'], 'bully.conf') + + ip_list = self.parameter_dict['ip-list'] + print 'Creating bully configuration with ips : %s\n' % ip_list + + conf = self.createFile(confpath, + self.substituteTemplate( + self.getTemplateFilename('bully.conf.in'), + { + 'self_id': int(self.parameter_dict['number']), + 'ip_list': ip_list + } + )) + path_list.append(conf) + + slap_connection = self.buildout['slap-connection'] + + if self.optionIsTrue('enable-bully-service', default=False): + bully_dir = self.options['services'] + else: + bully_dir = self.options['bin'] + + bully_wrapper = self.createPythonScript( + name=os.path.join(bully_dir, self.options['wrapper-bully']), + absolute_function='slapos.recipe.addresiliency.bully.run', + arguments={ + 'confpath': confpath, + 'server_url': slap_connection['server-url'], + 'key_file': slap_connection.get('key-file'), + 'cert_file': slap_connection.get('cert-file'), + 'computer_id': slap_connection['computer-id'], + 'partition_id': slap_connection['partition-id'], + 'software': slap_connection['software-release-url'], + 'namebase': self.parameter_dict['namebase'], + }) + + path_list.append(bully_wrapper) + + takeover_wrapper = self.createPythonScript( + name=os.path.join(self.options['bin'], self.options['wrapper-takeover']), + absolute_function='slapos.recipe.addresiliency.takeover.run', + arguments={ + 'server_url': slap_connection['server-url'], + 'key_file': slap_connection.get('key-file'), + 'cert_file': slap_connection.get('cert-file'), + 'computer_id': slap_connection['computer-id'], + 'partition_id': slap_connection['partition-id'], + 'software': slap_connection['software-release-url'], + 'namebase': self.parameter_dict['namebase'], + }) + + path_list.append(takeover_wrapper) + + return path_list + + diff --git a/slapos/recipe/addresiliency/bully.py b/slapos/recipe/addresiliency/bully.py new file mode 100644 index 0000000000000000000000000000000000000000..c71f3efca5fadb10aeb84cb2c02a96f59c8b7c37 --- /dev/null +++ b/slapos/recipe/addresiliency/bully.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- + +import logging +import Queue +import socket +import thread +import time + +import slapos.recipe.addresiliency.renamer +import slapos + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + + +BASE_PORT = 50000 +SLEEPING_MINS = 2 # XXX was 30, increase after testing + +MSG_PING = 'ping' +MSG_HALT = 'halt' +MSG_VICTORY = 'victory' + +MSG_OK = 'ok' + +STATE_NORMAL = 'normal' +STATE_WAITINGCONFIRM = 'waitingConfirm' +STATE_ELECTION = 'election' +STATE_REORGANIZATION = 'reorganization' + + + +## Leader is always number 0 + +class ResilientInstance(object): + + def __init__(self, comm, renamer, confpath): + self.comm = comm + self.participant_id = 0 + self.state = STATE_NORMAL + self.halter_id = 0 + self.inElection = False + self.alive = True + + self.mainCanal = self.comm.create_canal([MSG_PING, MSG_HALT, MSG_VICTORY]) + + self.renamer = renamer + self.okCanal = self.comm.create_canal([MSG_OK]) + self.confpath = confpath + self.loadConnectionInfo() + + def loadConnectionInfo(self): + params = open(self.confpath, 'r').readlines() + self.total_participants = len(params[0].split()) + new_id = int(params[1]) + if self.participant_id != new_id: + self.halter_id = new_id + self.participant_id = new_id + log.debug('I am {} of {}'.format(self.participant_id, self.total_participants)) + + ## Needs to be changed to use the master + def aliveManagement(self): + while self.alive: + log.info('XXX sleeping for %d minutes' % SLEEPING_MINS) + time.sleep(SLEEPING_MINS*60) + if self.participant_id == 0: + continue + self.comm.send(MSG_PING, 0) + message, sender = self.okCanal.get() + if message: + continue + self.election() + + def listen(self): + while self.alive: + self.comm.recv() + + def main(self): + while self.alive: + message, sender = self.mainCanal.get() + if message == MSG_PING: + self.comm.send(MSG_OK, sender) + + elif message == MSG_HALT: + self.state = STATE_WAITINGCONFIRM + self.halter_id = int(sender) + self.comm.send(MSG_OK, sender) + + elif message == MSG_VICTORY: + if int(sender) == self.halter_id and self.state == STATE_WAITINGCONFIRM: + log.info('{} thinks {} is the leader'.format(self.participant_id, sender)) + self.comm.send(MSG_OK, sender) + self.state = STATE_NORMAL + + def election(self): + self.inElection = True + self.loadConnectionInfo() + # Check if I'm the highest instance alive + for higher in range(self.participant_id + 1, self.total_participants): + self.comm.send(MSG_PING, higher) + message, sender = self.okCanal.get() + if message: + log.info('{} is alive ({})'.format(higher, self.participant_id)) + self.inElection = False + return False + continue + + if not self.alive: + return False + + # I should be the new coordinator, halt those below me + log.info('Should be ME : {}'.format(self.participant_id)) + self.state = STATE_ELECTION + self.halter_id = self.participant_id + ups = [] + for lower in range(self.participant_id): + self.comm.send(MSG_HALT, lower) + message, sender = self.okCanal.get() + if message: + ups.append(lower) + + #Broadcast Victory + self.state = STATE_REORGANIZATION + for up in ups: + self.comm.send(MSG_VICTORY, up) + message, sender = self.okCanal.get() + if message: + continue + log.info('Something is wrong... let\'s start over') + return self.election() + self.state = STATE_NORMAL + self.active = True + log.info('{} Is THE LEADER'.format(self.participant_id)) + + self.renamer.failover() + + self.inElection = False + + return True + + + +class FilteredCanal(object): + + def __init__(self, accept, timeout): + self.accept = accept + self.queue = Queue.Queue() + self.timeout = timeout + + def append(self, message, sender): + if message in self.accept: + self.queue.put([message, sender]) + + def get(self): + try: + return self.queue.get(timeout=self.timeout) + except Queue.Empty: + return [None, None] + + + +class Wrapper(object): + + def __init__(self, confpath, timeout=20): + self.canals = [] + self.ips = [] + self.participant_id = 0 + self.timeout = timeout + self.confpath = confpath + self.getConnectionInfo() + self.socket = None + + def getConnectionInfo(self): + params = open(self.confpath, 'r').readlines() + self.ips = params[0].split() + self.participant_id = int(params[1]) + log.debug('I am {} of {}'.format(self.participant_id, self.ips)) + + def start(self): + self.getConnectionInfo() + self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self.socket.bind((self.ips[self.participant_id], BASE_PORT + self.participant_id)) + self.socket.listen(5) + + def send(self, message, number): + self.getConnectionInfo() + try: + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + s.connect((self.ips[number], BASE_PORT + number)) + s.send(message + (' {}\n'.format(self.participant_id))) + except (socket.error, socket.herror, socket.gaierror, socket.timeout): + pass + finally: + s.close() + + def create_canal(self, accept): + created = FilteredCanal(accept, self.timeout) + self.canals.append(created) + return created + + def recv(self): + client, _ = self.socket.accept() + client_message = client.recv(1024) + if client_message: + message, sender = client_message.split() + for canal in self.canals: + canal.append(message, int(sender)) + + + + +def run(args): + confpath = args.pop('confpath') + + renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'), + key_file = args.pop('key_file'), + cert_file = args.pop('cert_file'), + computer_guid = args.pop('computer_id'), + partition_id = args.pop('partition_id'), + software_release = args.pop('software'), + namebase = args.pop('namebase')) + + if args: + raise ValueError('Unknown arguments: %s' % ', '.join(args)) + + wrapper = Wrapper(confpath=confpath, timeout=20) + + computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath) + + # idle waiting for connection infos + while computer.total_participants < 2: + computer.loadConnectionInfo() + time.sleep(30) + + log.info('Starting') + + computer.comm.start() + thread.start_new_thread(computer.listen, ()) + thread.start_new_thread(computer.aliveManagement, ()) + + computer.main() + diff --git a/slapos/recipe/addresiliency/renamer.py b/slapos/recipe/addresiliency/renamer.py new file mode 100644 index 0000000000000000000000000000000000000000..563580adbfa5421b6e6e7e5154fd349ce3dae0b2 --- /dev/null +++ b/slapos/recipe/addresiliency/renamer.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +import logging +import time + +import slapos + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + + + +class Renamer(object): + def __init__(self, server_url, key_file, cert_file, computer_guid, + partition_id, software_release, namebase): + self.server_url = server_url + self.key_file = key_file + self.cert_file = cert_file + self.computer_guid = computer_guid + self.partition_id = partition_id + self.software_release = software_release + self.namebase = namebase + + + def _failover(self): + """\ + This method does + + - retrieve the broken computer partition + - change its reference to 'broken-...' and its software type to 'frozen' + - retrieve the winner computer partition (attached to this process) + - change its reference to replace the broken one. + later, slapgrid will change its software_type as well. + + Then, after running slapgrid-cp a few times, the winner takes over and + a new cp is created to replace it as an importer. + """ + + slap = slapos.slap.slap() + slap.initializeConnection(self.server_url, self.key_file, self.cert_file) + + # partition that will take over. + cp_winner = slap.registerComputerPartition(computer_guid=self.computer_guid, + partition_id=self.partition_id) + # XXX although we can already rename cp_winner, to change its software type we need to + # get hold of the root cp as well + + cp_exporter_ref = self.namebase + '0' # this is ok. the boss is always number zero. + + # partition to be deactivated + cp_broken = cp_winner.request(software_release=self.software_release, + software_type='frozen', + state='stopped', + partition_reference=cp_exporter_ref) + + broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime())) + + log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref)) + + cp_broken.rename(new_name=broken_new_ref) + + cp_broken.stopped() + + log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref)) + + # update name (and later, software type) for the partition that will take over + + cp_winner.rename(new_name=cp_exporter_ref) + cp_winner.bang(message='partitions have been renamed!') + + + + def failover(self): + try: + self._failover() + log.info('Renaming done') + except slapos.slap.ServerError: + log.info('Internal server error') + diff --git a/slapos/recipe/addresiliency/takeover.py b/slapos/recipe/addresiliency/takeover.py new file mode 100644 index 0000000000000000000000000000000000000000..d2b6a78789b0533b274d5cfddf758a24accec9be --- /dev/null +++ b/slapos/recipe/addresiliency/takeover.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import slapos.recipe.addresiliency.renamer + +def run(args): + renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'), + key_file = args.pop('key_file'), + cert_file = args.pop('cert_file'), + computer_guid = args.pop('computer_id'), + partition_id = args.pop('partition_id'), + software_release = args.pop('software'), + namebase = args.pop('namebase')) + + renamer.failover() + + diff --git a/slapos/recipe/addresiliency/template/conf.in.in b/slapos/recipe/addresiliency/template/bully.conf.in similarity index 100% rename from slapos/recipe/addresiliency/template/conf.in.in rename to slapos/recipe/addresiliency/template/bully.conf.in diff --git a/slapos/recipe/addresiliency/template/bully.py.in b/slapos/recipe/addresiliency/template/bully.py.in index a1b3a7a75c13e1252f0b63d7b0f075b22a2b2b9f..cae59e4248c0d5c21ccb21a156e850ad6a6ecec9 100644 --- a/slapos/recipe/addresiliency/template/bully.py.in +++ b/slapos/recipe/addresiliency/template/bully.py.in @@ -8,6 +8,7 @@ import sys sys.path[:] = %(syspath)s +import slapos from slapos import slap as slapmodule port = 50000 @@ -37,9 +38,11 @@ def rename_broken_and_stop(): slap.initializeConnection('%(server_url)s', '%(key_file)s', '%(cert_file)s') - computer_partition = slap.registerComputerPartition('%(computer_id)s', - '%(partition_id)s') - broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0') + computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s', + partition_id='%(partition_id)s') + broken = computer_partition.request(software_release='%(software)s', + software_type='frozen', + partition_reference='%(namebase)s0') broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime()))) broken.stopped() diff --git a/slapos/recipe/addresiliency/template/bully_new.py.in b/slapos/recipe/addresiliency/template/bully_new.py.in index 77d187e5736dda76032a0af6231c03fa2b99e7ef..cbebfda5742025b89d92552713f8c4a7f3afcd28 100644 --- a/slapos/recipe/addresiliency/template/bully_new.py.in +++ b/slapos/recipe/addresiliency/template/bully_new.py.in @@ -1,61 +1,81 @@ #!%(executable)s +import logging +import os import socket -import time import sys import thread import time -import os sys.path[:] = %(syspath)s from slapos import slap as slapmodule +import slapos + +BASE_PORT = 50000 +SLEEPING_MINS = 2 + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + + + +class Renamer(object): + def __init__(self, server_url, key_file, cert_file, computer_guid, + partition_id, software_release, namebase): + self.server_url = server_url + self.key_file = key_file + self.cert_file = cert_file + self.computer_guid = computer_guid + self.partition_id = partition_id + self.software_release = software_release + self.namebase = namebase + + def _failover(self): + slap = slapmodule.slap() + slap.initializeConnection(self.server_url, + self.key_file, + self.cert_file) + computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid, + partition_id=self.partition_id) + broken = computer_partition.request(software_release=self.software_release, + software_type='frozen', + partition_reference=self.namebase+'0') + + broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime()))) + broken.stopped() + computer_partition.rename(self.namebase+'0') + + def failover(self): + try: + log.info('renaming done') + except slapos.slap.slap.ServerError: + log.info('Internal server error') -port = 50000 -size = 1024 - - -def rename_broken_and_stop(): - try: - slap = slapmodule.slap() - slap.initializeConnection('%(server_url)s', - '%(key_file)s', - '%(cert_file)s') - computer_partition = slap.registerComputerPartition('%(computer_id)s', - '%(partition_id)s') - broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0') - - broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime()))) - broken.stopped() - computer_partition.rename('%(namebase)s0') - print 'renaming done\n' - except slapos.slap.slap.ServerError: - print 'Internal server error\n' ## Leader is always number 0 class ResilientInstance(object): - def __init__(self, comm): + def __init__(self, comm, renamer, confpath): self.comm = comm self.id = 0 self.state = 'normal' self.halter = 0 - self.nbComp = nbComp self.inElection = False self.alive = True self.lastPing = time.clock() - self.mainCanal = self.comm.canal(['ping', 'halt', - 'victory']) + self.mainCanal = self.comm.canal(['ping', 'halt', 'victory']) + self.renamer = renamer self.okCanal = self.comm.canal(['ok']) + self.confpath = confpath self.loadConnectionInfos() - def loadConnectionInfos(self): - file = open('%(confpath)s', 'r') + file = open(self.confpath, 'r') params = file.read().split('\n') file.close() self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')]) @@ -67,7 +87,8 @@ class ResilientInstance(object): ## Needs to be changed to use the master def aliveManagement(self): while self.alive: - time.sleep(30*60) + log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS) + time.sleep(SLEEPING_MINS*60) if self.id == 0: continue self.comm.send('ping', 0) @@ -84,7 +105,7 @@ class ResilientInstance(object): while self.alive: message, sender = self.mainCanal.get() if message == 'ping': - self.comm.send('ok', sender) + self.comm.send('ok', sender) elif message == 'halt': self.state = 'waitingConfirm' @@ -93,7 +114,7 @@ class ResilientInstance(object): elif message == 'victory': if int(sender) == int(self.halter) and self.state == 'waitingConfirm': - print '%s thinks %s is the leader\n' % (self.id, sender) + log.info('{} thinks {} is the leader'.format(self.id, sender)) self.comm.send('ok', sender) self.state = 'normal' @@ -105,7 +126,7 @@ class ResilientInstance(object): self.comm.send('ping', higher) message, sender = self.okCanal.get() if message: - #print '%s is alive (%s)\n' % (higher, self.id) + log.info('{} is alive ({})'.format(higher, self.id)) self.inElection = False return False continue @@ -114,7 +135,7 @@ class ResilientInstance(object): return False #I should be the new coordinator, halt those below me - print 'Should be ME : %s \n' % self.id + log.info('Should be ME : {}'.format(self.id)) self.state = 'election' self.halter = self.id ups = [] @@ -131,13 +152,13 @@ class ResilientInstance(object): message, sender = self.okCanal.get() if message: continue - print 'Something is wrong... let\'s start over\n' + log.info('Something is wrong... let\'s start over') return self.election() self.state = 'normal' self.active = True - print '%s Is THE LEADER \n' % self.id + log.info('{} Is THE LEADER'.format(self.id)) - rename_broken_and_stop() + self.renamer.failover() self.inElection = False @@ -164,27 +185,24 @@ class FilteredCanal(object): self.lock.acquire() if self.list: self.lock.release() - val = self.list[0] - self.list = self.list[1:] - return val + return self.list.pop(0) self.lock.release() return [None, None] class Wrapper(object): - def __init__(self, timeout=20): - self.read_pipes = [os.fdopen(x) for x in read_pipes] - self.write_pipes = write_pipes + def __init__(self, confpath, timeout=20): self.canals = [] self.ips = [] self.id = 0 self.timeout = timeout + self.confpath = confpath self.getConnectionInfos() self.socket = None def getConnectionInfos(self): - file = open('%(confpath)s', 'r') + file = open(self.confpath, 'r') params = file.read().split('\n') file.close() self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')] @@ -193,17 +211,17 @@ class Wrapper(object): def start(self): self.getConnectionInfos() self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - self.socket.bind((self.ips[self.id], port + self.id)) - s.listen(5) + self.socket.bind((self.ips[self.id], BASE_PORT + self.id)) + self.socket.listen(5) def send(self, message, number): self.getConnectionInfos() try: s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - s.connect((self.ips[number], port + number)) - s.send(message + (' %s\n' % self.id)) + s.connect((self.ips[number], BASE_PORT + number)) + s.send(message + (' {}\n'.format(self.id))) except (socket.error, socket.herror, socket.gaierror, socket.timeout): - pass + pass finally: s.close() @@ -213,31 +231,48 @@ class Wrapper(object): return created def recv(self): - client, _ = s.accept() + client, _ = self.socket.accept() client_message = client.recv(1024) if client_message: message, sender = client_message.split() for canal in self.canals: - canal.append(message, sender) + canal.append(message, int(sender)) + + + +def main(): + + renamer = Renamer(server_url = '%(server_url)s', + key_file = '%(key_file)s', + cert_file = '%(cert_file)s', + computer_guid = '%(computer_id)s', + partition_id = '%(partition_id)s', + software_release = '%(software)s', + namebase = '%(namebase)s') + + confpath = '%(confpath)s' + wrapper = Wrapper(confpath=confpath, timeout=20) -wrapper = createWrapper(20) + computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath) -computer = ResilientInstance(wrapper) + #idle waiting for connection infos + while computer.nbComp < 2 : + computer.loadConnectionInfos() + time.sleep(30) -#idle waiting for connection infos -while computer.nbComp < 2 : - computer.loadConnectionInfos() - time.sleep(30) + log.info('Starting') -print 'Starting\n' + computer.comm.start() + thread.start_new_thread(computer.listen, ()) + thread.start_new_thread(computer.main, ()) + thread.start_new_thread(computer.aliveManagement, ()) -computer.comm.start() -thread.start_new_thread(computer.listen, ()) -thread.start_new_thread(computer.main, ()) -thread.start_new_thread(computer.aliveManagement, ()) + while True: + # XXX tight loop + continue -while True: - continue +if __name__ == '__main__': + main() diff --git a/slapos/recipe/dcron.py b/slapos/recipe/dcron.py index 7de2668f610a2613b25c28905cac5510155bfd90..3b15ce35e4876e1527ac2a5ea6a4773268fdfa69 100644 --- a/slapos/recipe/dcron.py +++ b/slapos/recipe/dcron.py @@ -33,26 +33,22 @@ class Recipe(GenericBaseRecipe): def install(self): self.logger.info("Installing dcron...") - path_list = [] + options = self.options + script = self.createWrapper(name=options['binary'], + command=options['dcrond-binary'].strip(), + parameters=[ + '-s', options['cron-entries'], + '-c', options['crontabs'], + '-t', options['cronstamps'], + '-f', '-l', '5', + '-M', options['catcher'] + ]) - cronstamps = self.options['cronstamps'] - cron_d = self.options['cron-entries'] - crontabs = self.options['crontabs'] - catcher = self.options['catcher'] - - binary = self.options['binary'] - - script = self.createPythonScript(binary, - 'slapos.recipe.librecipe.execute.execute', - [self.options['dcrond-binary'].strip(), '-s', cron_d, '-c', crontabs, - '-t', cronstamps, '-f', '-l', '5', '-M', catcher] - ) - path_list.append(script) self.logger.debug('Main cron executable created at : %r', script) self.logger.info("dcron successfully installed.") - return path_list + return [script] diff --git a/slapos/recipe/dropbear.py b/slapos/recipe/dropbear.py index a19882d01a6ca271806359bf2bc102dc747d8ead..dec2f45d31f6b133c90d6d2b635224daaea524bb 100644 --- a/slapos/recipe/dropbear.py +++ b/slapos/recipe/dropbear.py @@ -158,6 +158,24 @@ class Client(GenericBaseRecipe): return [wrapper] + +def keysplit(s): + """ + Split a string like "ssh-rsa AKLFKJSL..... ssh-rsa AAAASAF...." + and return the individual key_type + key strings. + """ + si = iter(s.split(' ')) + while True: + key_type = next(si) + try: + key_value = next(si) + except StopIteration: + # odd number of elements, should not happen, yield the last one by itself + yield key_type + break + yield '%s %s' % (key_type, key_value) + + class AddAuthorizedKey(GenericBaseRecipe): def install(self): @@ -167,7 +185,9 @@ class AddAuthorizedKey(GenericBaseRecipe): path_list.append(ssh) authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys')) - for key in self.options['key'].split(' '): + for key in keysplit(self.options['key']): + # XXX key might actually be the string 'None' or 'null' authorized_keys.append(key) return path_list + diff --git a/slapos/recipe/equeue.py b/slapos/recipe/equeue.py index bc52654fd2e3c3d388e7a2937f4910a012b0fbf2..ea95b4c53171e2fbc25a4074384e1f2e3651ddf0 100644 --- a/slapos/recipe/equeue.py +++ b/slapos/recipe/equeue.py @@ -30,17 +30,19 @@ class Recipe(GenericBaseRecipe): def install(self): - commandline = [self.options['equeue-binary']] - commandline.extend(['--database', self.options['database']]) - commandline.extend(['-l', self.options['log']]) + parameters = [ + '--database', self.options['database'], + '-l', self.options['log'], + ] if 'loglevel' in self.options: - commandline.extend(['--loglevel', self.options['loglevel']]) + parameters.extend(['--loglevel', self.options['loglevel']]) - commandline.append(self.options['socket']) + parameters.append(self.options['socket']) + + wrapper = self.createWrapper(name=self.options['wrapper'], + command=self.options['equeue-binary'], + parameters=parameters) + + return [wrapper] - return [self.createPythonScript( - self.options['wrapper'], - 'slapos.recipe.librecipe.execute.execute', - commandline, - )] diff --git a/slapos/recipe/librecipe/__init__.py b/slapos/recipe/librecipe/__init__.py index 0fac5cfee3e0dc6ecd7285cadb6724e9bdbb8cdf..ce9de366ae365f131b26155cb5f158a58ca073c7 100644 --- a/slapos/recipe/librecipe/__init__.py +++ b/slapos/recipe/librecipe/__init__.py @@ -154,13 +154,6 @@ class BaseSlapRecipe: self._writeExecutable(wrapper_path, file_content) return wrapper_path - def createReportRunningWrapper(self, file_content): - """Creates report runnig wrapper and returns its path""" - report_wrapper_path = os.path.join(self.wrapper_report_directory, - 'slapreport') - self._writeExecutable(report_wrapper_path, file_content) - return report_wrapper_path - def substituteTemplate(self, template_location, mapping_dict): """Returns template content after substitution""" return open(template_location, 'r').read() % mapping_dict diff --git a/slapos/recipe/librecipe/generic.py b/slapos/recipe/librecipe/generic.py index 00ee68479ac00e9610318afddc0df487ad6ac5af..b48fb779ca6d57bc52399a328ba2bdbf48b73fbf 100644 --- a/slapos/recipe/librecipe/generic.py +++ b/slapos/recipe/librecipe/generic.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: set et sts=2: ############################################################################## # # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. @@ -24,6 +26,7 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## +import io import logging import os import sys @@ -90,6 +93,21 @@ class GenericBaseRecipe(object): def createExecutable(self, name, content, mode=0700): return self.createFile(name, content, mode) + def addLineToFile(self, filepath, line, encoding='utf8'): + """Append a single line to a text file, if the line does not exist yet. + + line must be unicode.""" + + try: + lines = io.open(filepath, 'r', encoding=encoding).readlines() + except IOError: + lines = [] + + if not line in lines: + lines.append(line) + with io.open(filepath, 'w+', encoding=encoding) as f: + f.write(u'\n'.join(lines)) + def createPythonScript(self, name, absolute_function, arguments=''): """Create a python script using zc.buildout.easy_install.scripts @@ -109,25 +127,31 @@ class GenericBaseRecipe(object): path, arguments=arguments)[0] return script - def createWrapper(self, name, command, parameters): + def createWrapper(self, name, command, parameters, comments=[], parameters_extra=False): """ Creates a very simple (one command) shell script for process replacement. Takes care of quoting. """ - q = shlex.quote - lines = [ - '#!/bin/sh', - 'exec %s' % shlex.quote(command) - ] + lines = [ '#!/bin/sh' ] + + for comment in comments: + lines.append('# %s' % comment) + + lines.append('exec %s' % shlex.quote(command)) for param in parameters: - if len(lines[-1]) < 30: + if len(lines[-1]) < 40: lines[-1] += ' ' + shlex.quote(param) else: lines[-1] += ' \\' lines.append('\t' + shlex.quote(param)) + if parameters_extra: + # pass-through further parameters + lines[-1] += ' \\' + lines.append('\t$@') + content = '\n'.join(lines) + '\n' return self.createFile(name, content, 0700) diff --git a/slapos/recipe/mydumper.py b/slapos/recipe/mydumper.py index 75dd3e2bcbf340aec16227f3604ff1e34b92af91..9507520ca3770253fe75b7791ab171a88b405e0a 100644 --- a/slapos/recipe/mydumper.py +++ b/slapos/recipe/mydumper.py @@ -28,85 +28,76 @@ import subprocess from slapos.recipe.librecipe import GenericBaseRecipe -def dump(args): - mydumper_cmd = [args['mydumper']] - mydumper_cmd.extend(['-B', args['database']]) - if args['socket'] is not None: - mydumper_cmd.extend(['-S', args['socket']]) - else: - mydumper_cmd.extend(['-h', args['host']]) - mydumper_cmd.etxned(['-P', args['port']]) - - mydumper_cmd.extend(['-u', args['user']]) - if args['password'] is not None: - mydumper_cmd.extend(['-p', args['password']]) - - if args['compression']: - mydumper_cmd.append('--compress') +def _mydumper_base_cmd(mydumper, database, user, password, + socket=None, host=None, port=None, **kw): + cmd = [mydumper] + cmd.extend(['-B', database]) - if args['rows'] is not None: - mydumper_cmd.extend(['-r', args['rows']]) - - mydumper_cmd.extend(['-o', args['directory']]) + if socket: + cmd.extend(['-S', socket]) + else: + cmd.extend(['-h', host]) + cmd.extend(['-P', port]) - subprocess.check_call(mydumper_cmd) + cmd.extend(['-u', user]) + if password: + cmd.extend(['-p', password]) + return cmd -def do_import(args): - mydumper_cmd = [args['mydumper']] - mydumper_cmd.extend(['-B', args['database']]) +def do_export(args): + cmd = _mydumper_base_cmd(**args) - if args['socket'] is not None: - mydumper_cmd.extend(['-S', args['socket']]) - else: - mydumper_cmd.extend(['-h', args['host']]) - mydumper_cmd.etxned(['-P', args['port']]) + if args['compression']: + cmd.append('--compress') - mydumper_cmd.extend(['-u', args['user']]) - if args['password'] is not None: - mydumper_cmd.extend(['-p', args['password']]) + if args['rows'] is not None: + cmd.extend(['-r', args['rows']]) - mydumper_cmd.append('--overwrite-tables') + cmd.extend(['-o', args['directory']]) - mydumper_cmd.extend(['-d', args['directory']]) + subprocess.check_call(cmd) - subprocess.check_call(mydumper_cmd) +def do_import(args): + cmd = _mydumper_base_cmd(**args) + cmd.append('--overwrite-tables') + cmd.extend(['-d', args['directory']]) + subprocess.check_call(cmd) class Recipe(GenericBaseRecipe): def install(self): - # Host or socket should be defined - try: - self.options['host'] - except: - self.options['socket'] - - config = dict(database=self.options['database'], - socket=self.options.get('socket'), - host=self.options.get('host'), - port=self.options.get('port', 3306), - directory=self.options['backup-directory'], - user=self.options['user'], - password=self.options.get('password'), - ) - - name = __name__ + config = { + 'database': self.options['database'], + 'directory': self.options['backup-directory'], + 'user': self.options['user'], + 'password': self.options.get('password'), + } + + if self.options.get('host'): + config['host'] = self.options['host'] + config['port'] = self.options.get('port', 3306) + elif self.options.get('socket'): + config['socket'] = self.options['socket'] + else: + raise ValueError("host or socket must be defined") + if self.optionIsTrue('import', False): - config.update(mydumper=self.options['myloader-binary']) - name += '.do_import' + function = do_import + config['mydumper'] = self.options['myloader-binary'] else: - config.update(mydumper=self.options['mydumper-binary'], - compression=self.optionIsTrue('compression', default=False), - rows=self.options.get('rows'), - ) - name += '.dump' + function = do_export + config['mydumper'] = self.options['mydumper-binary'] + config['compression'] = self.optionIsTrue('compression', default=False) + config['rows'] = self.options.get('rows') - wrapper = self.createPythonScript(self.options['wrapper'], - name, - config) + wrapper = self.createPythonScript(name=self.options['wrapper'], + absolute_function = '%s.%s' % (__name__, function.func_name), + arguments=config) return [wrapper] + diff --git a/slapos/recipe/notifier.py b/slapos/recipe/notifier.py index a946673c037fd01b1a98e11b91a4111ca7db90af..0eb954014d84fd4541555790fa118a9299f96932 100644 --- a/slapos/recipe/notifier.py +++ b/slapos/recipe/notifier.py @@ -31,62 +31,76 @@ from slapos.recipe.librecipe import GenericBaseRecipe class Recipe(GenericBaseRecipe): def install(self): - commandline = [self.options['server-binary']] - commandline.extend(['--callbacks', self.options['callbacks']]) - commandline.extend(['--feeds', self.options['feeds']]) - commandline.extend(['--equeue-socket', self.options['equeue-socket']]) - commandline.append(self.options['host']) - commandline.append(self.options['port']) + options = self.options + script = self.createWrapper(name=options['wrapper'], + command=options['server-binary'], + parameters=[ + '--callbacks', options['callbacks'], + '--feeds', options['feeds'], + '--equeue-socket', options['equeue-socket'], + options['host'], options['port'] + ], + comments=[ + '', + 'Upon receiving a notification, execute the callback(s).', + '']) + return [script] - return [self.createPythonScript(self.options['wrapper'], - 'slapos.recipe.librecipe.execute.execute', - commandline)] class Callback(GenericBaseRecipe): def createCallback(self, notification_id, callback): callback_id = sha512(notification_id).hexdigest() - callback = self.createFile(os.path.join(self.options['callbacks'], - callback_id), - callback) - return callback + + filepath = os.path.join(self.options['callbacks'], callback_id) + self.addLineToFile(filepath, callback) + return filepath def install(self): + # XXX this path is returned multiple times, one for each callback that has been added. return [self.createCallback(self.options['on-notification-id'], self.options['callback'])] class Notify(GenericBaseRecipe): - def createNotifier(self, notifier_binary, executable, wrapper, **kwargs): - if not os.path.exists(kwargs['log']): + def createNotifier(self, notifier_binary, wrapper, executable, + log, title, notification_url, feed_url): + + if not os.path.exists(log): # Just a touch - open(kwargs['log'], 'w').close() + open(log, 'w').close() - commandline = [notifier_binary, - '-l', kwargs['log'], - '--title', kwargs['title'], - '--feed', kwargs['feed_url'], - '--notification-url'] + parameters = [ + '-l', log, + '--title', title, + '--feed', feed_url, + '--notification-url', + ] + parameters.extend(notification_url.split(' ')) + parameters.extend(['--executable', executable]) - commandline.extend(kwargs['notification_url'].split(' ')) - commandline.extend(['--executable', executable]) + return self.createWrapper(name=wrapper, + command=notifier_binary, + parameters=parameters, + comments=[ + '', + 'Call an executable and send notification(s).', + '']) - return self.createPythonScript(wrapper, - 'slapos.recipe.librecipe.execute.execute', - [str(i) for i in commandline]) def install(self): - feedurl = self.unparseUrl(scheme='http', host=self.options['host'], - port=self.options['port'], - path='/get/%s' % self.options['name']) - - script = self.createNotifier( - self.options['notifier-binary'], - wrapper=self.options['wrapper'], - executable=self.options['executable'], - log=os.path.join(self.options['feeds'], self.options['name']), - title=self.options['title'], - notification_url=self.options['notify'], - feed_url=feedurl, - ) + feed_url = self.unparseUrl(scheme='http', host=self.options['host'], + port=self.options['port'], + path='/get/%s' % self.options['name']) + + log = os.path.join(self.options['feeds'], self.options['name']) + + options = self.options + script = self.createNotifier(notifier_binary=options['notifier-binary'], + wrapper=options['wrapper'], + executable=options['executable'], + log=log, + title=options['title'], + notification_url=options['notify'], + feed_url=feed_url) return [script] diff --git a/slapos/recipe/pbs.py b/slapos/recipe/pbs.py index ac08a928f99e168ed43246dde4e3f6bb4efb2797..3cdf9b498bc69c408428956eea70f3c9522aa3f7 100644 --- a/slapos/recipe/pbs.py +++ b/slapos/recipe/pbs.py @@ -24,14 +24,15 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## -from json import loads as unjson + from hashlib import sha512 -from urlparse import urlparse +import inspect +import json import os +import signal import subprocess import sys -import signal -import inspect +import urlparse from slapos.recipe.librecipe import GenericSlapRecipe from slapos.recipe.dropbear import KnownHostsFile @@ -43,8 +44,7 @@ from slapos import slap as slapmodule def promise(args): def failed_ssh(partition, ssh): - # Bad python 2 syntax, looking forward python 3 to have print(file=) - print >> sys.stderr, "SSH Connection failed" + sys.stderr.write("SSH Connection failed\n") try: ssh.terminate() except: @@ -75,16 +75,20 @@ def promise(args): slap = slapmodule.slap() slap.initializeConnection(args['server_url'], - key_file=args.get('key_file'), cert_file=args.get('cert_file')) + key_file=args.get('key_file'), + cert_file=args.get('cert_file')) + partition = slap.registerComputerPartition(args['computer_id'], args['partition_id']) + ssh = subprocess.Popen([args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args], + stdin=subprocess.PIPE, + stdout=open(os.devnull, 'w'), + stderr=open(os.devnull, 'w')) + # Rdiff Backup protocol quit command quitcommand = 'q' + chr(255) + chr(0) * 7 - ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args] - ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE, - stdout=open(os.devnull), stderr=open(os.devnull)) ssh.stdin.write(quitcommand) ssh.stdin.flush() ssh.stdin.close() @@ -113,7 +117,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback): promise_path = os.path.join(self.options['promises-directory'], url_hash) - parsed_url = urlparse(url) + parsed_url = urlparse.urlparse(url) promise_dict = self.promise_base_dict.copy() promise_dict.update(user=parsed_url.username, host=parsed_url.hostname, @@ -127,15 +131,17 @@ class Recipe(GenericSlapRecipe, Notify, Callback): host = parsed_url.hostname known_hosts_file[host] = entry['server-key'] - remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \ + # XXX use -y because the host might not yet be in the + # trusted hosts file until the next time slapgrid is run. + + remote_schema = '%(ssh)s -y -p %%s %(user)s@%(host)s' % \ { 'ssh': self.options['sshclient-binary'], 'user': parsed_url.username, 'host': parsed_url.hostname, } - command = [self.options['rdiffbackup-binary']] - command.extend(['--remote-schema', remote_schema]) + parameters = ['--remote-schema', remote_schema] remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port, 'path': parsed_url.path} @@ -144,38 +150,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback): name_hash) if entry['type'] == 'push': - command.extend(['--restore-as-of', 'now']) - command.append('--force') - command.extend([local_directory, remote_directory]) + parameters.extend(['--restore-as-of', 'now']) + parameters.append('--force') + parameters.extend([local_directory, remote_directory]) + comments = ['','Push data to a PBS *-import instance.',''] else: - command.extend([remote_directory, local_directory]) + parameters.extend([remote_directory, local_directory]) + comments = ['','Pull data from a PBS *-export instance.',''] wrapper_basepath = os.path.join(self.options['wrappers-directory'], url_hash) - wrapper_path = wrapper_basepath if 'notify' in entry: - wrapper_path = '%s_raw' % wrapper_basepath + wrapper_path = wrapper_basepath + '_raw' + else: + wrapper_path = wrapper_basepath - wrapper = self.createPythonScript( - wrapper_path, - 'slapos.recipe.librecipe.execute.execute', - [str(i) for i in command] - ) + wrapper = self.createWrapper(name=wrapper_path, + command=self.options['rdiffbackup-binary'], + parameters=parameters, + comments = comments) path_list.append(wrapper) if 'notify' in entry: feed_url = '%s/get/%s' % (self.options['notifier-url'], entry['notification-id']) - wrapper = self.createNotifier( - self.options['notifier-binary'], - wrapper=wrapper_basepath, - executable=wrapper_path, - log=os.path.join(self.options['feeds'], entry['notification-id']), - title=entry.get('title', 'Untitled'), - notification_url=entry['notify'], - feed_url=feed_url, - ) + wrapper = self.createNotifier(notifier_binary=self.options['notifier-binary'], + wrapper=wrapper_basepath, + executable=wrapper_path, + log=os.path.join(self.options['feeds'], entry['notification-id']), + title=entry.get('title', 'Untitled'), + notification_url=entry['notify'], + feed_url=feed_url, + ) path_list.append(wrapper) #self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference']) @@ -190,40 +197,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback): return path_list + def _install(self): path_list = [] - if self.optionIsTrue('client', True): self.logger.info("Client mode") slap_connection = self.buildout['slap-connection'] - self.promise_base_dict = dict( - server_url=slap_connection['server-url'], - computer_id=slap_connection['computer-id'], - cert_file=slap_connection.get('cert-file'), - key_file=slap_connection.get('key-file'), - partition_id=slap_connection['partition-id'], - ssh_client=self.options['sshclient-binary'], - ) - - slaves = unjson(self.options['slave-instance-list']) + self.promise_base_dict = { + 'server_url': slap_connection['server-url'], + 'computer_id': slap_connection['computer-id'], + 'cert_file': slap_connection.get('cert-file'), + 'key_file': slap_connection.get('key-file'), + 'partition_id': slap_connection['partition-id'], + 'ssh_client': self.options['sshclient-binary'], + } + + slaves = json.loads(self.options['slave-instance-list']) known_hosts = KnownHostsFile(self.options['known-hosts']) with known_hosts: + # XXX this API could be cleaner for slave in slaves: path_list.extend(self.add_slave(slave, known_hosts)) - - else: - command = [self.options['rdiffbackup-binary']] self.logger.info("Server mode") - command.extend(['--restrict', self.options['path']]) - command.append('--server') - wrapper = self.createPythonScript( - self.options['wrapper'], - 'slapos.recipe.librecipe.execute.execute', - command) + wrapper = self.createWrapper(name=self.options['wrapper'], + command=self.options['rdiffbackup-binary'], + parameters=[ + '--restrict', self.options['path'], + '--server' + ]) path_list.append(wrapper) return path_list + diff --git a/slapos/recipe/postgres/__init__.py b/slapos/recipe/postgres/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c44d627bacc9a46b573e7dfab0a39ab7301c21a5 --- /dev/null +++ b/slapos/recipe/postgres/__init__.py @@ -0,0 +1,260 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import md5 +import os +import subprocess +import textwrap +from zc.buildout import UserError + +from slapos.recipe.librecipe import GenericBaseRecipe + + + +class Recipe(GenericBaseRecipe): + """\ + This recipe creates: + + - a Postgres cluster + - configuration to allow connections from IPV6 only (or unix socket) + - a superuser with provided name and generated password + - a database with provided name + - a foreground start script in the services directory + + then adds the connection URL to the options. + The URL can be used as-is (ie. in sqlalchemy) or by the _urlparse.py recipe. + """ + + def fetch_ipv6_host(self, options): + """ + Returns a string represtation of ipv6_host. + May receive a regular string, a set or a string serialized by buildout. + """ + ipv6_host = options['ipv6_host'] + + if isinstance(ipv6_host, set): + return ipv6_host.pop() + else: + return ipv6_host + + + def _options(self, options): + options['password'] = self.generatePassword() + options['url'] = 'postgresql://%(user)s:%(password)s@[%(ipv4_host)s]:%(port)s/%(dbname)s' % options + + + def install(self): + pgdata = self.options['pgdata-directory'] + + if not os.path.exists(pgdata): + self.createCluster() + self.createConfig() + self.createDatabase() + self.createSuperuser() + self.createRunScript() + + return [ + # XXX should we really return something here? + # os.path.join(pgdata, 'postgresql.conf') + ] + + + def check_exists(self, path): + if not os.path.isfile(path): + raise IOError('File not found: %s' % path) + + + def createCluster(self): + initdb_binary = os.path.join(self.options['bin'], 'initdb') + self.check_exists(initdb_binary) + + pgdata = self.options['pgdata-directory'] + + try: + subprocess.check_call([initdb_binary, + '-D', pgdata, + '-A', 'ident', + '-E', 'UTF8', + ]) + except subprocess.CalledProcessError: + raise UserError('Could not create cluster directory in %s' % pgdata) + + + def createConfig(self): + pgdata = self.options['pgdata-directory'] + + with open(os.path.join(pgdata, 'postgresql.conf'), 'wb') as cfg: + cfg.write(textwrap.dedent("""\ + listen_addresses = '%s,%s' + logging_collector = on + log_rotation_size = 50MB + max_connections = 100 + datestyle = 'iso, mdy' + + lc_messages = 'en_US.UTF-8' + lc_monetary = 'en_US.UTF-8' + lc_numeric = 'en_US.UTF-8' + lc_time = 'en_US.UTF-8' + default_text_search_config = 'pg_catalog.english' + + unix_socket_directory = '%s' + unix_socket_permissions = 0700 + """ % ( + self.options['ipv4_host'], + self.fetch_ipv6_host(self.options), + pgdata, + ))) + + + with open(os.path.join(pgdata, 'pg_hba.conf'), 'wb') as cfg: + # see http://www.postgresql.org/docs/9.1/static/auth-pg-hba-conf.html + + cfg.write(textwrap.dedent("""\ + # TYPE DATABASE USER ADDRESS METHOD + + # "local" is for Unix domain socket connections only (check unix_socket_permissions!) + local all all ident + host all all 127.0.0.1/32 md5 + host all all %s/32 md5 + host all all ::1/128 md5 + host all all %s/128 md5 + """ % (self.options['ipv4_host'], self.fetch_ipv6_host(self.options)))) + + + def createDatabase(self): + self.runPostgresCommand(cmd='CREATE DATABASE "%s"' % self.options['dbname']) + + + def createSuperuser(self): + """ + Creates a Postgres superuser - other than "slapuser#" for use by the application. + """ + + # http://postgresql.1045698.n5.nabble.com/Algorithm-for-generating-md5-encrypted-password-not-found-in-documentation-td4919082.html + + user = self.options['user'] + password = self.options['password'] + + # encrypt the password to avoid storing in the logs + enc_password = 'md5' + md5.md5(password+user).hexdigest() + + self.runPostgresCommand(cmd="""CREATE USER "%s" ENCRYPTED PASSWORD '%s' SUPERUSER""" % (user, enc_password)) + + + def runPostgresCommand(self, cmd): + """ + Executes a command in single-user mode, with no daemon running. + + Multiple commands can be executed by providing newlines, + preceeded by backslash, between them. + See http://www.postgresql.org/docs/9.1/static/app-postgres.html + """ + + pgdata = self.options['pgdata-directory'] + postgres_binary = os.path.join(self.options['bin'], 'postgres') + + try: + p = subprocess.Popen([postgres_binary, + '--single', + '-D', pgdata, + 'postgres', + ], stdin=subprocess.PIPE) + + p.communicate(cmd+'\n') + except subprocess.CalledProcessError: + raise UserError('Could not create database %s' % pgdata) + + + def createRunScript(self): + """ + Creates a script that runs postgres in the foreground. + 'exec' is used to allow easy control by supervisor. + """ + content = textwrap.dedent("""\ + #!/bin/sh + exec %(bin)s/postgres \\ + -D %(pgdata-directory)s + """ % self.options) + name = os.path.join(self.options['services'], 'postgres-start') + self.createExecutable(name, content=content) + + + +class ExportRecipe(GenericBaseRecipe): + + def install(self): + pgdata = self.options['pgdata-directory'] + wrapper = self.options['wrapper'] + self.createBackupScript(wrapper) + return [wrapper] + + + def createBackupScript(self, wrapper): + """ + Create a script to backup the database in 'custom' format. + """ + content = textwrap.dedent("""\ + #!/bin/sh + umask 077 + %(bin)s/pg_dump \\ + --host=%(pgdata-directory)s \\ + --format=custom \\ + --file=%(backup-directory)s/database.dump \\ + %(dbname)s + """ % self.options) + self.createExecutable(wrapper, content=content) + + + +class ImportRecipe(GenericBaseRecipe): + + def install(self): + pgdata = self.options['pgdata-directory'] + wrapper = self.options['wrapper'] + self.createRestoreScript(wrapper) + return [wrapper] + + + def createRestoreScript(self, wrapper): + """ + Create a script to restore the database from 'custom' format. + """ + content = textwrap.dedent("""\ + #!/bin/sh + %(bin)s/pg_restore \\ + --host=%(pgdata-directory)s \\ + --dbname=%(dbname)s \\ + --clean \\ + --no-owner \\ + --no-acl \\ + %(backup-directory)s/database.dump + """ % self.options) + self.createExecutable(wrapper, content=content) + + + + diff --git a/slapos/recipe/request.py b/slapos/recipe/request.py index e326810f0306a4dbf0d218df6003d8cd7e8b2047..b0c785c16d1348d5266ca3e24a333169cd6d77ea 100644 --- a/slapos/recipe/request.py +++ b/slapos/recipe/request.py @@ -123,28 +123,41 @@ class Recipe(object): isSlave = options.get('slave', '').lower() in \ librecipe.GenericBaseRecipe.TRUE_VALUES - self.instance = instance = request(software_url, software_type, - name, partition_parameter_kw=partition_parameter_kw, - filter_kw=filter_kw, shared=isSlave) + + self._raise_request_exception = None + self.instance = None + try: + self.instance = request(software_url, software_type, + name, partition_parameter_kw=partition_parameter_kw, + filter_kw=filter_kw, shared=isSlave) + # XXX what is the right way to get a global id? + options['instance_guid'] = self.instance.getId() + except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady) as exc: + self._raise_request_exception = exc for param in return_parameters: + options['connection-%s' % param] = '' + if not self.instance: + continue try: options['connection-%s' % param] = str( - instance.getConnectionParameter(param)) - except (slapmodule.NotFoundError, slapmodule.ServerError): - options['connection-%s' % param] = '' + self.instance.getConnectionParameter(param)) + except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady): if self.failed is None: self.failed = param def install(self): + if self._raise_request_exception: + raise self._raise_request_exception + if self.failed is not None: # Check instance status to know if instance has been deployed try: - if self.instance.getComputerId() is not None: + if self.instance._computer_id is not None: status = self.instance.getState() else: status = 'not ready yet' - except (slapmodule.NotFoundError, slapmodule.ServerError): + except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady): status = 'not ready yet' except AttributeError: status = 'unknown' @@ -159,14 +172,17 @@ class Recipe(object): class RequestOptional(Recipe): """ - Request a SlapOS instance. Won't fail if instance is not ready. + Request a SlapOS instance. Won't fail if request failed or is not ready. Same as slapos.cookbook:request, but won't raise in case of problem. """ def install(self): - if self.failed is not None: + if self._raise_request_exception: + self.logger.warning('Optional request failed:') + self.logger.warning(self._raise_request_exception) + elif self.failed is not None: # Check instance status to know if instance has been deployed try: - if self.instance.getComputerId() is not None: + if self.instance._computer_id is not None: status = self.instance.getState() else: status = 'not ready yet' diff --git a/slapos/recipe/slapmonitor/__init__.py b/slapos/recipe/slapmonitor/__init__.py index 6dc288366a80aac5ff868ffbb1ba21ad21666fd1..4b353af6096d53855e3ca986031b79d8fd389500 100644 --- a/slapos/recipe/slapmonitor/__init__.py +++ b/slapos/recipe/slapmonitor/__init__.py @@ -24,31 +24,31 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## -from slapos.recipe.librecipe import GenericBaseRecipe -class Recipe(GenericBaseRecipe): - """ - Slapmonitor instance configuration. - """ +from slapos.recipe.librecipe import GenericBaseRecipe - def __init__(self, buildout, name, options): - return GenericBaseRecipe.__init__(self, buildout, name, options) +class MonitorRecipe(GenericBaseRecipe): def install(self): - config = dict( - pid_file_path=self.options['pid-file'], - database_path=self.options['database-path'], - slapmonitor_path = self.options['slapmonitor-path'], - shell_path=self.options['shell-path'], - ) + options = self.options + script = self.createWrapper(name=options['path'], + command=options['slapmonitor-path'], + parameters=[ + options['pid-file'], + options['database-path'], + ]) + return [script] - # Runners - runner_path = self.createExecutable( - self.options['path'], - self.substituteTemplate(self.getTemplateFilename('slapmonitor_run.in'), - config)) - return [runner_path] +class MonitorXMLRecipe(GenericBaseRecipe): + + def install(self): + options = self.options + script = self.createWrapper(name=options['path'], + command=options['slapmonitor-xml-path'], + parameters=[ + options['database-path'], + ], + parameters_extra=True) + return [script] - def update(self): - pass diff --git a/slapos/recipe/slapmonitor/template/slapmonitor_run.in b/slapos/recipe/slapmonitor/template/slapmonitor_run.in deleted file mode 100644 index c0f96430765f9b79b2fd92a89146a49fbee2adb9..0000000000000000000000000000000000000000 --- a/slapos/recipe/slapmonitor/template/slapmonitor_run.in +++ /dev/null @@ -1,6 +0,0 @@ -#!%(shell_path)s -# BEWARE: This file is operated by slapgrid -# BEWARE: It will be overwritten automatically -# -exec %(slapmonitor_path)s %(pid_file_path)s %(database_path)s - diff --git a/slapos/recipe/slapreport/__init__.py b/slapos/recipe/slapreport/__init__.py index e5f6a57ddf5c33ce521752b481dca80e8fefab4f..d9f5bcb0a450b13d120b51bf8997764296624bc1 100644 --- a/slapos/recipe/slapreport/__init__.py +++ b/slapos/recipe/slapreport/__init__.py @@ -24,36 +24,23 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## + from slapos.recipe.librecipe import GenericBaseRecipe class Recipe(GenericBaseRecipe): - """ - Slapmonitor instance configuration. - """ - - def __init__(self, buildout, name, options): - return GenericBaseRecipe.__init__(self, buildout, name, options) def install(self): - config = dict( - pid_file_path=self.options['pid-file'], - consumption_log_path=self.options['consumption-log-path'], - database_path=self.options['database-path'], - slapreport_path = self.options['slapreport-path'], - logbox_ip = self.options['logbox-ip'], - logbox_port = self.options['logbox-port'], - logbox_user = self.options['logbox-user'], - logbox_passwd = self.options['logbox-passwd'], - shell_path=self.options['shell-path'], - ) - - # Runners - runner_path = self.createExecutable( - self.options['path'], - self.substituteTemplate(self.getTemplateFilename('slapreport_run.in'), - config)) - - return [runner_path] + options = self.options + script = self.createWrapper(name=options['path'], + command=options['slapreport-path'], + parameters=[ + options['pid-file'], + options['consumption-log-path'], + options['database-path'], + options['logbox-ip'], + options['logbox-port'], + options['logbox-user'], + options['logbox-passwd'], + ]) + return [script] - def update(self): - pass diff --git a/slapos/recipe/slapreport/template/slapreport_run.in b/slapos/recipe/slapreport/template/slapreport_run.in deleted file mode 100644 index 19bafc12040a9bd7440ee0b01cd8dbde5f1179c9..0000000000000000000000000000000000000000 --- a/slapos/recipe/slapreport/template/slapreport_run.in +++ /dev/null @@ -1,5 +0,0 @@ -#!%(shell_path)s -# BEWARE: This file is operated by slapgrid -# BEWARE: It will be overwritten automatically -# -exec %(slapreport_path)s %(pid_file_path)s %(consumption_log_path)s %(database_path)s %(logbox_ip)s %(logbox_port)s %(logbox_user)s %(logbox_passwd)s diff --git a/software/drupal/software.cfg b/software/drupal/software.cfg index e9bf0e7caeacf4d70d83a6a7c6a532de75791156..8c705226623290ab0da4cda51cbcfecaa829bc68 100644 --- a/software/drupal/software.cfg +++ b/software/drupal/software.cfg @@ -12,7 +12,6 @@ recipe = slapos.recipe.build:download-unpacked url = http://ftp.drupal.org/files/projects/drupal-7.16.tar.gz md5sum = 352497b2df94b5308e31cb8da020b631 - [download-patch-hide-dbsetup] recipe = hexagonit.recipe.download url = ${:_profile_base_location_}/${:filename} diff --git a/software/maarch/config.php.in b/software/maarch/config.php.in new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/software/maarch/instance-custom.cfg.in b/software/maarch/instance-custom.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..4829ce57bf11aeac4b36b78b1bf27d82519b7924 --- /dev/null +++ b/software/maarch/instance-custom.cfg.in @@ -0,0 +1,15 @@ +[buildout] + +[maarch-configuration] +recipe = slapos.recipe.maarch:default +htdocs = $${apache-php:htdocs} +db_host = $${postgres-urlparse:host} +db_port = $${postgres-urlparse:port} +db_dbname = $${postgres-urlparse:path} +db_username = $${postgres-urlparse:username} +db_password = $${postgres-urlparse:password} +language = en +php_ini_dir = $${directory:php-ini-dir} +root_docservers = $${buildout:directory}/srv/docservers +dependency = $${apache-php:recipe} + diff --git a/software/maarch/software.cfg b/software/maarch/software.cfg new file mode 100644 index 0000000000000000000000000000000000000000..29edbbe45e1e6beec6331281edd65ad3c9a5fa6c --- /dev/null +++ b/software/maarch/software.cfg @@ -0,0 +1,76 @@ +[buildout] + +extends = + ../../stack/lapp/buildout.cfg + +# += since we need rdiff-backup and friends +parts += + slapos-cookbook + apache-php-postgres + pear-modules + eggs + instance + instance-apache-php + +#---------------- +#-- +#-- Main application part +#-- XXX provide a better URL + +[application] +url = http://downloads.sourceforge.net/project/maarch/Maarch%20Entreprise/MaarchEntreprise-1.3.zip?r=http%3A%2F%2Fwww.maarch.org%2Ftelecharger&ts=1347961624&use_mirror=ignum +md5sum = 5c2c859dee9d0dde3ba959474fd5fc86 + + +#---------------- +#-- +#-- We don't need this static configuration file. +#-- An empty file is provided because it is required by the lapp stack. +#-- + +[application-template] +recipe = slapos.recipe.download +url = ${:_profile_base_location_}/config.php.in +md5sum = d41d8cd98f00b204e9800998ecf8427e +download-only = True +filename = template.in +mode = 0644 +location = ${buildout:parts-directory}/${:_buildout_section_name_} + +[application-configuration] +location = config.php + + +#---------------- +#-- +#-- Define parts that will be executed later, in the instance. +#-- + +[custom-application-deployment] +path = ${custom-application-deployment-template:output} +part-list = maarch-configuration + +[custom-application-deployment-template] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-custom.cfg.in +output = ${buildout:directory}/instance-custom.cfg +md5sum = 28f6ad42b73806901ac2cd4ec563d73b +mode = 0644 + + +#---------------- +#-- +#-- Install PHP channels and modules (only if they are not already installed). +#-- + +[pear-modules] +recipe = cp.recipe.cmd +pear = ${apache-php-postgres:location}/bin/pear +install_cmd = + ${:pear} channel-info pear.maarch.org >/dev/null || ${:pear} channel-discover pear.maarch.org + ${:pear} info maarch/CLITools-0.3.1 >/dev/null || ${:pear} install maarch/CLITools-0.3.1 + ${:pear} info MIME_Type >/dev/null || ${:pear} install MIME_Type + +#---------------- + + diff --git a/software/mioga/instance.cfg b/software/mioga/instance.cfg new file mode 100644 index 0000000000000000000000000000000000000000..fffebea8fe2c6c76697fe4be87552f265614a35d --- /dev/null +++ b/software/mioga/instance.cfg @@ -0,0 +1,17 @@ +[buildout] +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} + +parts = instance + +[instance] +recipe = ${instance-recipe:egg}:${instance-recipe:module} +source = ${application:location} +template = ${application-template:location}/${application-template:filename} +configuration = ${application-configuration:location} + +httpd_binary = ${apache:location}/bin/httpd +mysql_binary = ${mariadb:location}/bin/mysql +mysql_install_binary = ${mariadb:location}/bin/mysql_install_db +mysql_upgrade_binary = ${mariadb:location}/bin/mysql_upgrade +mysqld_binary = ${mariadb:location}/libexec/mysqld diff --git a/software/mioga/software.cfg b/software/mioga/software.cfg new file mode 100644 index 0000000000000000000000000000000000000000..10f53fd5ff34144dcb48e4ca703a256774444ffb --- /dev/null +++ b/software/mioga/software.cfg @@ -0,0 +1,22 @@ +[buildout] +extends = + ../../component/postgresql/buildout.cfg + ../../component/apache-perl/buildout.cfg + ../../stack/slapos.cfg + +[networkcache] +# Cedric de Saint Martin signature certificate +signature-certificate-list = + -----BEGIN CERTIFICATE----- + MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV + BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT + MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA + wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB + D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1 + P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O + BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp + cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr + b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV + Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby + If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY= + -----END CERTIFICATE----- diff --git a/software/postgres/instance.cfg.in b/software/postgres/instance.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..8a7c190584cc9ff2943378041f0bacd983bfec28 --- /dev/null +++ b/software/postgres/instance.cfg.in @@ -0,0 +1,69 @@ +[buildout] +parts = + symlinks + publish + postgres-instance + + +# Define egg directories to be the one from Software Release +# (/opt/slapgrid/...) +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + + +[instance-parameters] +# Fetches parameters defined in SlapOS Master for this instance +recipe = slapos.cookbook:slapconfiguration +computer = $${slap-connection:computer-id} +partition = $${slap-connection:partition-id} +url = $${slap-connection:server-url} +key = $${slap-connection:key-file} +cert = $${slap-connection:cert-file} + + +[directories] +recipe = slapos.cookbook:mkdirectory +bin = $${buildout:directory}/bin +etc = $${buildout:directory}/etc +services = $${directories:etc}/run/ +promises = $${directories:etc}/promise/ +var = $${buildout:directory}/var + + +[symlinks] +recipe = cns.recipe.symlink +symlink_target = $${directories:bin} +symlink_base = ${postgresql:location}/bin + + +[postgres-instance] +# create cluster, configuration files and a database +recipe = slapos.cookbook:postgres + +# Options +ipv6_host = $${instance-parameters:ipv6} +ipv4_host = $${slap-network-information:local-ipv4} +user = user +port = 5432 +dbname = db +# pgdata_directory is created by initdb, and should not exist beforehand. +pgdata-directory = $${directories:var}/data +services = $${directories:services} +bin = $${directories:bin} + + +[publish] +recipe = slapos.cookbook:publishurl +url = $${postgres-instance:url} + + +[slap-connection] +# part to migrate to new - separated words +computer-id = $${slap_connection:computer_id} +partition-id = $${slap_connection:partition_id} +server-url = $${slap_connection:server_url} +software-release-url = $${slap_connection:software_release_url} +key-file = $${slap_connection:key_file} +cert-file = $${slap_connection:cert_file} + diff --git a/software/postgres/software.cfg b/software/postgres/software.cfg new file mode 100644 index 0000000000000000000000000000000000000000..46d21841829f08b1472a6ca8dcd370f71a9ee02a --- /dev/null +++ b/software/postgres/software.cfg @@ -0,0 +1,25 @@ +[buildout] + +extends = + ../../stack/slapos.cfg + ../../component/postgresql/buildout.cfg + +parts = + eggs + slapos-cookbook + instance-template + postgresql + +[instance-template] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance.cfg.in +output = ${buildout:directory}/template.cfg +#md5sum = +mode = 0644 + + +[eggs] +recipe = zc.recipe.egg +eggs = + cns.recipe.symlink + diff --git a/stack/lamp/apache/instance-apache-backup.cfg b/stack/lamp/apache/instance-apache-backup.cfg.in similarity index 77% rename from stack/lamp/apache/instance-apache-backup.cfg rename to stack/lamp/apache/instance-apache-backup.cfg.in index 69f14c2fa48acd7e4f7e1b8a48b4351aad3db126..5464bd05cce4316b8d63d6a1cae27b3656c9241b 100644 --- a/stack/lamp/apache/instance-apache-backup.cfg +++ b/stack/lamp/apache/instance-apache-backup.cfg.in @@ -16,32 +16,92 @@ eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} offline = true -[urls] -recipe = slapos.cookbook:publish -url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/ -ssh-public-key = $${sshkeys-dropbear:public-key-value} -ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path} -[apache-proxy] -recipe = slapos.cookbook:apacheproxy -url = $${slap-parameter:proxy-url} -pid-file = $${basedirectory:run}/apache.pid -lock-file = $${basedirectory:run}/apache.lock -ip = $${slap-network-information:global-ipv6} -port = 8080 -error-log = $${directory:httpd-log}/error.log -access-log = $${directory:httpd-log}/access.log -httpd-conf = $${rootdirectory:etc}/apache.conf -wrapper = $${basedirectory:services}/apache +#---------------- +#-- +#-- Creation of all needed directories. -promise = $${basedirectory:promises}/apache +[rootdirectory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +var = $${buildout:directory}/var +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin +tmp = $${buildout:directory}/tmp -httpd-binary = ${apache:location}/bin/httpd +[basedirectory] +recipe = slapos.cookbook:mkdirectory +log = $${rootdirectory:var}/log +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +backup = $${rootdirectory:srv}/backup +promises = $${rootdirectory:etc}/promise + +[directory] +recipe = slapos.cookbook:mkdirectory +htdocs = $${rootdirectory:srv}/www +logrotate-entries = $${rootdirectory:etc}/logrotate.d +logrotate-backup = $${basedirectory:backup}/logrotate +cronstamps = $${rootdirectory:etc}/cronstamps +cron-entries = $${rootdirectory:etc}/cron.d +crontabs = $${rootdirectory:etc}/crontabs +ssh = $${rootdirectory:etc}/ssh +sshkeys = $${rootdirectory:srv}/sshkeys +httpd-log = $${basedirectory:log}/apache + + +#---------------- +#-- +#-- Deploy cron. + +[cron] +recipe = slapos.cookbook:cron +dcrond-binary = ${dcron:location}/sbin/crond +cron-entries = $${directory:cron-entries} +crontabs = $${directory:crontabs} +cronstamps = $${directory:cronstamps} +catcher = $${cron-simplelogger:wrapper} +binary = $${basedirectory:services}/crond + +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = $${rootdirectory:bin}/cron_simplelogger +log = $${basedirectory:log}/crond.log + + +#---------------- +#-- +#-- Deploy logrotate. + +[cron-entry-logrotate] +<= cron +recipe = slapos.cookbook:cron.d +name = logrotate +frequency = 0 0 * * * +command = $${logrotate:wrapper} + +[logrotate] +recipe = slapos.cookbook:logrotate +# Binaries +logrotate-binary = ${logrotate:location}/usr/sbin/logrotate +gzip-binary = ${gzip:location}/bin/gzip +gunzip-binary = ${gzip:location}/bin/gunzip +# Directories +wrapper = $${rootdirectory:bin}/logrotate +conf = $${rootdirectory:etc}/logrotate.conf +logrotate-entries = $${directory:logrotate-entries} +backup = $${directory:logrotate-backup} +state-file = $${rootdirectory:srv}/logrotate.status + + +#---------------- +#-- +#-- sshkeys [sshkeys-directory] recipe = slapos.cookbook:mkdirectory -requests = $${directory:sshkeys}/requests/ -keys = $${directory:sshkeys}/keys/ +requests = $${directory:sshkeys}/requests +keys = $${directory:sshkeys}/keys [sshkeys-authority] recipe = slapos.cookbook:sshkeys_authority @@ -60,6 +120,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub private-key = $${dropbear-server:rsa-keyfile} wrapper = $${basedirectory:services}/sshd + +#---------------- +#-- +#-- Dropbear. + [dropbear-server] recipe = slapos.cookbook:dropbear host = $${slap-network-information:global-ipv6} @@ -75,6 +140,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear recipe = slapos.cookbook:dropbear.add_authorized_key key = $${slap-parameter:authorized-key} + +#---------------- +#-- +#-- rdiff + [rdiff-backup-server] recipe = slapos.cookbook:pbs client = false @@ -82,18 +152,26 @@ path = $${directory:htdocs} wrapper = $${rootdirectory:bin}/rdiffbackup-server rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup -[logrotate] -recipe = slapos.cookbook:logrotate -# Binaries -logrotate-binary = ${logrotate:location}/usr/sbin/logrotate -gzip-binary = ${gzip:location}/bin/gzip -gunzip-binary = ${gzip:location}/bin/gunzip -# Directories -wrapper = $${rootdirectory:bin}/logrotate -conf = $${rootdirectory:etc}/logrotate.conf -logrotate-entries = $${directory:logrotate-entries} -backup = $${directory:logrotate-backup} -state-file = $${rootdirectory:srv}/logrotate.status + +#---------------- +#-- +#-- Apache Proxy. + +[apache-proxy] +recipe = slapos.cookbook:apacheproxy +url = $${slap-parameter:proxy-url} +pid-file = $${basedirectory:run}/apache.pid +lock-file = $${basedirectory:run}/apache.lock +ip = $${slap-network-information:global-ipv6} +port = 8080 +error-log = $${directory:httpd-log}/error.log +access-log = $${directory:httpd-log}/access.log +httpd-conf = $${rootdirectory:etc}/apache.conf +wrapper = $${basedirectory:services}/apache + +promise = $${basedirectory:promises}/apache + +httpd-binary = ${apache:location}/bin/httpd [logrotate-entry-apache] <= logrotate @@ -106,51 +184,14 @@ sharedscripts = true notifempty = true create = true -[cron] -recipe = slapos.cookbook:cron -dcrond-binary = ${dcron:location}/sbin/crond -cron-entries = $${directory:cron-entries} -crontabs = $${directory:crontabs} -cronstamps = $${directory:cronstamps} -catcher = $${cron-simplelogger:wrapper} -binary = $${basedirectory:services}/crond - -[cron-simplelogger] -recipe = slapos.cookbook:simplelogger -wrapper = $${rootdirectory:bin}/cron_simplelogger -log = $${basedirectory:log}/crond.log - -[cron-entry-logrotate] -<= cron -recipe = slapos.cookbook:cron.d -name = logrotate -frequency = 0 0 * * * -command = $${logrotate:wrapper} -[rootdirectory] -recipe = slapos.cookbook:mkdirectory -etc = $${buildout:directory}/etc/ -var = $${buildout:directory}/var/ -srv = $${buildout:directory}/srv/ -bin = $${buildout:directory}/bin/ -tmp = $${buildout:directory}/tmp/ +#---------------- +#-- +#-- Publish instance parameters. -[basedirectory] -recipe = slapos.cookbook:mkdirectory -log = $${rootdirectory:var}/log/ -services = $${rootdirectory:etc}/run/ -run = $${rootdirectory:var}/run/ -backup = $${rootdirectory:srv}/backup/ -promises = $${rootdirectory:etc}/promise/ +[urls] +recipe = slapos.cookbook:publish +url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/ +ssh-public-key = $${sshkeys-dropbear:public-key-value} +ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path} -[directory] -recipe = slapos.cookbook:mkdirectory -htdocs = $${rootdirectory:srv}/www/ -logrotate-entries = $${rootdirectory:etc}/logrotate.d/ -logrotate-backup = $${basedirectory:backup}/logrotate/ -cronstamps = $${rootdirectory:etc}/cronstamps/ -cron-entries = $${rootdirectory:etc}/cron.d/ -crontabs = $${rootdirectory:etc}/crontabs/ -ssh = $${rootdirectory:etc}/ssh/ -sshkeys = $${rootdirectory:srv}/sshkeys -httpd-log = $${basedirectory:log}/apache/ diff --git a/stack/lamp/apache/instance-apache-import.cfg b/stack/lamp/apache/instance-apache-import.cfg index a3c61383eb7cfa688d860a702f84adef6aa5518b..271da77a2bddce76c08d4656ae4613d5a7bb1a0f 100644 --- a/stack/lamp/apache/instance-apache-import.cfg +++ b/stack/lamp/apache/instance-apache-import.cfg @@ -1,6 +1,6 @@ [buildout] -extends = ${template-apache-php:output} - ${template-pbsready-export:output} +extends = ${instance-apache-php:output} + ${pbsready-export:output} parts = apache-proxy @@ -13,6 +13,7 @@ parts = dropbear-server dropbear-server-pbs-authorized-key + [apache-proxy] recipe = slapos.cookbook:apacheproxy url = $${slap-parameter:proxy-url} @@ -29,10 +30,103 @@ promise = $${basedirectory:promises}/apache httpd-binary = ${apache:location}/bin/httpd + +#---------------- +#-- +#-- Creation of all needed directories. + +[rootdirectory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +var = $${buildout:directory}/var +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin +tmp = $${buildout:directory}/tmp + +[basedirectory] +recipe = slapos.cookbook:mkdirectory +log = $${rootdirectory:var}/log +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +backup = $${rootdirectory:srv}/backup +promises = $${rootdirectory:etc}/promise + +[directory] +recipe = slapos.cookbook:mkdirectory +htdocs = $${rootdirectory:srv}/www +logrotate-entries = $${rootdirectory:etc}/logrotate.d +logrotate-backup = $${basedirectory:backup}/logrotate +cronstamps = $${rootdirectory:etc}/cronstamps +cron-entries = $${rootdirectory:etc}/cron.d +crontabs = $${rootdirectory:etc}/crontabs +ssh = $${rootdirectory:etc}/ssh +sshkeys = $${rootdirectory:srv}/sshkeys +httpd-log = $${basedirectory:log}/apache + + +#---------------- +#-- +#-- Deploy cron. + +[cron] +recipe = slapos.cookbook:cron +dcrond-binary = ${dcron:location}/sbin/crond +cron-entries = $${directory:cron-entries} +crontabs = $${directory:crontabs} +cronstamps = $${directory:cronstamps} +catcher = $${cron-simplelogger:wrapper} +binary = $${basedirectory:services}/crond + +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = $${rootdirectory:bin}/cron_simplelogger +log = $${basedirectory:log}/crond.log + + +#---------------- +#-- +#-- Deploy logrotate. + +[cron-entry-logrotate] +<= cron +recipe = slapos.cookbook:cron.d +name = logrotate +frequency = 0 0 * * * +command = $${logrotate:wrapper} + +[logrotate] +recipe = slapos.cookbook:logrotate +# Binaries +logrotate-binary = ${logrotate:location}/usr/sbin/logrotate +gzip-binary = ${gzip:location}/bin/gzip +gunzip-binary = ${gzip:location}/bin/gunzip +# Directories +wrapper = $${rootdirectory:bin}/logrotate +conf = $${rootdirectory:etc}/logrotate.conf +logrotate-entries = $${directory:logrotate-entries} +backup = $${directory:logrotate-backup} +state-file = $${rootdirectory:srv}/logrotate.status + +[logrotate-entry-apache] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = apache +log = $${apache-proxy:error-log} $${apache-proxy:access-log} +frequency = daily +rotate-num = 30 +sharedscripts = true +notifempty = true +create = true + + +#---------------- +#-- +#-- sshkeys + [sshkeys-directory] recipe = slapos.cookbook:mkdirectory -requests = $${directory:sshkeys}/requests/ -keys = $${directory:sshkeys}/keys/ +requests = $${directory:sshkeys}/requests +keys = $${directory:sshkeys}/keys [sshkeys-authority] recipe = slapos.cookbook:sshkeys_authority @@ -51,6 +145,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub private-key = $${dropbear-server:rsa-keyfile} wrapper = $${basedirectory:services}/sshd + +#---------------- +#-- +#-- Dropbear. + [dropbear-server] recipe = slapos.cookbook:dropbear host = $${slap-network-information:global-ipv6} @@ -66,6 +165,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear recipe = slapos.cookbook:dropbear.add_authorized_key key = $${slap-parameter:authorized-key} + +#---------------- +#-- +#-- rdiff + [rdiff-backup-server] recipe = slapos.cookbook:pbs client = false @@ -73,75 +177,3 @@ path = $${directory:htdocs} wrapper = $${rootdirectory:bin}/rdiffbackup-server rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup -[logrotate] -recipe = slapos.cookbook:logrotate -# Binaries -logrotate-binary = ${logrotate:location}/usr/sbin/logrotate -gzip-binary = ${gzip:location}/bin/gzip -gunzip-binary = ${gzip:location}/bin/gunzip -# Directories -wrapper = $${rootdirectory:bin}/logrotate -conf = $${rootdirectory:etc}/logrotate.conf -logrotate-entries = $${directory:logrotate-entries} -backup = $${directory:logrotate-backup} -state-file = $${rootdirectory:srv}/logrotate.status - -[logrotate-entry-apache] -<= logrotate -recipe = slapos.cookbook:logrotate.d -name = apache -log = $${apache-proxy:error-log} $${apache-proxy:access-log} -frequency = daily -rotate-num = 30 -sharedscripts = true -notifempty = true -create = true - -[cron] -recipe = slapos.cookbook:cron -dcrond-binary = ${dcron:location}/sbin/crond -cron-entries = $${directory:cron-entries} -crontabs = $${directory:crontabs} -cronstamps = $${directory:cronstamps} -catcher = $${cron-simplelogger:wrapper} -binary = $${basedirectory:services}/crond - -[cron-simplelogger] -recipe = slapos.cookbook:simplelogger -wrapper = $${rootdirectory:bin}/cron_simplelogger -log = $${basedirectory:log}/crond.log - -[cron-entry-logrotate] -<= cron -recipe = slapos.cookbook:cron.d -name = logrotate -frequency = 0 0 * * * -command = $${logrotate:wrapper} - -[rootdirectory] -recipe = slapos.cookbook:mkdirectory -etc = $${buildout:directory}/etc/ -var = $${buildout:directory}/var/ -srv = $${buildout:directory}/srv/ -bin = $${buildout:directory}/bin/ -tmp = $${buildout:directory}/tmp/ - -[basedirectory] -recipe = slapos.cookbook:mkdirectory -log = $${rootdirectory:var}/log/ -services = $${rootdirectory:etc}/run/ -run = $${rootdirectory:var}/run/ -backup = $${rootdirectory:srv}/backup/ -promises = $${rootdirectory:etc}/promise/ - -[directory] -recipe = slapos.cookbook:mkdirectory -htdocs = $${rootdirectory:srv}/www/ -logrotate-entries = $${rootdirectory:etc}/logrotate.d/ -logrotate-backup = $${basedirectory:backup}/logrotate/ -cronstamps = $${rootdirectory:etc}/cronstamps/ -cron-entries = $${rootdirectory:etc}/cron.d/ -crontabs = $${rootdirectory:etc}/crontabs/ -ssh = $${rootdirectory:etc}/ssh/ -sshkeys = $${rootdirectory:srv}/sshkeys -httpd-log = $${basedirectory:log}/apache/ diff --git a/stack/lamp/apache/instance-apache-php.cfg b/stack/lamp/apache/instance-apache-php.cfg.in similarity index 77% rename from stack/lamp/apache/instance-apache-php.cfg rename to stack/lamp/apache/instance-apache-php.cfg.in index f349cbd6946671827c44f1093079aea7a3727420..203faacc3944c2b82379833c85750bb928250196 100644 --- a/stack/lamp/apache/instance-apache-php.cfg +++ b/stack/lamp/apache/instance-apache-php.cfg.in @@ -10,7 +10,8 @@ parts = cron-entry-logrotate promise slapmonitor - slapreport + slapmonitor-xml + frontend-promise content-promise publish-connection-informations @@ -20,46 +21,98 @@ develop-eggs-directory = ${buildout:develop-eggs-directory} offline = true -# Creation of all needed directories +#---------------- +#-- +#-- Creation of all needed directories. + [rootdirectory] recipe = slapos.cookbook:mkdirectory -etc = $${buildout:directory}/etc/ -var = $${buildout:directory}/var/ -srv = $${buildout:directory}/srv/ -bin = $${buildout:directory}/bin/ -tmp = $${buildout:directory}/tmp/ +etc = $${buildout:directory}/etc +var = $${buildout:directory}/var +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin +tmp = $${buildout:directory}/tmp [basedirectory] recipe = slapos.cookbook:mkdirectory -log = $${rootdirectory:var}/log/ -services = $${rootdirectory:etc}/run/ -run = $${rootdirectory:var}/run/ -backup = $${rootdirectory:srv}/backup/ -promises = $${rootdirectory:etc}/promise/ +log = $${rootdirectory:var}/log +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +backup = $${rootdirectory:srv}/backup +promises = $${rootdirectory:etc}/promise [directory] recipe = slapos.cookbook:mkdirectory -ca-dir = $${rootdirectory:srv}/ssl/ -httpd-log = $${basedirectory:log}/apache/ -php-ini-dir = $${rootdirectory:etc}/php/ -tmp-php = $${rootdirectory:tmp}/php/ -logrotate-entries = $${rootdirectory:etc}/logrotate.d/ -logrotate-backup = $${basedirectory:backup}/logrotate/ -stunnel-conf = $${rootdirectory:etc}/stunnel/ -cronstamps = $${rootdirectory:etc}/cronstamps/ -cron-entries = $${rootdirectory:etc}/cron.d/ -crontabs = $${rootdirectory:etc}/crontabs/ +cron-entries = $${rootdirectory:etc}/cron.d +crontabs = $${rootdirectory:etc}/crontabs +cronstamps = $${rootdirectory:etc}/cronstamps +ca-dir = $${rootdirectory:srv}/ssl +httpd-log = $${basedirectory:log}/apache +php-ini-dir = $${rootdirectory:etc}/php +tmp-php = $${rootdirectory:tmp}/php +logrotate-entries = $${rootdirectory:etc}/logrotate.d +logrotate-backup = $${basedirectory:backup}/logrotate +report = $${rootdirectory:etc}/report +stunnel-conf = $${rootdirectory:etc}/stunnel +xml-report = $${rootdirectory:var}/xml_report [cadirectory] recipe = slapos.cookbook:mkdirectory -requests = $${directory:ca-dir}/requests/ -private = $${directory:ca-dir}/private/ -certs = $${directory:ca-dir}/certs/ -newcerts = $${directory:ca-dir}/newcerts/ -crl = $${directory:ca-dir}/crl/ +requests = $${directory:ca-dir}/requests +private = $${directory:ca-dir}/private +certs = $${directory:ca-dir}/certs +newcerts = $${directory:ca-dir}/newcerts +crl = $${directory:ca-dir}/crl + + +#---------------- +#-- +#-- Deploy cron. + +[cron] +recipe = slapos.cookbook:cron +dcrond-binary = ${dcron:location}/sbin/crond +cron-entries = $${directory:cron-entries} +crontabs = $${directory:crontabs} +cronstamps = $${directory:cronstamps} +catcher = $${cron-simplelogger:wrapper} +binary = $${basedirectory:services}/crond + +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = $${rootdirectory:bin}/cron_simplelogger +log = $${basedirectory:log}/crond.log + + +#---------------- +#-- +#-- Deploy logrotate. + +[cron-entry-logrotate] +<= cron +recipe = slapos.cookbook:cron.d +name = logrotate +frequency = 0 0 * * * +command = $${logrotate:wrapper} +[logrotate] +recipe = slapos.cookbook:logrotate +# Binaries +logrotate-binary = ${logrotate:location}/usr/sbin/logrotate +gzip-binary = ${gzip:location}/bin/gzip +gunzip-binary = ${gzip:location}/bin/gunzip +# Directories +wrapper = $${rootdirectory:bin}/logrotate +conf = $${rootdirectory:etc}/logrotate.conf +logrotate-entries = $${directory:logrotate-entries} +backup = $${directory:logrotate-backup} +state-file = $${rootdirectory:srv}/logrotate.status + + +#---------------- +#-- +#-- Deploy stunnel. -# Deploy stunnel [stunnel] recipe = slapos.cookbook:stunnel client = true @@ -76,8 +129,22 @@ pid-file = $${basedirectory:run}/stunnel.pid wrapper = $${rootdirectory:bin}/raw_stunnel post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate +[logrotate-entry-stunnel] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = stunnel +log = $${stunnel:log-file} +frequency = daily +rotate-num = 30 +notifempty = true +create = true +post = $${stunnel:post-rotate-script} + + +#---------------- +#-- +#-- Certificate stuff. -# Certificate stuffs [certificate-authority] recipe = slapos.cookbook:certificate_authority openssl-binary = ${openssl:location}/bin/openssl @@ -98,7 +165,10 @@ key-file = $${stunnel:key-file} cert-file = $${stunnel:cert-file} -# Request MariaDB instance and parse its URL +#---------------- +#-- +#-- Request MariaDB instance and parse its URL. + [request-mariadb] <= slap-connection recipe = slapos.cookbook:request @@ -114,7 +184,10 @@ recipe = slapos.cookbook:urlparse url = $${request-mariadb:connection-url} -# Deploy Apache + PHP application +#---------------- +#-- +#-- Deploy Apache + PHP application. + [apache-php] recipe = slapos.cookbook:apachephp source = ${application:location} @@ -142,21 +215,6 @@ mysql-database = $${mariadb-urlparse:path} mysql-host = $${stunnel:local-host} mysql-port = $${stunnel:local-port} - -# Deploy logrotate, cron, configure it -[logrotate] -recipe = slapos.cookbook:logrotate -# Binaries -logrotate-binary = ${logrotate:location}/usr/sbin/logrotate -gzip-binary = ${gzip:location}/bin/gzip -gunzip-binary = ${gzip:location}/bin/gunzip -# Directories -wrapper = $${rootdirectory:bin}/logrotate -conf = $${rootdirectory:etc}/logrotate.conf -logrotate-entries = $${directory:logrotate-entries} -backup = $${directory:logrotate-backup} -state-file = $${rootdirectory:srv}/logrotate.status - [logrotate-entry-apache] <= logrotate recipe = slapos.cookbook:logrotate.d @@ -168,40 +226,11 @@ sharedscripts = true notifempty = true create = true -[logrotate-entry-stunnel] -<= logrotate -recipe = slapos.cookbook:logrotate.d -name = stunnel -log = $${stunnel:log-file} -frequency = daily -rotate-num = 30 -notifempty = true -create = true -post = $${stunnel:post-rotate-script} - -[cron-simplelogger] -recipe = slapos.cookbook:simplelogger -wrapper = $${rootdirectory:bin}/cron_simplelogger -log = $${basedirectory:log}/crond.log - -[cron] -recipe = slapos.cookbook:cron -dcrond-binary = ${dcron:location}/sbin/crond -cron-entries = $${directory:cron-entries} -crontabs = $${directory:crontabs} -cronstamps = $${directory:cronstamps} -catcher = $${cron-simplelogger:wrapper} -binary = $${basedirectory:services}/crond -[cron-entry-logrotate] -<= cron -recipe = slapos.cookbook:cron.d -name = logrotate -frequency = 0 0 * * * -command = $${logrotate:wrapper} +#---------------- +#-- +#-- Request frontend. - -# Request frontend [request-frontend] <= slap-connection recipe = slapos.cookbook:requestoptional @@ -215,37 +244,38 @@ return = site_url config-custom_domain = $${slap-parameter:domain} -# Deploy slapmonitor +#---------------- +#-- +#-- Deploy slapmonitor. + [slapmonitor] recipe = slapos.cookbook:slapmonitor pid-file = $${basedirectory:run}/apache.pid database-path = $${basedirectory:log}/slapmonitor.db -shell-path = ${dash:location}/bin/dash slapmonitor-path = ${buildout:bin-directory}/slapmonitor path = $${basedirectory:services}/slapmonitor -[slapreport] -recipe = slapos.cookbook:slapreport -pid-file = $${basedirectory:run}/apache.pid +[slapmonitor-xml] +recipe = slapos.cookbook:slapmonitor-xml database-path = $${basedirectory:log}/slapmonitor.db -consumption-log-path = $${basedirectory:log}/instance_consumption.log -logbox-ip = 87.98.152.12 -logbox-port = 5122 -logbox-user = admin -logbox-passwd = passer -shell-path = ${dash:location}/bin/dash -slapreport-path = ${buildout:bin-directory}/slapreport -path = $${basedirectory:services}/slapreport +slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml +path = $${directory:report}/slapmonitor-xml -# Publish all instance parameters (url of instance) +#---------------- +#-- +#-- Publish instance parameters. + [publish-connection-informations] recipe = slapos.cookbook:publish backend_url = $${apache-php:url} url = $${request-frontend:connection-site_url} -# Deploy promises scripts +#---------------- +#-- +#-- Deploy promises scripts. + [promise] recipe = slapos.cookbook:check_port_listening path = $${basedirectory:promises}/apache @@ -266,6 +296,9 @@ url = $${request-frontend:connection-site_url} dash_path = ${dash:location}/bin/dash curl_path = ${curl:location}/bin/curl + + + [slap-parameter] # Default value if no domain is specified domain = @@ -274,3 +307,4 @@ logbox-ip = logbox-port = logbox-user = logbox-passwd = + diff --git a/stack/lamp/buildout.cfg b/stack/lamp/buildout.cfg index 5c9033d04eaa7c182e55c06caa2ff333928a0b6c..fd3914ea4439ad3a25ac7d5c9e1c1f3344b442ae 100644 --- a/stack/lamp/buildout.cfg +++ b/stack/lamp/buildout.cfg @@ -10,15 +10,17 @@ parts = rdiff-backup dropbear eggs - instance-recipe-egg - template + instance - template-apache-php - template-mariadb + instance-apache-php + instance-mariadb #Contains the importer and exporter recipes for mariadb - template-mariadb-import - template-mariadb-export + instance-mariadb-import + instance-mariadb-export + + instance-default-root + extends = ../resilient/buildout.cfg @@ -48,79 +50,85 @@ extends = # Compile dir is for plugins, there's no plugin in LAMP keep-compile-dir = false -[instance-recipe] -egg = slapos.cookbook -module = lamp.request - -[instance-recipe-egg] -recipe = zc.recipe.egg -eggs = ${instance-recipe:egg} - [application] recipe = hexagonit.recipe.download #If provided tarball does not containt top directory this option shall be changed to false strip-top-level-dir = true -[template] + +#---------------- +#-- Instance-level buildout profiles. + +[instance] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance.cfg -output = ${buildout:directory}/template.cfg -md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11 +url = ${:_profile_base_location_}/instance.cfg.in +output = ${buildout:directory}/instance.cfg +md5sum = 38bdcf0a8263d4a19bd6a35c0cd00340 mode = 0644 -[template-apache-php] +[instance-apache-php] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/apache/instance-apache-php.cfg -output = ${buildout:directory}/template-apache-php.cfg -md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966 +url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in +output = ${buildout:directory}/instance-apache-php.cfg +md5sum = fc29d853dcd0802dd61c60b09e898c11 mode = 0644 -[template-apache-backup] +[instance-apache-backup] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg -output = ${buildout:directory}/template-apache-backup.cfg -md5sum = cfb77ac8785e0d125a785f69a5339014 +url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in +output = ${buildout:directory}/instance-apache-backup.cfg +md5sum = 48f969d82319a9d145570f5f0fd27672 mode = 0644 [template-resilient-lamp] recipe = slapos.recipe.template:jinja2 -template = ${:_profile_base_location_}/instance-resilient.cfg -rendered = ${buildout:directory}/template-resilient.cfg +template = ${:_profile_base_location_}/template-resilient.cfg.in +rendered = ${buildout:directory}/instance-resilient.cfg -context = key templateapache template-apache-php:output +context = key templateapache instance-apache-php:output key dropbear dropbear:location key buildout buildout:bin-directory import-list = file parts template-parts:destination file replicated template-replicated:destination -md5sum = 03aafcba5c626a4a1bd180d71007be1e +md5sum = 5605ad8896c2718854bf26148c4ae940 mode = 0644 -[template-mariadb] +[instance-mariadb] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg -output = ${buildout:directory}/template-mariadb.cfg -md5sum = fa9dc10efbcf61119f4cbab37c741322 +url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg.in +output = ${buildout:directory}/instance-mariadb.cfg +md5sum = ba8dd08dfd5e6a9dc614693d066eb21d mode = 0644 -[template-mariadb-import] +[instance-mariadb-import] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg -output = ${buildout:directory}/template-mariadb-import.cfg -md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b +url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg.in +output = ${buildout:directory}/instance-mariadb-import.cfg +md5sum = ea43b8ed38a55a11b027fc283c0e718a mode = 0644 -[template-mariadb-export] +[instance-mariadb-export] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg -output = ${buildout:directory}/template-mariadb-export.cfg -md5sum = 4b7dec765265b27c8235419b82ca7b02 +url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg.in +output = ${buildout:directory}/instance-mariadb-export.cfg +md5sum = 685c8abf0f487c72273846002ec631a0 mode = 0644 -# Dummy parts in case no application configuration file is needed +[instance-default-root] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-default-root.cfg.in +output = ${buildout:directory}/instance-default-root.cfg +md5sum = 53c9020f7a0b5203f976e069e455787b +mode = 0644 + + +#---------------- +#-- Dummy parts in case no application configuration file is needed + [application-template] filename = location = @@ -128,6 +136,8 @@ location = [application-configuration] location = +#---------------- + [eggs] recipe = zc.recipe.egg eggs = diff --git a/stack/lamp/instance-default-root.cfg.in b/stack/lamp/instance-default-root.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..dd7df3d246102fac6be8e871bd823b2d02d9c983 --- /dev/null +++ b/stack/lamp/instance-default-root.cfg.in @@ -0,0 +1,10 @@ +[buildout] +parts = request-apache + +[request-apache] +<= slap-connection +recipe = slapos.cookbook:request +software-url = $${slap-connection:software-release-url} +software-type = apache +name = Apache + diff --git a/stack/lamp/instance.cfg b/stack/lamp/instance.cfg deleted file mode 100644 index 6bb628f1cb00435d0e4081b812e40b9312f7dc35..0000000000000000000000000000000000000000 --- a/stack/lamp/instance.cfg +++ /dev/null @@ -1,23 +0,0 @@ -[buildout] - -extends = - ${template-switchsoftware:output} - -parts = - switch_softwaretype - -eggs-directory = ${buildout:eggs-directory} -develop-eggs-directory = ${buildout:develop-eggs-directory} -offline = true - -[switch_softwaretype] -recipe = slapos.cookbook:softwaretype -default = ${template-apache-php:output} -resilient = ${template-resilient-lamp:rendered} -mariadb = ${template-mariadb:output} -mariadb-import = ${template-mariadb-import:output} -mariadb-export = ${template-mariadb-export:output} -pull-backup = ${template-pull-backup:output} -apache-backup = ${template-apache-backup:output} -frozen = ${template-frozen:output} - diff --git a/stack/lamp/instance.cfg.in b/stack/lamp/instance.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..f14758a2defa8702ac0e0ea0c3b53b6b4365ea55 --- /dev/null +++ b/stack/lamp/instance.cfg.in @@ -0,0 +1,27 @@ +[buildout] + +parts = + switch_softwaretype + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +[switch_softwaretype] +recipe = slapos.cookbook:softwaretype +default = ${instance-apache-php:output} +resilient = ${template-resilient-lamp:rendered} +mariadb = ${instance-mariadb:output} +mariadb-import = ${instance-mariadb-import:output} +mariadb-export = ${instance-mariadb-export:output} +apache-backup = ${instance-apache-backup:output} + +# To prepend an empty root instance: +#default = ${instance-default-root:output} +#apache = ${instance-apache-php:output} + +#frozen creates a syntax error, meaning it can keep its data. +#It's dirty as hell, it needs to be replaced. +frozen = ${instance-frozen:output} +pull-backup = ${template-pull-backup:output} + diff --git a/stack/lamp/mariadb/instance-mariadb-export.cfg b/stack/lamp/mariadb/instance-mariadb-export.cfg.in similarity index 76% rename from stack/lamp/mariadb/instance-mariadb-export.cfg rename to stack/lamp/mariadb/instance-mariadb-export.cfg.in index fd7f5b93f001e6cd0324249410f4a6d834b29210..5bdeea81de923142599bcd5780709c9029c8875b 100644 --- a/stack/lamp/mariadb/instance-mariadb-export.cfg +++ b/stack/lamp/mariadb/instance-mariadb-export.cfg.in @@ -1,6 +1,6 @@ [buildout] -extends = ${template-mariadb:output} - ${template-pbsready-export:output} +extends = ${instance-mariadb:output} + ${pbsready-export:output} parts += mariadb @@ -12,4 +12,4 @@ socket = $${mariadb:socket} user = root mydumper-binary = ${mydumper:location}/bin/mydumper database = $${mariadb:database} -import = false \ No newline at end of file +import = false diff --git a/stack/lamp/mariadb/instance-mariadb-import.cfg b/stack/lamp/mariadb/instance-mariadb-import.cfg.in similarity index 78% rename from stack/lamp/mariadb/instance-mariadb-import.cfg rename to stack/lamp/mariadb/instance-mariadb-import.cfg.in index 1b58b02a42fa185aae84ddb6bf3792d4515fdbb2..42c06973a7578f28535360fa6e5a7259b399ed6c 100644 --- a/stack/lamp/mariadb/instance-mariadb-import.cfg +++ b/stack/lamp/mariadb/instance-mariadb-import.cfg.in @@ -1,6 +1,6 @@ [buildout] -extends = ${template-mariadb:output} - ${template-pbsready-import:output} +extends = ${instance-mariadb:output} + ${pbsready-import:output} parts += mariadb diff --git a/stack/lamp/mariadb/instance-mariadb.cfg b/stack/lamp/mariadb/instance-mariadb.cfg.in similarity index 77% rename from stack/lamp/mariadb/instance-mariadb.cfg rename to stack/lamp/mariadb/instance-mariadb.cfg.in index 3ed46960997fb935498c6c5ae0c7aab1fe14f40a..df70e00b10fe002d9a38f4688c440ef73bf0a788 100644 --- a/stack/lamp/mariadb/instance-mariadb.cfg +++ b/stack/lamp/mariadb/instance-mariadb.cfg.in @@ -12,111 +12,80 @@ parts = cron cron-entry-logrotate slapmonitor - slapreport + slapmonitor-xml gzip-binary = ${gzip:location}/bin/gzip +# Define egg directories to be the one from Software Release +# (/opt/slapgrid/...) eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} offline = true -[urls] -recipe = slapos.cookbook:publish -url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database} -ip = $${slap-network-information:global-ipv6} - -[mariadb] -recipe = slapos.cookbook:mysql -# Options -recovering = false -user = user -port = 3306 -ip = $${slap-network-information:local-ipv4} -database = db +#---------------- +#-- +#-- Creation of all needed directories. -# Paths -wrapper = $${basedirectory:services}/mariadb -update-wrapper = $${basedirectory:services}/mariadb_update -logrotate-post = $${rootdirectory:bin}/mariadb_post_logrotate -data-directory = $${directory:mariadb-data} -pid-file = $${basedirectory:run}/mariadb.pid -socket = $${basedirectory:run}/mariadb.sock -error-log = $${basedirectory:log}/mariadb_error.log -conf-file = $${rootdirectory:etc}/mariadb.cnf -promise = $${basedirectory:promises}/mysql - -# Binary information -mysql-base-directory = ${mariadb:location} -mysql-binary = ${mariadb:location}/bin/mysql -mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db -mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade -mysqld-binary = ${mariadb:location}/bin/mysqld +[rootdirectory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +var = $${buildout:directory}/var +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin +[basedirectory] +recipe = slapos.cookbook:mkdirectory +log = $${rootdirectory:var}/log +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +script = $${rootdirectory:etc}/script +backup = $${rootdirectory:srv}/backup +promises = $${rootdirectory:etc}/promise -[slapmonitor] -recipe = slapos.cookbook:slapmonitor -pid-file = $${basedirectory:run}/mariadb.pid -database-path = $${basedirectory:log}/slapmonitor.db -shell-path = ${dash:location}/bin/dash -slapmonitor-path = ${buildout:bin-directory}/slapmonitor -path = $${basedirectory:services}/slapmonitor +[directory] +recipe = slapos.cookbook:mkdirectory +cron-entries = $${rootdirectory:etc}/cron.d +crontabs = $${rootdirectory:etc}/crontabs +cronstamps = $${rootdirectory:etc}/cronstamps +ca-dir = $${rootdirectory:srv}/ssl +mariadb-data = $${rootdirectory:srv}/mariadb +logrotate-entries = $${rootdirectory:etc}/logrotate.d +logrotate-backup = $${basedirectory:backup}/logrotate +report = $${rootdirectory:etc}/report +stunnel-conf = $${rootdirectory:etc}/stunnel +xml-report = $${rootdirectory:var}/xml_report + + +#---------------- +#-- +#-- Deploy cron. -[slapreport] -recipe = slapos.cookbook:slapreport -pid-file = $${basedirectory:run}/mariadb.pid -consumption-log-path = $${basedirectory:log}/instance_consumption.log -database-path = $${basedirectory:log}/slapmonitor.db -logbox-ip = 87.98.152.12 -logbox-port = 5122 -logbox-user = admin -logbox-passwd = passer -shell-path = ${dash:location}/bin/dash -slapreport-path = ${buildout:bin-directory}/slapreport -path = $${basedirectory:services}/slapreport +[cron] +recipe = slapos.cookbook:cron +dcrond-binary = ${dcron:location}/sbin/crond +cron-entries = $${directory:cron-entries} +crontabs = $${directory:crontabs} +cronstamps = $${directory:cronstamps} +catcher = $${cron-simplelogger:wrapper} +binary = $${basedirectory:services}/crond -[certificate-authority] -recipe = slapos.cookbook:certificate_authority -openssl-binary = ${openssl:location}/bin/openssl -ca-dir = $${directory:ca-dir} -requests-directory = $${cadirectory:requests} -wrapper = $${basedirectory:services}/ca -ca-private = $${cadirectory:private} -ca-certs = $${cadirectory:certs} -ca-newcerts = $${cadirectory:newcerts} -ca-crl = $${cadirectory:crl} +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = $${rootdirectory:bin}/cron_simplelogger +log = $${basedirectory:log}/crond.log -[cadirectory] -recipe = slapos.cookbook:mkdirectory -requests = $${directory:ca-dir}/requests/ -private = $${directory:ca-dir}/private/ -certs = $${directory:ca-dir}/certs/ -newcerts = $${directory:ca-dir}/newcerts/ -crl = $${directory:ca-dir}/crl/ -[ca-stunnel] -<= certificate-authority -recipe = slapos.cookbook:certificate_authority.request -executable = $${stunnel:wrapper} -wrapper = $${basedirectory:services}/stunnel -key-file = $${stunnel:key-file} -cert-file = $${stunnel:cert-file} +#---------------- +#-- +#-- Deploy logrotate. -[stunnel] -recipe = slapos.cookbook:stunnel -stunnel-binary = ${stunnel:location}/bin/stunnel -wrapper = $${rootdirectory:bin}/stunnel -log-file = $${basedirectory:log}/stunnel.log -config-file = $${directory:stunnel-conf}/stunnel.conf -key-file = $${directory:stunnel-conf}/stunnel.key -cert-file = $${directory:stunnel-conf}/stunnel.crt -pid-file = $${basedirectory:run}/stunnel.pid -local-host = $${mariadb:ip} -local-port = $${mariadb:port} -remote-host = $${slap-network-information:global-ipv6} -remote-port = 6446 -client = false -post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate +[cron-entry-logrotate] +<= cron +recipe = slapos.cookbook:cron.d +name = logrotate +frequency = 0 0 * * * +command = $${logrotate:wrapper} [logrotate] recipe = slapos.cookbook:logrotate @@ -164,53 +133,119 @@ rotate-num = 30 notifempty = true create = true -[cron] -recipe = slapos.cookbook:cron -dcrond-binary = ${dcron:location}/sbin/crond -cron-entries = $${directory:cron-entries} -crontabs = $${directory:crontabs} -cronstamps = $${directory:cronstamps} -catcher = $${cron-simplelogger:wrapper} -binary = $${basedirectory:services}/crond +#---------------- +#-- +#-- Deploy stunnel. -[cron-simplelogger] -recipe = slapos.cookbook:simplelogger -wrapper = $${rootdirectory:bin}/cron_simplelogger -log = $${basedirectory:log}/crond.log +[stunnel] +recipe = slapos.cookbook:stunnel +stunnel-binary = ${stunnel:location}/bin/stunnel +wrapper = $${rootdirectory:bin}/stunnel +log-file = $${basedirectory:log}/stunnel.log +config-file = $${directory:stunnel-conf}/stunnel.conf +key-file = $${directory:stunnel-conf}/stunnel.key +cert-file = $${directory:stunnel-conf}/stunnel.crt +pid-file = $${basedirectory:run}/stunnel.pid +local-host = $${mariadb:ip} +local-port = $${mariadb:port} +remote-host = $${slap-network-information:global-ipv6} +remote-port = 6446 +client = false +post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate -[cron-entry-logrotate] -<= cron -recipe = slapos.cookbook:cron.d -name = logrotate -frequency = 0 0 * * * -command = $${logrotate:wrapper} -[rootdirectory] -recipe = slapos.cookbook:mkdirectory -etc = $${buildout:directory}/etc/ -var = $${buildout:directory}/var/ -srv = $${buildout:directory}/srv/ -bin = $${buildout:directory}/bin/ +#---------------- +#-- +#-- Certificate stuff. -[basedirectory] -recipe = slapos.cookbook:mkdirectory -log = $${rootdirectory:var}/log/ -services = $${rootdirectory:etc}/run/ -run = $${rootdirectory:var}/run/ -script = $${rootdirectory:etc}/script/ -backup = $${rootdirectory:srv}/backup/ -promises = $${rootdirectory:etc}/promise/ +[certificate-authority] +recipe = slapos.cookbook:certificate_authority +openssl-binary = ${openssl:location}/bin/openssl +ca-dir = $${directory:ca-dir} +requests-directory = $${cadirectory:requests} +wrapper = $${basedirectory:services}/ca +ca-private = $${cadirectory:private} +ca-certs = $${cadirectory:certs} +ca-newcerts = $${cadirectory:newcerts} +ca-crl = $${cadirectory:crl} -[directory] +[cadirectory] recipe = slapos.cookbook:mkdirectory -cron-entries = $${rootdirectory:etc}/cron.d/ -crontabs = $${rootdirectory:etc}/crontabs/ -cronstamps = $${rootdirectory:etc}/cronstamps/ -ca-dir = $${rootdirectory:srv}/ssl/ -mariadb-data = $${rootdirectory:srv}/mariadb/ -logrotate-backup = $${basedirectory:backup}/logrotate/ -stunnel-conf = $${rootdirectory:etc}/stunnel/ -logrotate-entries = $${rootdirectory:etc}/logrotate.d/ +requests = $${directory:ca-dir}/requests/ +private = $${directory:ca-dir}/private/ +certs = $${directory:ca-dir}/certs/ +newcerts = $${directory:ca-dir}/newcerts/ +crl = $${directory:ca-dir}/crl/ + +[ca-stunnel] +<= certificate-authority +recipe = slapos.cookbook:certificate_authority.request +executable = $${stunnel:wrapper} +wrapper = $${basedirectory:services}/stunnel +key-file = $${stunnel:key-file} +cert-file = $${stunnel:cert-file} + + +#---------------- +#-- +#-- Creates a MariaDB configuration file, and a database. + +[mariadb] +recipe = slapos.cookbook:mysql + +# Options +recovering = false +user = user +port = 3306 +ip = $${slap-network-information:local-ipv4} +database = db + +# Paths +wrapper = $${basedirectory:services}/mariadb +update-wrapper = $${basedirectory:services}/mariadb_update +logrotate-post = $${rootdirectory:bin}/mariadb_post_logrotate +data-directory = $${directory:mariadb-data} +pid-file = $${basedirectory:run}/mariadb.pid +socket = $${basedirectory:run}/mariadb.sock +error-log = $${basedirectory:log}/mariadb_error.log +conf-file = $${rootdirectory:etc}/mariadb.cnf +promise = $${basedirectory:promises}/mysql + +# Binary information +mysql-base-directory = ${mariadb:location} +mysql-binary = ${mariadb:location}/bin/mysql +mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db +mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade +mysqld-binary = ${mariadb:location}/bin/mysqld + + +#---------------- +#-- +#-- Deploy slapmonitor. + +[slapmonitor] +recipe = slapos.cookbook:slapmonitor +pid-file = $${basedirectory:run}/mariadb.pid +database-path = $${basedirectory:log}/slapmonitor.db +slapmonitor-path = ${buildout:bin-directory}/slapmonitor +path = $${basedirectory:services}/slapmonitor + +[slapmonitor-xml] +recipe = slapos.cookbook:slapmonitor-xml +database-path = $${basedirectory:log}/slapmonitor.db +slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml +path = $${directory:report}/slapmonitor-xml + + +#---------------- +#-- +#-- Publish instance parameters. + +[urls] +recipe = slapos.cookbook:publish +url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database} +ip = $${slap-network-information:global-ipv6} + [slap-parameter] #Default value if no ssh parameters specified diff --git a/stack/lamp/instance-resilient.cfg b/stack/lamp/template-resilient.cfg.in similarity index 90% rename from stack/lamp/instance-resilient.cfg rename to stack/lamp/template-resilient.cfg.in index e2f840c7e64d63ae7924b371dee5e8decc821e7c..e6c7e4b6c929558e25e3b99860edb52bc6d2843d 100644 --- a/stack/lamp/instance-resilient.cfg +++ b/stack/lamp/template-resilient.cfg.in @@ -6,7 +6,8 @@ extends = {{templateapache}} -parts = +# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended +parts += {{ parts.replicate("mariadb","3") }} request-apache-backup-1 request-apache-backup-2 @@ -44,8 +45,8 @@ slave = false [sshkeys-directory] recipe = slapos.cookbook:mkdirectory -requests = ${directory:sshkeys}/requests/ -keys = ${directory:sshkeys}/keys/ +requests = ${directory:sshkeys}/requests +keys = ${directory:sshkeys}/keys [sshkeys-authority] recipe = slapos.cookbook:sshkeys_authority @@ -121,6 +122,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url} config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull config-frequency = 30 * * * * slave = true +sla = instance_guid +sla-instance_guid = ${request-pull-backup-server:instance_guid} [request-pull-backup-server-apache-2] <= request-pbs-common @@ -134,6 +137,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url} config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull config-frequency = 30 * * * * slave = true +sla = instance_guid +sla-instance_guid = ${request-pull-backup-server:instance_guid} [request-pull-backup-server-apache-backup-1] @@ -146,6 +151,8 @@ config-type = push config-server-key = ${request-apache-backup-1:connection-ssh-public-key} config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id} slave = true +sla = instance_guid +sla-instance_guid = ${request-pull-backup-server:instance_guid} [request-pull-backup-server-apache-backup-2] <= request-pbs-common @@ -157,8 +164,10 @@ config-type = push config-server-key = ${request-apache-backup-2:connection-ssh-public-key} config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id} slave = true +sla = instance_guid +sla-instance_guid = ${request-pull-backup-server:instance_guid} [directory] -ssh = ${rootdirectory:etc}/ssh/ +ssh = ${rootdirectory:etc}/ssh sshkeys = ${rootdirectory:srv}/sshkeys diff --git a/stack/lapp/README.txt b/stack/lapp/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..043bac8c65a8b4bc5dbe68f875cab06fbab6ba31 --- /dev/null +++ b/stack/lapp/README.txt @@ -0,0 +1,16 @@ + +LAPP stack +========== + +This fork of the LAMP stack provides: + + - a Postgres instance, with an empty database and a 'postgres' superuser. + Log rotation is handled by Postgres itself. + + - symlinks to all the postgres binaries, usable through unix socket + with no further authentication, or through ipv6 + + - a psycopg2 (postgres driver) egg + + - configuration for a maarch instance (this part should be brought outside the stack) + diff --git a/stack/lapp/apache/instance-apache-backup.cfg.in b/stack/lapp/apache/instance-apache-backup.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..5464bd05cce4316b8d63d6a1cae27b3656c9241b --- /dev/null +++ b/stack/lapp/apache/instance-apache-backup.cfg.in @@ -0,0 +1,197 @@ +[buildout] + +parts = + urls + apache-proxy + logrotate + logrotate-entry-apache + cron + cron-entry-logrotate + sshkeys-authority + sshkeys-dropbear + dropbear-server + dropbear-server-pbs-authorized-key + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + + +#---------------- +#-- +#-- Creation of all needed directories. + +[rootdirectory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +var = $${buildout:directory}/var +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin +tmp = $${buildout:directory}/tmp + +[basedirectory] +recipe = slapos.cookbook:mkdirectory +log = $${rootdirectory:var}/log +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +backup = $${rootdirectory:srv}/backup +promises = $${rootdirectory:etc}/promise + +[directory] +recipe = slapos.cookbook:mkdirectory +htdocs = $${rootdirectory:srv}/www +logrotate-entries = $${rootdirectory:etc}/logrotate.d +logrotate-backup = $${basedirectory:backup}/logrotate +cronstamps = $${rootdirectory:etc}/cronstamps +cron-entries = $${rootdirectory:etc}/cron.d +crontabs = $${rootdirectory:etc}/crontabs +ssh = $${rootdirectory:etc}/ssh +sshkeys = $${rootdirectory:srv}/sshkeys +httpd-log = $${basedirectory:log}/apache + + +#---------------- +#-- +#-- Deploy cron. + +[cron] +recipe = slapos.cookbook:cron +dcrond-binary = ${dcron:location}/sbin/crond +cron-entries = $${directory:cron-entries} +crontabs = $${directory:crontabs} +cronstamps = $${directory:cronstamps} +catcher = $${cron-simplelogger:wrapper} +binary = $${basedirectory:services}/crond + +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = $${rootdirectory:bin}/cron_simplelogger +log = $${basedirectory:log}/crond.log + + +#---------------- +#-- +#-- Deploy logrotate. + +[cron-entry-logrotate] +<= cron +recipe = slapos.cookbook:cron.d +name = logrotate +frequency = 0 0 * * * +command = $${logrotate:wrapper} + +[logrotate] +recipe = slapos.cookbook:logrotate +# Binaries +logrotate-binary = ${logrotate:location}/usr/sbin/logrotate +gzip-binary = ${gzip:location}/bin/gzip +gunzip-binary = ${gzip:location}/bin/gunzip +# Directories +wrapper = $${rootdirectory:bin}/logrotate +conf = $${rootdirectory:etc}/logrotate.conf +logrotate-entries = $${directory:logrotate-entries} +backup = $${directory:logrotate-backup} +state-file = $${rootdirectory:srv}/logrotate.status + + +#---------------- +#-- +#-- sshkeys + +[sshkeys-directory] +recipe = slapos.cookbook:mkdirectory +requests = $${directory:sshkeys}/requests +keys = $${directory:sshkeys}/keys + +[sshkeys-authority] +recipe = slapos.cookbook:sshkeys_authority +request-directory = $${sshkeys-directory:requests} +keys-directory = $${sshkeys-directory:keys} +wrapper = $${basedirectory:services}/sshkeys_authority +keygen-binary = ${dropbear:location}/bin/dropbearkey + +[sshkeys-dropbear] +<= sshkeys-authority +recipe = slapos.cookbook:sshkeys_authority.request +name = dropbear +type = rsa +executable = $${dropbear-server:wrapper} +public-key = $${dropbear-server:rsa-keyfile}.pub +private-key = $${dropbear-server:rsa-keyfile} +wrapper = $${basedirectory:services}/sshd + + +#---------------- +#-- +#-- Dropbear. + +[dropbear-server] +recipe = slapos.cookbook:dropbear +host = $${slap-network-information:global-ipv6} +port = 2222 +home = $${directory:ssh} +wrapper = $${rootdirectory:bin}/raw_sshd +shell = $${rdiff-backup-server:wrapper} +rsa-keyfile = $${directory:ssh}/server_key.rsa +dropbear-binary = ${dropbear:location}/sbin/dropbear + +[dropbear-server-pbs-authorized-key] +<= dropbear-server +recipe = slapos.cookbook:dropbear.add_authorized_key +key = $${slap-parameter:authorized-key} + + +#---------------- +#-- +#-- rdiff + +[rdiff-backup-server] +recipe = slapos.cookbook:pbs +client = false +path = $${directory:htdocs} +wrapper = $${rootdirectory:bin}/rdiffbackup-server +rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup + + +#---------------- +#-- +#-- Apache Proxy. + +[apache-proxy] +recipe = slapos.cookbook:apacheproxy +url = $${slap-parameter:proxy-url} +pid-file = $${basedirectory:run}/apache.pid +lock-file = $${basedirectory:run}/apache.lock +ip = $${slap-network-information:global-ipv6} +port = 8080 +error-log = $${directory:httpd-log}/error.log +access-log = $${directory:httpd-log}/access.log +httpd-conf = $${rootdirectory:etc}/apache.conf +wrapper = $${basedirectory:services}/apache + +promise = $${basedirectory:promises}/apache + +httpd-binary = ${apache:location}/bin/httpd + +[logrotate-entry-apache] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = apache +log = $${apache-proxy:error-log} $${apache-proxy:access-log} +frequency = daily +rotate-num = 30 +sharedscripts = true +notifempty = true +create = true + + +#---------------- +#-- +#-- Publish instance parameters. + +[urls] +recipe = slapos.cookbook:publish +url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/ +ssh-public-key = $${sshkeys-dropbear:public-key-value} +ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path} + diff --git a/stack/lapp/apache/instance-apache-php.cfg.in b/stack/lapp/apache/instance-apache-php.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..6060647e7d2d52ff95f49d5d4904b5db1b63932c --- /dev/null +++ b/stack/lapp/apache/instance-apache-php.cfg.in @@ -0,0 +1,313 @@ +[buildout] +extends = ${custom-application-deployment:path} + +parts = + certificate-authority + ca-stunnel + logrotate + logrotate-entry-apache + logrotate-entry-stunnel + cron + cron-entry-logrotate + promise + slapmonitor + slapmonitor-xml + + frontend-promise + content-promise + publish-connection-informations + ${custom-application-deployment:part-list} + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + + +#---------------- +#-- +#-- Creation of all needed directories. + +[rootdirectory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +var = $${buildout:directory}/var +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin +tmp = $${buildout:directory}/tmp + +[basedirectory] +recipe = slapos.cookbook:mkdirectory +log = $${rootdirectory:var}/log +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +backup = $${rootdirectory:srv}/backup +promises = $${rootdirectory:etc}/promise + +[directory] +recipe = slapos.cookbook:mkdirectory +cron-entries = $${rootdirectory:etc}/cron.d +crontabs = $${rootdirectory:etc}/crontabs +cronstamps = $${rootdirectory:etc}/cronstamps +ca-dir = $${rootdirectory:srv}/ssl +httpd-log = $${basedirectory:log}/apache +php-ini-dir = $${rootdirectory:etc}/php +tmp-php = $${rootdirectory:tmp}/php +logrotate-entries = $${rootdirectory:etc}/logrotate.d +logrotate-backup = $${basedirectory:backup}/logrotate +report = $${rootdirectory:etc}/report +stunnel-conf = $${rootdirectory:etc}/stunnel +xml-report = $${rootdirectory:var}/xml_report + +[cadirectory] +recipe = slapos.cookbook:mkdirectory +requests = $${directory:ca-dir}/requests +private = $${directory:ca-dir}/private +certs = $${directory:ca-dir}/certs +newcerts = $${directory:ca-dir}/newcerts +crl = $${directory:ca-dir}/crl + + +#---------------- +#-- +#-- Deploy cron. + +[cron] +recipe = slapos.cookbook:cron +dcrond-binary = ${dcron:location}/sbin/crond +cron-entries = $${directory:cron-entries} +crontabs = $${directory:crontabs} +cronstamps = $${directory:cronstamps} +catcher = $${cron-simplelogger:wrapper} +binary = $${basedirectory:services}/crond + +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = $${rootdirectory:bin}/cron_simplelogger +log = $${basedirectory:log}/crond.log + + +#---------------- +#-- +#-- Deploy logrotate. + +[cron-entry-logrotate] +<= cron +recipe = slapos.cookbook:cron.d +name = logrotate +frequency = 0 0 * * * +command = $${logrotate:wrapper} + +[logrotate] +recipe = slapos.cookbook:logrotate +# Binaries +logrotate-binary = ${logrotate:location}/usr/sbin/logrotate +gzip-binary = ${gzip:location}/bin/gzip +gunzip-binary = ${gzip:location}/bin/gunzip +# Directories +wrapper = $${rootdirectory:bin}/logrotate +conf = $${rootdirectory:etc}/logrotate.conf +logrotate-entries = $${directory:logrotate-entries} +backup = $${directory:logrotate-backup} +state-file = $${rootdirectory:srv}/logrotate.status + + +#---------------- +#-- +#-- Deploy stunnel. + +[stunnel] +recipe = slapos.cookbook:stunnel +client = true +stunnel-binary = ${stunnel:location}/bin/stunnel +remote-host = $${postgres-urlparse:host} +remote-port = $${postgres-urlparse:port} +local-host = $${slap-network-information:local-ipv4} +local-port = 33060 +log-file = $${basedirectory:log}/stunnel.log +config-file = $${directory:stunnel-conf}/stunnel.conf +key-file = $${directory:stunnel-conf}/stunnel.key +cert-file = $${directory:stunnel-conf}/stunnel.crt +pid-file = $${basedirectory:run}/stunnel.pid +wrapper = $${rootdirectory:bin}/raw_stunnel +post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate + +[logrotate-entry-stunnel] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = stunnel +log = $${stunnel:log-file} +frequency = daily +rotate-num = 30 +notifempty = true +create = true +post = $${stunnel:post-rotate-script} + + +#---------------- +#-- +#-- Certificate stuff. + +[certificate-authority] +recipe = slapos.cookbook:certificate_authority +openssl-binary = ${openssl:location}/bin/openssl +ca-dir = $${directory:ca-dir} +requests-directory = $${cadirectory:requests} +wrapper = $${basedirectory:services}/ca +ca-private = $${cadirectory:private} +ca-certs = $${cadirectory:certs} +ca-newcerts = $${cadirectory:newcerts} +ca-crl = $${cadirectory:crl} + +[ca-stunnel] +<= certificate-authority +recipe = slapos.cookbook:certificate_authority.request +executable = $${stunnel:wrapper} +wrapper = $${basedirectory:services}/stunnel +key-file = $${stunnel:key-file} +cert-file = $${stunnel:cert-file} + + +#---------------- +#-- +#-- Request Postgres instance and parse its URL. + +[request-postgres] +<= slap-connection +recipe = slapos.cookbook:request +name = Postgres +software-url = $${slap-connection:software-release-url} +software-type = postgres +return = url +sla = computer_guid +sla-computer_guid = $${slap-connection:computer-id} + +[postgres-urlparse] +recipe = slapos.cookbook:urlparse +url = $${request-postgres:connection-url} + + +#---------------- +#-- +#-- Deploy Apache + PHP application. +#-- Despite the names of mysql-* parameters, they are not really specific to mysql. + +[apache-php] +recipe = slapos.cookbook:apachephp +source = ${application:location} +template = ${application-template:location}/${application-template:filename} +configuration = ${application-configuration:location} + +htdocs = $${rootdirectory:srv}/www/ +pid-file = $${basedirectory:run}/apache.pid +lock-file = $${basedirectory:run}/apache.lock +ip = $${slap-network-information:global-ipv6} +port = 8080 +url = http://[$${:ip}]:$${:port}/ +error-log = $${directory:httpd-log}/error.log +access-log = $${directory:httpd-log}/access.log +php-ini-dir = $${directory:php-ini-dir} +tmp-dir = $${directory:tmp-php} +httpd-conf = $${rootdirectory:etc}/apache.conf +wrapper = $${basedirectory:services}/apache + +httpd-binary = ${apache:location}/bin/httpd + +mysql-username = $${postgres-urlparse:username} +mysql-password = $${postgres-urlparse:password} +mysql-database = $${postgres-urlparse:path} +mysql-host = $${stunnel:local-host} +mysql-port = $${stunnel:local-port} + +[logrotate-entry-apache] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = apache +log = $${apache-php:error-log} $${apache-php:access-log} +frequency = daily +rotate-num = 30 +sharedscripts = true +notifempty = true +create = true + + +#---------------- +#-- +#-- Request frontend. + +[request-frontend] +<= slap-connection +recipe = slapos.cookbook:requestoptional +name = Frontend +# XXX We have hardcoded SR URL here. +software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg +slave = true +config = url custom_domain +config-url = http://[$${apache-php:ip}]:$${apache-php:port}/ +return = site_url +config-custom_domain = $${slap-parameter:domain} + + +#---------------- +#-- +#-- Deploy slapmonitor. + +[slapmonitor] +recipe = slapos.cookbook:slapmonitor +pid-file = $${basedirectory:run}/apache.pid +database-path = $${basedirectory:log}/slapmonitor.db +slapmonitor-path = ${buildout:bin-directory}/slapmonitor +path = $${basedirectory:services}/slapmonitor + +[slapmonitor-xml] +recipe = slapos.cookbook:slapmonitor-xml +database-path = $${basedirectory:log}/slapmonitor.db +slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml +path = $${directory:report}/slapmonitor-xml + + +#---------------- +#-- +#-- Publish instance parameters. + +[publish-connection-informations] +recipe = slapos.cookbook:publish +backend_url = $${apache-php:url} +url = $${request-frontend:connection-site_url} + + +#---------------- +#-- +#-- Deploy promises scripts. + +[promise] +recipe = slapos.cookbook:check_port_listening +path = $${basedirectory:promises}/apache +hostname = $${apache-php:ip} +port = $${apache-php:port} + +[frontend-promise] +recipe = slapos.cookbook:check_url_available +path = $${basedirectory:promises}/frontend +url = $${request-frontend:connection-site_url} +dash_path = ${dash:location}/bin/dash +curl_path = ${curl:location}/bin/curl + +[content-promise] +recipe = slapos.cookbook:check_page_content +path = $${basedirectory:promises}/content +url = $${request-frontend:connection-site_url} +dash_path = ${dash:location}/bin/dash +curl_path = ${curl:location}/bin/curl + + + + +[slap-parameter] +# Default value if no domain is specified +domain = +# Default value if no ssh parameter is specified +logbox-ip = +logbox-port = +logbox-user = +logbox-passwd = + diff --git a/stack/lapp/buildout.cfg b/stack/lapp/buildout.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0caefb1b337f98216f5377f2e3ebb8bcca1fc281 --- /dev/null +++ b/stack/lapp/buildout.cfg @@ -0,0 +1,168 @@ +[buildout] +parts = + apache-php-postgres + rdiff-backup + dropbear + eggs + instance + psycopg2 + + instance-apache-php + instance-postgres + +#Contains the importer and exporter recipes for postgres + instance-postgres-import + instance-postgres-export + + instance-default-root + + +extends = + ../resilient/buildout.cfg + ../../component/apache/buildout.cfg + ../../component/apache-php/buildout.cfg + ../../component/dash/buildout.cfg + ../../component/dcron/buildout.cfg + ../../component/gzip/buildout.cfg + ../../component/logrotate/buildout.cfg + ../../component/lxml-python/buildout.cfg + ../../component/postgresql/buildout.cfg + ../../component/rdiff-backup/buildout.cfg + ../../component/stunnel/buildout.cfg + ../../component/dropbear/buildout.cfg + ../slapos.cfg + + +#---------------- +#-- Application-specific part (maarch, etc.) + +[application] +recipe = hexagonit.recipe.download +strip-top-level-dir = true + + +#---------------- +#-- Instance-level buildout profiles. + +[instance] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance.cfg.in +output = ${buildout:directory}/instance.cfg +md5sum = 1aaf3ea7b14e09e66904bdb80e3cfe2f +mode = 0644 + +[instance-apache-php] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in +output = ${buildout:directory}/instance-apache-php.cfg +md5sum = bed286b680bd8cd494da080cdc229f1e +mode = 0644 + +[instance-apache-backup] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in +output = ${buildout:directory}/instance-apache-backup.cfg +md5sum = 48f969d82319a9d145570f5f0fd27672 +mode = 0644 + +[template-resilient-lapp] +recipe = slapos.recipe.template:jinja2 +template = ${:_profile_base_location_}/template-resilient.cfg.in +rendered = ${buildout:directory}/instance-resilient.cfg + +context = key templateapache instance-apache-php:output + key dropbear dropbear:location + key buildout buildout:bin-directory + +import-list = file parts template-parts:destination + file replicated template-replicated:destination + +md5sum = 525f50e60d0a96557a552de6afa4ab88 +mode = 0644 + + +[instance-postgres] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/postgres/instance-postgres.cfg.in +output = ${buildout:directory}/instance-postgres.cfg +md5sum = c5cd2a644fcd8450bc5d13bf53ec9f7d +mode = 0644 + +[instance-postgres-import] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/postgres/instance-postgres-import.cfg.in +output = ${buildout:directory}/instance-postgres-import.cfg +md5sum = 1989ba2164dd5f79793a04e0a02ea515 +mode = 0644 + +[instance-postgres-export] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/postgres/instance-postgres-export.cfg.in +output = ${buildout:directory}/instance-postgres-export.cfg +md5sum = 7bce31bc22a731a8fc6119aee96586f5 +mode = 0644 + + +#---------------- +#-- Postgres driver for Python recipes. + +[psycopg2-env] +PATH = ${postgresql:location}/bin:%(PATH)s + + +[psycopg2] +recipe = zc.recipe.egg:custom +egg = psycopg2 +define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3 +environment = psycopg2-env +include-dirs = + ${postgresql:location}/include +library-dirs = + ${postgresql:location}/lib +rpath = + ${postgresql:location}/lib + +[instance-default-root] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-default-root.cfg.in +output = ${buildout:directory}/instance-default-root.cfg +md5sum = 53c9020f7a0b5203f976e069e455787b +mode = 0644 + +#---------------- +#-- +#-- Optional part allowing applications using this stack to run a custom +#-- instance.cfg at the end of Apache/PHP instance deployment. +#-- To use it in your application, just override those two parameters, like: + +[custom-application-deployment] +# path = /path/to/instance-custom.cfg +# part-list = part1 part2 +# See software/maarch/software.cfg for an example. +path = +part-list = + +#---------------- +#-- Dummy parts in case no application configuration file is needed + +[application-template] +filename = +location = + +[application-configuration] +location = + +#---------------- + +[eggs] +recipe = zc.recipe.egg +eggs = + ${lxml-python:egg} + ${psycopg2:egg} + slapos.toolbox + cns.recipe.symlink + + +[versions] +meld3 = 0.6.10 + diff --git a/stack/lapp/instance-default-root.cfg.in b/stack/lapp/instance-default-root.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..dd7df3d246102fac6be8e871bd823b2d02d9c983 --- /dev/null +++ b/stack/lapp/instance-default-root.cfg.in @@ -0,0 +1,10 @@ +[buildout] +parts = request-apache + +[request-apache] +<= slap-connection +recipe = slapos.cookbook:request +software-url = $${slap-connection:software-release-url} +software-type = apache +name = Apache + diff --git a/stack/lapp/instance.cfg.in b/stack/lapp/instance.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..e6e1c26e697fc472d5cbb155eb584b6bca8bb3ac --- /dev/null +++ b/stack/lapp/instance.cfg.in @@ -0,0 +1,24 @@ +[buildout] + +parts = + switch_softwaretype + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +[switch_softwaretype] +recipe = slapos.cookbook:softwaretype +default = ${instance-default-root:output} +apache = ${instance-apache-php:output} +resilient = ${template-resilient-lapp:rendered} +postgres = ${instance-postgres:output} +postgres-import = ${instance-postgres-import:output} +postgres-export = ${instance-postgres-export:output} +apache-backup = ${instance-apache-backup:output} + +#frozen creates a syntax error, meaning it can keep its data. +#It's dirty as hell, it needs to be replaced. +frozen = ${instance-frozen:output} +pull-backup = ${template-pull-backup:output} + diff --git a/stack/lapp/postgres/instance-postgres-export.cfg.in b/stack/lapp/postgres/instance-postgres-export.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..9d54971c99d75364ddcc9bb9a087f17f607a1bd9 --- /dev/null +++ b/stack/lapp/postgres/instance-postgres-export.cfg.in @@ -0,0 +1,18 @@ +[buildout] +extends = ${instance-postgres:output} + ${pbsready-export:output} + + +parts += + urls + postgres-instance + postgres-promise + + +[exporter] +recipe = slapos.cookbook:postgres.export +wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter +bin = $${postgres-instance:bin} +pgdata-directory = $${postgres-instance:pgdata-directory} +backup-directory = $${postgres-instance:backup-directory} +dbname = $${postgres-instance:dbname} diff --git a/stack/lapp/postgres/instance-postgres-import.cfg.in b/stack/lapp/postgres/instance-postgres-import.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..5b2ac52a9eebcb928d5f54b2d91c90e4c378beb9 --- /dev/null +++ b/stack/lapp/postgres/instance-postgres-import.cfg.in @@ -0,0 +1,15 @@ +[buildout] +extends = ${instance-postgres:output} + ${pbsready-import:output} + + +parts += + postgres-instance + +[importer] +recipe = slapos.cookbook:postgres.import +wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-importer +bin = $${postgres-instance:bin} +pgdata-directory = $${postgres-instance:pgdata-directory} +backup-directory = $${postgres-instance:backup-directory} +dbname = $${postgres-instance:dbname} diff --git a/stack/lapp/postgres/instance-postgres.cfg.in b/stack/lapp/postgres/instance-postgres.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..58473dd82826e3207b08cccc5b59771e3876b93c --- /dev/null +++ b/stack/lapp/postgres/instance-postgres.cfg.in @@ -0,0 +1,254 @@ +[buildout] +parts = + urls + postgres-instance + postgres-promise + stunnel + certificate-authority + ca-stunnel + logrotate + logrotate-entry-stunnel + logrotate-entry-cron + cron + cron-entry-logrotate + slapmonitor + slapmonitor-xml + +gzip-binary = ${gzip:location}/bin/gzip + +# Define egg directories to be the one from Software Release +# (/opt/slapgrid/...) +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + + +#---------------- +#-- +#-- Creation of all needed directories. + +[rootdirectory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +var = $${buildout:directory}/var +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin + +[basedirectory] +recipe = slapos.cookbook:mkdirectory +log = $${rootdirectory:var}/log +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +script = $${rootdirectory:etc}/script +backup = $${rootdirectory:srv}/backup +promises = $${rootdirectory:etc}/promise + +[directory] +recipe = slapos.cookbook:mkdirectory +cron-entries = $${rootdirectory:etc}/cron.d +crontabs = $${rootdirectory:etc}/crontabs +cronstamps = $${rootdirectory:etc}/cronstamps +ca-dir = $${rootdirectory:srv}/ssl +logrotate-backup = $${basedirectory:backup}/logrotate +report = $${rootdirectory:etc}/report +stunnel-conf = $${rootdirectory:etc}/stunnel +logrotate-entries = $${rootdirectory:etc}/logrotate.d +xml-report = $${rootdirectory:var}/xml_report + + +#---------------- +#-- +#-- Deploy cron. + +[cron] +recipe = slapos.cookbook:cron +dcrond-binary = ${dcron:location}/sbin/crond +cron-entries = $${directory:cron-entries} +crontabs = $${directory:crontabs} +cronstamps = $${directory:cronstamps} +catcher = $${cron-simplelogger:wrapper} +binary = $${basedirectory:services}/crond + +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = $${rootdirectory:bin}/cron_simplelogger +log = $${basedirectory:log}/crond.log + + +#---------------- +#-- +#-- Deploy logrotate. + +[cron-entry-logrotate] +<= cron +recipe = slapos.cookbook:cron.d +name = logrotate +frequency = 0 0 * * * +command = $${logrotate:wrapper} + +[logrotate] +recipe = slapos.cookbook:logrotate +# Binaries +logrotate-binary = ${logrotate:location}/usr/sbin/logrotate +gzip-binary = $${buildout:gzip-binary} +gunzip-binary = ${gzip:location}/bin/gunzip +# Directories +wrapper = $${rootdirectory:bin}/logrotate +conf = $${rootdirectory:etc}/logrotate.conf +logrotate-entries = $${directory:logrotate-entries} +backup = $${directory:logrotate-backup} +state-file = $${rootdirectory:srv}/logrotate.status + +[logrotate-entry-stunnel] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = stunnel +log = $${stunnel:log-file} +frequency = daily +rotate-num = 30 +notifempty = true +create = true +post = $${stunnel:post-rotate-script} + +[logrotate-entry-cron] +<= logrotate +recipe =slapos.cookbook:logrotate.d +name = crond +log = $${cron-simplelogger:log} +frequency = daily +rotate-num = 30 +notifempty = true +create = true + +#---------------- +#-- +#-- Deploy stunnel. +#-- XXX This is actually not needed with Postgres. + +[stunnel] +recipe = slapos.cookbook:stunnel +stunnel-binary = ${stunnel:location}/bin/stunnel +wrapper = $${rootdirectory:bin}/stunnel +log-file = $${basedirectory:log}/stunnel.log +config-file = $${directory:stunnel-conf}/stunnel.conf +key-file = $${directory:stunnel-conf}/stunnel.key +cert-file = $${directory:stunnel-conf}/stunnel.crt +pid-file = $${basedirectory:run}/stunnel.pid +local-host = $${postgres-instance:ipv4_host} +local-port = $${postgres-instance:port} +remote-host = $${slap-network-information:global-ipv6} +remote-port = 6446 +client = false +post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate + + +#---------------- +#-- +#-- Certificate stuff. + +[certificate-authority] +recipe = slapos.cookbook:certificate_authority +openssl-binary = ${openssl:location}/bin/openssl +ca-dir = $${directory:ca-dir} +requests-directory = $${cadirectory:requests} +wrapper = $${basedirectory:services}/ca +ca-private = $${cadirectory:private} +ca-certs = $${cadirectory:certs} +ca-newcerts = $${cadirectory:newcerts} +ca-crl = $${cadirectory:crl} + +[cadirectory] +recipe = slapos.cookbook:mkdirectory +requests = $${directory:ca-dir}/requests/ +private = $${directory:ca-dir}/private/ +certs = $${directory:ca-dir}/certs/ +newcerts = $${directory:ca-dir}/newcerts/ +crl = $${directory:ca-dir}/crl/ + +#---------------- +#-- +#-- Creates a Postgres cluster, configuration files, and a database. + +[postgres-instance] +recipe = slapos.cookbook:postgres +ipv6_host = $${slap-network-information:global-ipv6} +user = postgres +port = 5432 +dbname = db +# XXX the next line is required by stunnel, not by us +ipv4_host = $${slap-network-information:local-ipv4} +# pgdata_directory is created by initdb, and should not exist beforehand. +pgdata-directory = $${rootdirectory:var}/data +backup-directory = $${basedirectory:backup}/postgres +services = $${basedirectory:services} +bin = $${rootdirectory:bin} + +dependency-symlinks = $${symlinks:recipe} +[ca-stunnel] +<= certificate-authority +recipe = slapos.cookbook:certificate_authority.request +executable = $${stunnel:wrapper} +wrapper = $${basedirectory:services}/stunnel +key-file = $${stunnel:key-file} +cert-file = $${stunnel:cert-file} + +#---------------- +#-- +#-- Creates symlinks from the instance to the software release. + +[symlinks] +recipe = cns.recipe.symlink +symlink_target = $${rootdirectory:bin} +symlink_base = ${postgresql:location}/bin + + +#---------------- +#-- +#-- Deploy slapmonitor. + +[slapmonitor] +recipe = slapos.cookbook:slapmonitor +pid-file = $${postgres-instance:pgdata-directory}/postmaster.pid +database-path = $${basedirectory:log}/slapmonitor.db +slapmonitor-path = ${buildout:bin-directory}/slapmonitor +path = $${basedirectory:services}/slapmonitor + +[slapmonitor-xml] +recipe = slapos.cookbook:slapmonitor-xml +database-path = $${basedirectory:log}/slapmonitor.db +slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml +path = $${directory:report}/slapmonitor-xml + + +#---------------- +#-- +#-- Deploy promise scripts. + +[postgres-promise] +recipe = slapos.cookbook:check_port_listening +path = $${basedirectory:promises}/postgres +hostname = $${slap-network-information:global-ipv6} +port = $${postgres-instance:port} + + +#---------------- +#-- +#-- Publish instance parameters. + +[urls] +recipe = slapos.cookbook:publish +url = $${postgres-instance:url} +ip = $${slap-network-information:global-ipv6} + +#---------------- +#-- +#-- Fetches parameters defined in SlapOS Master for this instance + +[instance-parameters] +recipe = slapos.cookbook:slapconfiguration +computer = $${slap-connection:computer-id} +partition = $${slap-connection:partition-id} +url = $${slap-connection:server-url} +key = $${slap-connection:key-file} +cert = $${slap-connection:cert-file} + diff --git a/stack/lapp/template-resilient.cfg.in b/stack/lapp/template-resilient.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..2c1f06855206308c25a5adac9277780b35be26cb --- /dev/null +++ b/stack/lapp/template-resilient.cfg.in @@ -0,0 +1,173 @@ +{% import 'parts' as parts %} +{% import 'replicated' as replicated %} + +[buildout] + +extends = + {{templateapache}} + +# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended +parts += + {{ parts.replicate("postgres","3") }} + request-apache-backup-1 + request-apache-backup-2 + + request-pull-backup-server-apache-1 + request-pull-backup-server-apache-backup-1 + + request-pull-backup-server-apache-2 + request-pull-backup-server-apache-backup-2 + + publish-connection-informations + apache-php + stunnel + certificate-authority + ca-stunnel + logrotate + logrotate-entry-apache + logrotate-entry-stunnel + cron + cron-entry-logrotate + dropbear-server + sshkeys-authority + dropbear-server-pbs-authorized-key + + request-pull-backup-server + +{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }} + + +[request-pull-backup-server] +<= request-pbs-common +name = PBS (Pull Backup Server) +return = ssh-key notification-url feeds-url +slave = false + +[sshkeys-directory] +recipe = slapos.cookbook:mkdirectory +requests = ${directory:sshkeys}/requests +keys = ${directory:sshkeys}/keys + +[sshkeys-authority] +recipe = slapos.cookbook:sshkeys_authority +request-directory = ${sshkeys-directory:requests} +keys-directory = ${sshkeys-directory:keys} +wrapper = ${basedirectory:services}/sshkeys_authority +keygen-binary = {{dropbear}}/bin/dropbearkey + +[sshkeys-dropbear] +<= sshkeys-authority +recipe = slapos.cookbook:sshkeys_authority.request +name = dropbear +type = rsa +executable = ${dropbear-server:wrapper} +public-key = ${dropbear-server:rsa-keyfile}.pub +private-key = ${dropbear-server:rsa-keyfile} +wrapper = ${basedirectory:services}/sshd + +[dropbear-server] +recipe = slapos.cookbook:dropbear +host = ${slap-network-information:global-ipv6} +port = 2222 +home = ${directory:ssh} +wrapper = ${rootdirectory:bin}/raw_sshd +shell = ${rdiff-backup-server:wrapper} +rsa-keyfile = ${directory:ssh}/server_key.rsa +dropbear-binary = {{dropbear}}/sbin/dropbear + +[dropbear-server-pbs-authorized-key] +<= dropbear-server +recipe = slapos.cookbook:dropbear.add_authorized_key +key = ${request-pull-backup-server:connection-ssh-key} + +[rdiff-backup-server] +<= apache-php +recipe = slapos.cookbook:pbs +client = false +path = ${apache-php:htdocs} +wrapper = ${rootdirectory:bin}/rdiffbackup-server +rdiffbackup-binary = {{buildout}}/rdiff-backup + +[request-apache-backup-1] +<= slap-connection +recipe = slapos.cookbook:request +name = Apache Backup 1 +software-url = ${slap-connection:software-release-url} +software-type = apache-backup +return = url ssh-url ssh-public-key +config = authorized-key proxy-url +config-authorized-key = ${request-pull-backup-server:connection-ssh-key} +config-proxy-url = ${publish-connection-informations:url} + +[request-apache-backup-2] +<= slap-connection +recipe = slapos.cookbook:request +name = Apache Backup 2 +software-url = ${slap-connection:software-release-url} +software-type = apache-backup +return = url ssh-url ssh-public-key +config = authorized-key proxy-url +config-authorized-key = ${request-pull-backup-server:connection-ssh-key} +config-proxy-url = ${publish-connection-informations:url} + +[request-pull-backup-server-apache-1] +<= request-pbs-common +name = PBS pulling from Apache 1 +config = url name type server-key notify notification-id frequency +config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path} +config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache +config-type = pull +config-server-key = ${sshkeys-dropbear:public-key-value} +config-notify = ${request-pull-backup-server:connection-notification-url} +config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull +config-frequency = 30 * * * * +slave = true +sla = instance_guid +sla-instance_guid = ${request-pull-backup-server:instance_guid} + +[request-pull-backup-server-apache-2] +<= request-pbs-common +name = PBS pulling from Apache 2 +config = url name type server-key notify notification-id frequency +config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path} +config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache +config-type = pull +config-server-key = ${sshkeys-dropbear:public-key-value} +config-notify = ${request-pull-backup-server:connection-notification-url} +config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull +config-frequency = 30 * * * * +slave = true +sla = instance_guid +sla-instance_guid = ${request-pull-backup-server:instance_guid} + + +[request-pull-backup-server-apache-backup-1] +<= request-pbs-common +name = PBS pushing to ${request-apache-backup-1:name} +config = url name type server-key on-notification +config-url = ${request-apache-backup-1:connection-ssh-url} +config-name = ${request-pull-backup-server-apache-1:config-name} +config-type = push +config-server-key = ${request-apache-backup-1:connection-ssh-public-key} +config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id} +slave = true +sla = instance_guid +sla-instance_guid = ${request-pull-backup-server:instance_guid} + +[request-pull-backup-server-apache-backup-2] +<= request-pbs-common +name = PBS pushing to ${request-apache-backup-2:name} +config = url name type server-key on-notification +config-url = ${request-apache-backup-2:connection-ssh-url} +config-name = ${request-pull-backup-server-apache-2:config-name} +config-type = push +config-server-key = ${request-apache-backup-2:connection-ssh-public-key} +config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id} +slave = true +sla = instance_guid +sla-instance_guid = ${request-pull-backup-server:instance_guid} + + +[directory] +ssh = ${rootdirectory:etc}/ssh +sshkeys = ${rootdirectory:srv}/sshkeys diff --git a/stack/resilient/buildout.cfg b/stack/resilient/buildout.cfg index 6843bc75ff9dc5d88d382317c08be108b31c7874..9c7a3b47fb4161e26d15c97dfffb815ee8a6a8e8 100644 --- a/stack/resilient/buildout.cfg +++ b/stack/resilient/buildout.cfg @@ -1,77 +1,81 @@ [buildout] parts = -#Templates needed to setup automatic backup - template-pbsready - template-pbsready-import - template-pbsready-export + pbsready + pbsready-import + pbsready-export template-replicated template-parts - -#Frozen is the state used to not destroy a broken instance's content - template-frozen - + instance-frozen template-resilient - template-switchsoftware -[template-pbsready] +#---------------- +#-- +#-- Profiles needed to setup automated backup and recovery. +#-- + +[pbsready] +# Common parts for pbsready-import and pbsready-export. +# Provides rdiff-backup, notification queue, ssh authentication, +# dropbear server, and the bully script. recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance-pbsready.cfg -output = ${buildout:directory}/template-pbsready.cfg -md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99 +url = ${:_profile_base_location_}/pbsready.cfg.in +output = ${buildout:directory}/pbsready.cfg +md5sum = a2edaadfe652b4b131626b4801768f40 mode = 0644 -[template-pbsready-import] +[pbsready-import] +# An import instance has an importer script, which is called +# by the parent PBS instance when the dump content is propagated. recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance-pbsready-import.cfg -output = ${buildout:directory}/template-pbsready-import.cfg -md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4 +url = ${:_profile_base_location_}/pbsready-import.cfg.in +output = ${buildout:directory}/pbsready-import.cfg +md5sum = eda0c1574d8991f4f9e08e3707c2b04b mode = 0644 -[template-pbsready-export] +[pbsready-export] +# An export instance has an exporter script, and communicates +# to parent PBS instances to deliver the exported dump. recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance-pbsready-export.cfg -output = ${buildout:directory}/template-pbsready-export.cfg -md5sum = 29d36aac2008b173cb9ce5da9e88c0fa +url = ${:_profile_base_location_}/pbsready-export.cfg.in +output = ${buildout:directory}/pbsready-export.cfg +md5sum = dd56f9c74e580475a17a9afb1d220390 mode = 0644 [template-pull-backup] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance-pull-backup.cfg -output = ${buildout:directory}/template-pull-backup.cfg -md5sum = f88cc9192a63c88f83a9e5191075534e +url = ${:_profile_base_location_}/instance-pull-backup.cfg.in +output = ${buildout:directory}/instance-pull-backup.cfg +md5sum = 18b88cd012e886fbaa457b03928c2d10 mode = 0644 [template-replicated] recipe = slapos.recipe.download -url = ${:_profile_base_location_}/template-replicated.cfg -md5sum = 61780842ccda1600a102456b0da69ac9 +url = ${:_profile_base_location_}/template-replicated.cfg.in +md5sum = 63b5649f3cf1c9a77315382793d9593f mode = 0644 -destination = ${buildout:directory}/template-replicated.cfg +destination = ${buildout:directory}/template-replicated.cfg.in [template-parts] recipe = slapos.recipe.download -url = ${:_profile_base_location_}/template-parts.cfg -md5sum = f5fc27235725f05fdbde76a78ebc363e +url = ${:_profile_base_location_}/template-parts.cfg.in +md5sum = c942f82552fcb42fc74a5f896e0cd5f3 mode = 0644 -destination = ${buildout:directory}/template-parts.cfg +destination = ${buildout:directory}/template-parts.cfg.in -[template-frozen] +[instance-frozen] +# When an instance is detected as broken, its software type is changed to "frozen". +# On the next run of slapgrid-cp, the buildout profile is replaced by instance-frozen.cfg, +# which will run without removing any content because it raises an error. recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance-frozen.cfg -output = ${buildout:directory}/template-frozen.cfg +url = ${:_profile_base_location_}/instance-frozen.cfg.in +output = ${buildout:directory}/instance-frozen.cfg [template-resilient] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/resilient.cfg +url = ${:_profile_base_location_}/resilient.cfg.in output = ${buildout:directory}/resilient.cfg md5sum = 59e74d290d623de2c1e147e48f284fba mode = 0644 -[template-switchsoftware] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/switchsoftware.cfg -output = ${buildout:directory}/switchsoftware.cfg -md5sum = c94a0ed85fce2e72254ae956dce7e40d -mode = 0644 \ No newline at end of file diff --git a/stack/resilient/instance-frozen.cfg b/stack/resilient/instance-frozen.cfg.in similarity index 100% rename from stack/resilient/instance-frozen.cfg rename to stack/resilient/instance-frozen.cfg.in diff --git a/stack/resilient/instance-pull-backup.cfg b/stack/resilient/instance-pull-backup.cfg.in similarity index 67% rename from stack/resilient/instance-pull-backup.cfg rename to stack/resilient/instance-pull-backup.cfg.in index 90ae9733beffa39fdfc123e7f54ae3efbf70f6bf..6879b44060633b16c84e1b7380f31ae072ae021b 100644 --- a/stack/resilient/instance-pull-backup.cfg +++ b/stack/resilient/instance-pull-backup.cfg.in @@ -1,26 +1,63 @@ [buildout] parts = - connection-dict - pbs - cron - cron-entry-logrotate - logrotate - sshkeys-authority - sshkeys-dropbear + connection-dict + pbs + logrotate + cron + cron-entry-logrotate + sshkeys-authority + sshkeys-dropbear eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} offline = true -[connection-dict] -recipe = slapos.cookbook:publish -ssh-key = $${sshkeys-dropbear:public-key-value} -notification-url = http://[$${notifier:host}]:$${notifier:port}/notify -feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/ +#---------------- +#-- +#-- Creation of all needed directories. + +[rootdirectory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +home = $${buildout:directory}/home +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin +tmp = $${buildout:directory}/tmp +var = $${buildout:directory}/var + +[basedirectory] +recipe = slapos.cookbook:mkdirectory +log = $${rootdirectory:var}/log +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +backup = $${rootdirectory:srv}/backup +promises = $${rootdirectory:etc}/promise +ssh-home = $${rootdirectory:home}/ssh +notifier = $${rootdirectory:etc}/notifier + +[directory] +recipe = slapos.cookbook:mkdirectory +logrotate-entries = $${rootdirectory:etc}/logrotate.d +logrotate-backup = $${basedirectory:backup}/logrotate +cronstamps = $${rootdirectory:etc}/cronstamps +cron-entries = $${rootdirectory:etc}/cron.d +crontabs = $${rootdirectory:etc}/crontabs +cronoutput = $${basedirectory:log}/cron-ouput +pbs-backup = $${basedirectory:backup}/pbs +sshkeys = $${rootdirectory:srv}/sshkeys +pbs-wrappers = $${rootdirectory:bin}/pbs +dot-ssh = $${basedirectory:ssh-home}/.ssh +notifier-feeds = $${basedirectory:notifier}/feeds +notifier-callbacks = $${basedirectory:notifier}/callbacks + + + +#---------------- +#-- +#-- Set up the equeue and notifier. -## sets up the equeue for the notifier [equeue] recipe = slapos.cookbook:equeue socket = $${basedirectory:run}/equeue.sock @@ -29,7 +66,8 @@ database = $${rootdirectory:srv}/equeue.db wrapper = $${basedirectory:services}/equeue equeue-binary = ${buildout:bin-directory}/equeue - +# notifier.notify adds the [exporter, notifier] to the execution queue +# notifier.notify.callback sets up a callback [notifier] recipe = slapos.cookbook:notifier feeds = $${directory:notifier-feeds} @@ -42,40 +80,20 @@ wrapper = $${basedirectory:services}/notifier server-binary = ${buildout:bin-directory}/pubsubserver notifier-binary = ${buildout:bin-directory}/pubsubnotifier +[logrotate-entry-equeue] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = equeue +log = $${equeue:log} +frequency = daily +rotate-num = 30 -## Dropbear Client to provide ssh -[dropbear-client] -recipe = slapos.cookbook:dropbear.client -dbclient-binary = ${dropbear:location}/bin/dbclient -wrapper = $${rootdirectory:bin}/ssh -home = $${basedirectory:ssh-home} -identity-file = $${basedirectory:ssh-home}/id_rsa - -[sshkeys-directory] -recipe = slapos.cookbook:mkdirectory -requests = $${directory:sshkeys}/requests/ -keys = $${directory:sshkeys}/keys/ - -[sshkeys-authority] -recipe = slapos.cookbook:sshkeys_authority -request-directory = $${sshkeys-directory:requests} -keys-directory = $${sshkeys-directory:keys} -keygen-binary = ${dropbear:location}/bin/dropbearkey -wrapper = $${basedirectory:services}/sshkeys_authority -[sshkeys-dropbear] -<= sshkeys-authority -recipe = slapos.cookbook:sshkeys_authority.request -name = pbs -type = rsa -executable = $${dropbear-client:wrapper} -public-key = $${dropbear-client:identity-file}.pub -private-key = $${dropbear-client:identity-file} -wrapper = $${rootdirectory:bin}/do_backup +#---------------- +#-- +#-- The pull-backup-server contains every backup (incremental) +#-- to prevent a corrupt dump from destroying everything. - -## The pull-backup-server contains every backup (incremental). -## to prevent a corrupt dump from destroying everything. [pbs] <= notifier recipe = slapos.cookbook:pbs @@ -91,10 +109,9 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port}/ slave-instance-list = $${slap-parameter:slave_instance_list} -[cron-simplelogger] -recipe = slapos.cookbook:simplelogger -wrapper = $${rootdirectory:bin}/cron_simplelogger -log = $${basedirectory:log}/crond.log +#---------------- +#-- +#-- Deploy cron. [cron] recipe = slapos.cookbook:cron @@ -105,6 +122,16 @@ cronstamps = $${directory:cronstamps} catcher = $${cron-simplelogger:wrapper} binary = $${basedirectory:services}/crond +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = $${rootdirectory:bin}/cron_simplelogger +log = $${basedirectory:log}/crond.log + + +#---------------- +#-- +#-- Deploy logrotate. + [cron-entry-logrotate] <= cron recipe = slapos.cookbook:cron.d @@ -115,7 +142,7 @@ command = $${logrotate:wrapper} [logrotate] recipe = slapos.cookbook:logrotate # Binaries -logrotate-binary = ${logrotate:location}/sbin/logrotate +logrotate-binary = ${logrotate:location}/usr/sbin/logrotate gzip-binary = ${gzip:location}/bin/gzip gunzip-binary = ${gzip:location}/bin/gunzip # Directories @@ -125,15 +152,7 @@ logrotate-entries = $${directory:logrotate-entries} backup = $${directory:logrotate-backup} state-file = $${rootdirectory:srv}/logrotate.status -[logrotate-entry-equeue] -<= logrotate -recipe = slapos.cookbook:logrotate.d -name = equeue -log = $${equeue:log} -frequency = daily -rotate-num = 30 - -[logrotate-entry-equeue] +[logrotate-entry-cron] <= logrotate recipe = slapos.cookbook:logrotate.d name = cron @@ -142,45 +161,61 @@ frequency = daily rotate-num = 30 -[rootdirectory] -recipe = slapos.cookbook:mkdirectory -etc = $${buildout:directory}/etc/ -var = $${buildout:directory}/var/ -srv = $${buildout:directory}/srv/ -bin = $${buildout:directory}/bin/ -tmp = $${buildout:directory}/tmp/ +#---------------- +#-- +#-- sshkeys -[basedirectory] +[sshkeys-directory] recipe = slapos.cookbook:mkdirectory -log = $${rootdirectory:var}/log/ -services = $${rootdirectory:etc}/run/ -run = $${rootdirectory:var}/run/ -backup = $${rootdirectory:srv}/backup/ -promises = $${rootdirectory:etc}/promise/ +requests = $${directory:sshkeys}/requests +keys = $${directory:sshkeys}/keys -[directory] -recipe = slapos.cookbook:mkdirectory -cronstamps = $${rootdirectory:etc}/cronstamps/ -cron-entries = $${rootdirectory:etc}/cron.d/ -crontabs = $${rootdirectory:etc}/crontabs/ -cronoutput = $${basedirectory:log}/cron-ouput/ -pbs-backup = $${basedirectory:backup}/pbs/ -logrotate-entries = $${rootdirectory:etc}/logrotate.d/ -logrotate-backup = $${basedirectory:backup}/logrotate/ -sshkeys = $${rootdirectory:srv}/sshkeys -pbs-wrappers = $${rootdirectory:bin}/pbs/ -dot-ssh = $${basedirectory:ssh-home}/.ssh/ -notifier-feeds = $${basedirectory:notifier}/feeds/ -notifier-callbacks = $${basedirectory:notifier}/callbacks/ +[sshkeys-authority] +recipe = slapos.cookbook:sshkeys_authority +request-directory = $${sshkeys-directory:requests} +keys-directory = $${sshkeys-directory:keys} +wrapper = $${basedirectory:services}/sshkeys_authority +keygen-binary = ${dropbear:location}/bin/dropbearkey -[basedirectory] -ssh-home = $${rootdirectory:home}/ssh -notifier = $${rootdirectory:etc}/notifier/ +[sshkeys-dropbear] +<= sshkeys-authority +recipe = slapos.cookbook:sshkeys_authority.request +name = pbs +type = rsa +executable = $${dropbear-client:wrapper} +public-key = $${dropbear-client:identity-file}.pub +private-key = $${dropbear-client:identity-file} +wrapper = $${rootdirectory:bin}/do_backup -[rootdirectory] -home = $${buildout:directory}/home/ +#---------------- +#-- +#-- Dropbear. + +[dropbear-client] +recipe = slapos.cookbook:dropbear.client +dbclient-binary = ${dropbear:location}/bin/dbclient +wrapper = $${rootdirectory:bin}/ssh +home = $${basedirectory:ssh-home} +identity-file = $${basedirectory:ssh-home}/id_rsa + + +#---------------- +#-- +#-- Slave instance list (empty default). -# Default values [slap-parameter] slave_instance_list = [] + + + +#---------------- +#-- +#-- Publish instance parameters. + +[connection-dict] +recipe = slapos.cookbook:publish +ssh-key = $${sshkeys-dropbear:public-key-value} +notification-url = http://[$${notifier:host}]:$${notifier:port}/notify +feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/ + diff --git a/stack/resilient/instance-pbsready-export.cfg b/stack/resilient/pbsready-export.cfg.in similarity index 65% rename from stack/resilient/instance-pbsready-export.cfg rename to stack/resilient/pbsready-export.cfg.in index bcbea3c47caaa4d2810c99ed0229e555debe1e10..773636b6930e8c18149064ce4273e10efeb0e90f 100644 --- a/stack/resilient/instance-pbsready-export.cfg +++ b/stack/resilient/pbsready-export.cfg.in @@ -1,13 +1,15 @@ [buildout] -extends = ${template-pbsready:output} + +extends = ${pbsready:output} parts += cron-entry-backup [urls] notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name} -#notify launches executable, and once it's done, notifies the pull-backup-servers. [notifier-exporter] +# notifier.notify launches an (exporter) executable, and when finished, +# notifies the the pull-backup-servers. <= notifier recipe = slapos.cookbook:notifier.notify name = exporter @@ -16,8 +18,9 @@ executable = $${exporter:wrapper} wrapper = $${rootdirectory:bin}/exporter notify = $${slap-parameter:notify} -#adds the exporter to cron [cron-entry-backup] +# Schedule the periodic database dump. +# Through notifications, this triggers (one or more) incremental backups on PBS instances. <= cron recipe = slapos.cookbook:cron.d name = backup diff --git a/stack/resilient/instance-pbsready-import.cfg b/stack/resilient/pbsready-import.cfg.in similarity index 70% rename from stack/resilient/instance-pbsready-import.cfg rename to stack/resilient/pbsready-import.cfg.in index 845397a7c84c0985214a258e77903915bde1040f..4ad23f4352f1e03104bf398226efbd0afcccb4d2 100644 --- a/stack/resilient/instance-pbsready-import.cfg +++ b/stack/resilient/pbsready-import.cfg.in @@ -1,14 +1,15 @@ [buildout] -extends = ${template-pbsready:output} +extends = ${pbsready:output} parts += import-on-notification [urls] notification-url = http://[$${notifier:host}]:$${notifier:port}/notify -#Launches callback, when a notification is received [import-on-notification] +# notifier.callback runs a script when a notification (sent by a parent PBS) +# is received <= notifier recipe = slapos.cookbook:notifier.callback on-notification-id = $${slap-parameter:on-notification} diff --git a/stack/resilient/instance-pbsready.cfg b/stack/resilient/pbsready.cfg.in similarity index 71% rename from stack/resilient/instance-pbsready.cfg rename to stack/resilient/pbsready.cfg.in index d4e366dcc02e73e45673cb9a890922aac56cd3cb..052aeaef1a439b33c3d634eab23eff4f8a94138b 100644 --- a/stack/resilient/instance-pbsready.cfg +++ b/stack/resilient/pbsready.cfg.in @@ -1,6 +1,6 @@ [buildout] -parts = +parts += resiliency urls stunnel @@ -18,21 +18,99 @@ parts = dropbear-server-pbs-authorized-key notifier -# adds the resiliency script for the bully algorithm + +#---------------- +#-- +#-- Creation of all needed directories. + +[basedirectory] +services = $${rootdirectory:etc}/run +cache = $${rootdirectory:var}/cache +notifier = $${rootdirectory:etc}/notifier + +[directory] +backup = $${basedirectory:backup}/$${slap-parameter:namebase} +ssh = $${rootdirectory:etc}/ssh/ +sshkeys = $${rootdirectory:srv}/sshkeys +notifier-feeds = $${basedirectory:notifier}/feeds +notifier-callbacks = $${basedirectory:notifier}/callbacks + + +#---------------- +#-- +#-- Resiliency script for the bully algorithm + [resiliency] +# If enable-bully-service is true, the scripts will be run automatically. +# If false, they can be run with bin/bullly for all the PBSReady instances. +enable-bully-service = False recipe = slapos.cookbook:addresiliency -script = $${basedirectory:script} -run = $${basedirectory:services} +wrapper-bully = bully +wrapper-takeover = takeover +services = $${basedirectory:services} +bin = $${rootdirectory:bin} +etc = $${rootdirectory:etc} + + +#---------------- +#-- +#-- Sets up an rdiff-backup server (with a dropbear server for ssh) -# sets up an rdiff-backup server (with a dropbear server for ssh) [urls] ssh-public-key = $${sshkeys-dropbear:public-key-value} ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path} +[rdiff-backup-server] +recipe = slapos.cookbook:pbs +client = false +path = $${directory:backup} +wrapper = $${rootdirectory:bin}/rdiffbackup-server +rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup + + +#---------------- +#-- +#-- Set up the equeue and notifier. + +[equeue] +recipe = slapos.cookbook:equeue +socket = $${basedirectory:run}/equeue.sock +log = $${basedirectory:log}/equeue.log +database = $${rootdirectory:srv}/equeue.db +wrapper = $${basedirectory:services}/equeue +equeue-binary = ${buildout:bin-directory}/equeue + +# notifier.notify adds the [exporter, notifier] to the execution queue +# notifier.notify.callback sets up a callback +[notifier] +recipe = slapos.cookbook:notifier +feeds = $${directory:notifier-feeds} +callbacks = $${directory:notifier-callbacks} +id-file = $${rootdirectory:etc}/notifier.id +equeue-socket = $${equeue:socket} +host = $${slap-network-information:global-ipv6} +port = 8080 +wrapper = $${basedirectory:services}/notifier +server-binary = ${buildout:bin-directory}/pubsubserver +notifier-binary = ${buildout:bin-directory}/pubsubnotifier + +[logrotate-entry-equeue] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = equeue +log = $${equeue:log} +frequency = daily +rotate-num = 30 + + +#---------------- +#-- +#-- sshkeys + [sshkeys-directory] recipe = slapos.cookbook:mkdirectory -requests = $${directory:sshkeys}/requests/ -keys = $${directory:sshkeys}/keys/ +requests = $${directory:sshkeys}/requests +keys = $${directory:sshkeys}/keys [sshkeys-authority] recipe = slapos.cookbook:sshkeys_authority @@ -51,6 +129,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub private-key = $${dropbear-server:rsa-keyfile} wrapper = $${basedirectory:services}/sshd + +#---------------- +#-- +#-- Dropbear. + [dropbear-server] recipe = slapos.cookbook:dropbear host = $${slap-network-information:global-ipv6} @@ -66,57 +149,5 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear recipe = slapos.cookbook:dropbear.add_authorized_key key = $${slap-parameter:authorized-key} -[rdiff-backup-server] -recipe = slapos.cookbook:pbs -client = false -path = $${directory:backup} -wrapper = $${rootdirectory:bin}/rdiffbackup-server -rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup - - -## Sets up the execution queue for the notifier -[logrotate-entry-equeue] -<= logrotate -recipe = slapos.cookbook:logrotate.d -name = equeue -log = $${equeue:log} -frequency = daily -rotate-num = 30 - -[equeue] -recipe = slapos.cookbook:equeue -socket = $${basedirectory:run}/equeue.sock -log = $${basedirectory:log}/equeue.log -database = $${rootdirectory:srv}/equeue.db -wrapper = $${basedirectory:services}/equeue -equeue-binary = ${buildout:bin-directory}/equeue - - -## notifier.notify adds the [exporter, notifier] to the execution queue -## notifier.notify.callback sets up a callback -[notifier] -recipe = slapos.cookbook:notifier -feeds = $${directory:notifier-feeds} -callbacks = $${directory:notifier-callbacks} -id-file = $${rootdirectory:etc}/notifier.id -equeue-socket = $${equeue:socket} -host = $${slap-network-information:global-ipv6} -port = 8080 -wrapper = $${basedirectory:services}/notifier -server-binary = ${buildout:bin-directory}/pubsubserver -notifier-binary = ${buildout:bin-directory}/pubsubnotifier - -[basedirectory] -script = $${rootdirectory:etc}/script/ -services = $${rootdirectory:etc}/run/ -cache = $${rootdirectory:var}/cache/ -notifier = $${rootdirectory:etc}/notifier/ -[directory] -backup = $${basedirectory:backup}/$${slap-parameter:namebase} -ssh = $${rootdirectory:etc}/ssh/ -sshkeys = $${rootdirectory:srv}/sshkeys -notifier-feeds = $${basedirectory:notifier}/feeds/ -notifier-callbacks = $${basedirectory:notifier}/callbacks/ -script = $${basedirectory:script} diff --git a/stack/resilient/resilient.cfg b/stack/resilient/resilient.cfg.in similarity index 100% rename from stack/resilient/resilient.cfg rename to stack/resilient/resilient.cfg.in diff --git a/stack/resilient/switchsoftware.cfg b/stack/resilient/switchsoftware.cfg deleted file mode 100644 index 1dbdd069799213dd0d9e767d7defd0f4844a8ee0..0000000000000000000000000000000000000000 --- a/stack/resilient/switchsoftware.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[buildout] -parts = - switch_softwaretype - -[switch_softwaretype] -recipe = slapos.cookbook:softwaretype -pull-backup = ${template-pull-backup:output} -#frozen creates a syntax error, meaning it can keep its data. -#It's dirty as hell, it needs to be replaced. -frozen = ${template-frozen:output} diff --git a/stack/resilient/template-parts.cfg b/stack/resilient/template-parts.cfg.in similarity index 96% rename from stack/resilient/template-parts.cfg rename to stack/resilient/template-parts.cfg.in index 9cb5d1f1ceb5963efc714063f423ad790d4e082b..5f1837ee578dfcfd90b251bf80153af082e8171a 100644 --- a/stack/resilient/template-parts.cfg +++ b/stack/resilient/template-parts.cfg.in @@ -18,4 +18,4 @@ - {% endmacro %} \ No newline at end of file + {% endmacro %} diff --git a/stack/resilient/template-replicated.cfg b/stack/resilient/template-replicated.cfg.in similarity index 91% rename from stack/resilient/template-replicated.cfg rename to stack/resilient/template-replicated.cfg.in index 1c38ce63f2c0e2cf2183532200d14a3ea72dfe98..7bd950a7cade58bae45416d69969577f58c25d48 100644 --- a/stack/resilient/template-replicated.cfg +++ b/stack/resilient/template-replicated.cfg.in @@ -2,8 +2,6 @@ ## Tells the Backupable recipe that we want a backup [resilient] -config-script = bully.py -config-wrapper = bully config-namebase = {{namebase}} ## Every request is double to provide the 3 IPs. @@ -15,14 +13,11 @@ software-type = {{typeexport}} name = {{namebase}}0 return = url ssh-public-key ssh-url notification-id ip -config = number script wrapper authorized-key notify ip-list namebase - +config = number authorized-key notify ip-list namebase +config-number = 0 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %} - config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %} - config-ip-list = -config-number = 0 {% for id in range(1,nbbackup|int) %} @@ -37,14 +32,13 @@ software-url = ${slap-connection:software-release-url} software-type = {{typeimport}} return = url ssh-public-key ssh-url notification-url ip -config = number script wrapper authorized-key on-notification ip-list namebase +pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push +config = number authorized-key on-notification ip-list namebase +config-number = {{id}} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} -pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push - config-ip-list = -config-number = {{id}} {% endfor %} @@ -60,19 +54,15 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback recipe = slapos.cookbook:request name = {{namebase}}0 -config = number script wrapper authorized-key notify ip-list namebase - software-url = ${slap-connection:software-release-url} software-type = {{typeexport}} return = url ssh-public-key ssh-url notification-id ip - +config = number authorized-key notify ip-list namebase +config-number = 0 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %} config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %} - -config-number=0 - {% for id in range(1,nbbackup|int) %} [request-{{namebase}}-pseudo-replicating-{{id}}-2] <= slap-connection @@ -87,14 +77,13 @@ software-url = ${slap-connection:software-release-url} software-type = {{typeimport}} return = url ssh-public-key ssh-url notification-url -config = number script wrapper authorized-key on-notification ip-list namebase +pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push +config = number authorized-key on-notification ip-list namebase +config-number = {{id}} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} -pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push - -config-number = {{id}} {% endfor %} @@ -135,6 +124,8 @@ config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url} config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull config-title = Pulling from {{namebase}} slave = true +sla = instance_guid +sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid} [request-pull-backup-server-{{namebase}}-backup-{{id}}] <= request-pbs-common @@ -149,6 +140,8 @@ config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-noti config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id} config-title = Pushing to {{namebase}} backup {{id}} slave = true +sla = instance_guid +sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid} {% endfor %} -{% endmacro %} \ No newline at end of file +{% endmacro %}