Merge branch 'lapp-resilient3'

component/apache-perl/buildout.cfg

+[buildout]
+parts = apache-perl
+
+extends =
+  ../apache/buildout.cfg
+  ../perl/buildout.cfg
+  ../libuuid/buildout.cfg
+
+[apache-perl]
+# Note: Shall react on each build of apache and reinstall itself
+recipe = hexagonit.recipe.cmmi
+url = http://perl.apache.org/dist/mod_perl-2.0.5.tar.gz
+md5sum = 03d01d135a122bd8cebd0cd5b185d674
+configure-command =
+  ${perl:location}/bin/perl Makefile.PL
+configure-options =
+  MP_AP_PREFIX=${apache-2.2:location}
+
+environment =
+  # CPPFLAGS=-I${libuuid:location}/include
+  MP_CCOPTS=-L${libuuid:location}/lib

component/apache-php/buildout.cfg

@@ -55,10 +55,50 @@ configure-options =
   --enable-bz2
   --enable-ftp
 
+
+# Changing TMPDIR is required for PEAR installation.
+# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
+# XXX we could mkdir tmp there
+
 environment =
   PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
   LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid
+  TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
+
+
+
+
+[apache-php-postgres]
+<=apache-php
+configure-options =
+  --with-apxs2=${apache:location}/bin/apxs
+  --with-libxml-dir=${libxml2:location}
+  --with-zlib-dir=${zlib:location}
+  --with-bz2-dir=${bzip2:location}
+  --with-mcrypt=${libmcrypt:location}
+  --with-gd
+  --with-jpeg-dir=${libjpeg:location}
+  --with-png-dir=${libpng:location}
+  --enable-gd-native-ttf
+  --with-ttf
+  --with-freetype-dir=${freetype:location}
+  --with-curl=${curl:location}
+  --with-zip-dir=${zip:location}
+  --with-imap=${cclient:location}
+  --with-iconv-dir=${libiconv:location}
+  --with-gettext=${gettext:location}
+  --with-ldap=${openldap:location}
+  --with-imap-ssl
+  --with-openssl=${openssl:location}
+  --enable-libxml
+  --enable-mbstring
+  --enable-session
+  --enable-exif
+  --enable-zip
+  --enable-bz2
+  --enable-ftp
+  --with-pgsql=${postgresql:location}
 
 
 [libmcrypt]

component/postgresql/buildout.cfg

+[buildout]
+extends = 
+  ../openssl/buildout.cfg
+  ../readline/buildout.cfg
+  ../zlib/buildout.cfg
+  ../ncurses/buildout.cfg
+parts = postgresql
+ 
+[postgresql]
+recipe = hexagonit.recipe.cmmi
+url = http://ftp.postgresql.org/pub/source/v9.1.6/postgresql-9.1.6.tar.bz2
+md5sum = 000755f66c0de58bbd4cd2b89b45b8e2
+configure-options = --with-openssl
+environment =
+  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib 

setup.py

@@ -110,6 +110,9 @@ setup(name=name,
           'novnc = slapos.recipe.novnc:Recipe',
           'onetimeupload = slapos.recipe.onetimeupload:Recipe',
           'pbs = slapos.recipe.pbs:Recipe',
+          'postgres = slapos.recipe.postgres:Recipe',
+          'postgres.import = slapos.recipe.postgres:ImportRecipe',
+          'postgres.export = slapos.recipe.postgres:ExportRecipe',
           'proactive = slapos.recipe.proactive:Recipe',
           'publish = slapos.recipe.publish:Recipe',
           'publishurl = slapos.recipe.publishurl:Recipe',
@@ -126,7 +129,8 @@ setup(name=name,
           'siptester = slapos.recipe.siptester:SipTesterRecipe',
           'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
           'slapcontainer = slapos.recipe.container:Recipe',
-          'slapmonitor = slapos.recipe.slapmonitor:Recipe',
+          'slapmonitor = slapos.recipe.slapmonitor:MonitorRecipe',
+          'slapmonitor-xml = slapos.recipe.slapmonitor:MonitorXMLRecipe',
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
           'slaprunner.test = slapos.recipe.slaprunner:Test',

slapos/recipe/addresiliency/__init__.py

@@ -26,56 +26,73 @@
 ##############################################################################
 from slapos.recipe.librecipe import GenericSlapRecipe
 
-import sys
 import os
 
 
 class Recipe(GenericSlapRecipe):
     """ This class provides the installation of the resilience
-      script on the partition.
+        scripts on the partition.
+
+        bin/takeover will perform a rename (must be run manually).
+        bin/bully will monitor, run elections and perform renames when needed.
     """
 
     def _install(self):
         path_list = []
-    self_id = int(self.parameter_dict['number'])
-    ip = self.parameter_dict['ip-list'].split(' ')
-    print 'Creating bully script with  ips : %s\n' % ip
-    slap_connection = self.buildout['slap-connection']
 
-    path_conf = os.path.join(self.options['script'], 'conf.in')
-    path_bully = os.path.join(self.options['script'], self.parameter_dict['script'])
-    path_bully_new = os.path.join(self.options['script'], 'new.py')
-    path_run = os.path.join(self.options['run'], self.parameter_dict['wrapper'])
-    print 'paths: %s\n%s\n' % (path_run, path_bully)
-    bully_conf = dict(self_id=self_id,
-                      ip_list=ip,
-                      executable=sys.executable,
-                      syspath=sys.path,
-                      server_url=slap_connection['server-url'],
-                      key_file=slap_connection.get('key-file'),
-                      cert_file=slap_connection.get('cert-file'),
-                      computer_id=slap_connection['computer-id'],
-                      partition_id=slap_connection['partition-id'],
-                      software=slap_connection['software-release-url'],
-                      namebase=self.parameter_dict['namebase'],
-                      confpath=path_conf)
-    try:
-      conf = self.createFile(path_conf,
+        confpath = os.path.join(self.options['etc'], 'bully.conf')
+
+        ip_list = self.parameter_dict['ip-list']
+        print 'Creating bully configuration with ips : %s\n' % ip_list
+
+        conf = self.createFile(confpath,
                                self.substituteTemplate(
-                             self.getTemplateFilename('conf.in.in'),
-                             bully_conf))
+                               self.getTemplateFilename('bully.conf.in'),
+                               {
+                                   'self_id': int(self.parameter_dict['number']),
+                                   'ip_list': ip_list
+                                   }
+                               ))
         path_list.append(conf)
-      script = self.createExecutable(path_bully,
-                                     self.substituteTemplate(
-                                     self.getTemplateFilename('bully.py.in'),
-                                     bully_conf))
-      path_list.append(script)
-
-      wrapper = self.createPythonScript(
-          path_run,
-          'slapos.recipe.librecipe.execute.execute',
-          [path_bully])
-      path_list.append(wrapper)
-    except IOError:
-      pass
+
+        slap_connection = self.buildout['slap-connection']
+
+        if self.optionIsTrue('enable-bully-service', default=False):
+            bully_dir = self.options['services']
+        else:
+            bully_dir = self.options['bin']
+
+        bully_wrapper = self.createPythonScript(
+            name=os.path.join(bully_dir, self.options['wrapper-bully']),
+            absolute_function='slapos.recipe.addresiliency.bully.run',
+            arguments={
+                'confpath': confpath,
+                'server_url': slap_connection['server-url'],
+                'key_file': slap_connection.get('key-file'),
+                'cert_file': slap_connection.get('cert-file'),
+                'computer_id': slap_connection['computer-id'],
+                'partition_id': slap_connection['partition-id'],
+                'software': slap_connection['software-release-url'],
+                'namebase': self.parameter_dict['namebase'],
+            })
+
+        path_list.append(bully_wrapper)
+
+        takeover_wrapper = self.createPythonScript(
+            name=os.path.join(self.options['bin'], self.options['wrapper-takeover']),
+            absolute_function='slapos.recipe.addresiliency.takeover.run',
+            arguments={
+                'server_url': slap_connection['server-url'],
+                'key_file': slap_connection.get('key-file'),
+                'cert_file': slap_connection.get('cert-file'),
+                'computer_id': slap_connection['computer-id'],
+                'partition_id': slap_connection['partition-id'],
+                'software': slap_connection['software-release-url'],
+                'namebase': self.parameter_dict['namebase'],
+            })
+
+        path_list.append(takeover_wrapper)
+
         return path_list
+
+

slapos/recipe/addresiliency/bully.py

+# -*- coding: utf-8 -*-
+
+import logging
+import Queue
+import socket
+import thread
+import time
+
+import slapos.recipe.addresiliency.renamer
+import slapos
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+BASE_PORT = 50000
+SLEEPING_MINS = 2           # XXX was 30, increase after testing
+
+MSG_PING = 'ping'
+MSG_HALT = 'halt'
+MSG_VICTORY = 'victory'
+
+MSG_OK = 'ok'
+
+STATE_NORMAL = 'normal'
+STATE_WAITINGCONFIRM = 'waitingConfirm'
+STATE_ELECTION = 'election'
+STATE_REORGANIZATION = 'reorganization'
+
+
+
+## Leader is always number 0
+
+class ResilientInstance(object):
+
+    def __init__(self, comm, renamer, confpath):
+        self.comm = comm
+        self.participant_id = 0
+        self.state = STATE_NORMAL
+        self.halter_id = 0
+        self.inElection = False
+        self.alive = True
+
+        self.mainCanal = self.comm.create_canal([MSG_PING, MSG_HALT, MSG_VICTORY])
+
+        self.renamer = renamer
+        self.okCanal = self.comm.create_canal([MSG_OK])
+        self.confpath = confpath
+        self.loadConnectionInfo()
+
+    def loadConnectionInfo(self):
+        params = open(self.confpath, 'r').readlines()
+        self.total_participants = len(params[0].split())
+        new_id = int(params[1])
+        if self.participant_id != new_id:
+            self.halter_id = new_id
+            self.participant_id = new_id 
+        log.debug('I am {} of {}'.format(self.participant_id, self.total_participants))
+
+    ## Needs to be changed to use the master
+    def aliveManagement(self):
+        while self.alive:
+            log.info('XXX sleeping for %d minutes' % SLEEPING_MINS)
+            time.sleep(SLEEPING_MINS*60)
+            if self.participant_id == 0:
+                continue
+            self.comm.send(MSG_PING, 0)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            self.election()
+
+    def listen(self):
+        while self.alive:
+            self.comm.recv()
+
+    def main(self):
+        while self.alive:
+            message, sender = self.mainCanal.get()
+            if message == MSG_PING:
+                self.comm.send(MSG_OK, sender)
+
+            elif message == MSG_HALT:
+                self.state = STATE_WAITINGCONFIRM
+                self.halter_id = int(sender)
+                self.comm.send(MSG_OK, sender)
+
+            elif message == MSG_VICTORY:
+                if int(sender) == self.halter_id and self.state == STATE_WAITINGCONFIRM:
+                    log.info('{} thinks {} is the leader'.format(self.participant_id, sender))
+                    self.comm.send(MSG_OK, sender)
+                self.state = STATE_NORMAL
+
+    def election(self):
+        self.inElection = True
+        self.loadConnectionInfo()
+        # Check if I'm the highest instance alive
+        for higher in range(self.participant_id + 1, self.total_participants):
+            self.comm.send(MSG_PING, higher)
+            message, sender = self.okCanal.get()
+            if message:
+                log.info('{} is alive ({})'.format(higher, self.participant_id))
+                self.inElection = False
+                return False
+            continue
+
+        if not self.alive:
+            return False
+
+        # I should be the new coordinator, halt those below me
+        log.info('Should be ME : {}'.format(self.participant_id))
+        self.state = STATE_ELECTION
+        self.halter_id = self.participant_id
+        ups = []
+        for lower in range(self.participant_id):
+            self.comm.send(MSG_HALT, lower)
+            message, sender = self.okCanal.get()
+            if message:
+                ups.append(lower)
+
+        #Broadcast Victory
+        self.state = STATE_REORGANIZATION
+        for up in ups:
+            self.comm.send(MSG_VICTORY, up)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            log.info('Something is wrong... let\'s start over')
+            return self.election()
+        self.state = STATE_NORMAL
+        self.active = True
+        log.info('{} Is THE LEADER'.format(self.participant_id))
+
+        self.renamer.failover()
+
+        self.inElection = False
+
+        return True
+
+
+
+class FilteredCanal(object):
+
+    def __init__(self, accept, timeout):
+        self.accept = accept
+        self.queue = Queue.Queue()
+        self.timeout = timeout
+
+    def append(self, message, sender):
+        if message in self.accept:
+            self.queue.put([message, sender])
+
+    def get(self):
+        try:
+            return self.queue.get(timeout=self.timeout)
+        except Queue.Empty:
+            return [None, None]
+
+
+
+class Wrapper(object):
+
+    def __init__(self, confpath, timeout=20):
+        self.canals = []
+        self.ips = []
+        self.participant_id = 0
+        self.timeout = timeout
+        self.confpath = confpath
+        self.getConnectionInfo()
+        self.socket = None
+
+    def getConnectionInfo(self):
+        params = open(self.confpath, 'r').readlines()
+        self.ips = params[0].split()
+        self.participant_id = int(params[1])
+        log.debug('I am {} of {}'.format(self.participant_id, self.ips))
+
+    def start(self):
+        self.getConnectionInfo()
+        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        self.socket.bind((self.ips[self.participant_id], BASE_PORT + self.participant_id))
+        self.socket.listen(5)
+
+    def send(self, message, number):
+        self.getConnectionInfo()
+        try:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            s.connect((self.ips[number], BASE_PORT + number))
+            s.send(message + (' {}\n'.format(self.participant_id)))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+            pass
+        finally:
+            s.close()
+
+    def create_canal(self, accept):
+        created = FilteredCanal(accept, self.timeout)
+        self.canals.append(created)
+        return created
+
+    def recv(self):
+        client, _ = self.socket.accept()
+        client_message = client.recv(1024)
+        if client_message:
+            message, sender = client_message.split()
+            for canal in self.canals:
+                canal.append(message, int(sender))
+
+
+
+
+def run(args):
+    confpath = args.pop('confpath')
+
+    renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
+                                                          key_file = args.pop('key_file'),
+                                                          cert_file = args.pop('cert_file'),
+                                                          computer_guid = args.pop('computer_id'),
+                                                          partition_id = args.pop('partition_id'),
+                                                          software_release = args.pop('software'),
+                                                          namebase = args.pop('namebase'))
+
+    if args:
+        raise ValueError('Unknown arguments: %s' % ', '.join(args))
+
+    wrapper = Wrapper(confpath=confpath, timeout=20)
+
+    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
+
+    # idle waiting for connection infos
+    while computer.total_participants < 2:
+        computer.loadConnectionInfo()
+        time.sleep(30)
+
+    log.info('Starting')
+
+    computer.comm.start()
+    thread.start_new_thread(computer.listen, ())
+    thread.start_new_thread(computer.aliveManagement, ())
+
+    computer.main()
+

slapos/recipe/addresiliency/renamer.py

+# -*- coding: utf-8 -*-
+
+import logging
+import time
+
+import slapos
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+
+class Renamer(object):
+    def __init__(self, server_url, key_file, cert_file, computer_guid,
+                 partition_id, software_release, namebase):
+        self.server_url = server_url
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.computer_guid = computer_guid
+        self.partition_id = partition_id
+        self.software_release = software_release
+        self.namebase = namebase
+
+
+    def _failover(self):
+        """\
+        This method does
+
+        - retrieve the broken computer partition
+        - change its reference to 'broken-...' and its software type to 'frozen'
+        - retrieve the winner computer partition (attached to this process)
+        - change its reference to replace the broken one.
+          later, slapgrid will change its software_type as well.
+
+        Then, after running slapgrid-cp a few times, the winner takes over and
+        a new cp is created to replace it as an importer.
+        """
+
+        slap = slapos.slap.slap()
+        slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
+
+        # partition that will take over.
+        cp_winner = slap.registerComputerPartition(computer_guid=self.computer_guid,
+                                                   partition_id=self.partition_id)
+        # XXX although we can already rename cp_winner, to change its software type we need to
+        # get hold of the root cp as well
+
+        cp_exporter_ref = self.namebase + '0'       # this is ok. the boss is always number zero.
+
+        # partition to be deactivated
+        cp_broken = cp_winner.request(software_release=self.software_release,
+                                      software_type='frozen',
+                                      state='stopped',
+                                      partition_reference=cp_exporter_ref)
+
+        broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
+
+        log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
+
+        cp_broken.rename(new_name=broken_new_ref)
+
+        cp_broken.stopped()
+
+        log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
+
+        # update name (and later, software type) for the partition that will take over
+
+        cp_winner.rename(new_name=cp_exporter_ref)
+        cp_winner.bang(message='partitions have been renamed!')
+
+
+
+    def failover(self):
+        try:
+            self._failover()
+            log.info('Renaming done')
+        except slapos.slap.ServerError:
+            log.info('Internal server error')
+

slapos/recipe/addresiliency/takeover.py

+# -*- coding: utf-8 -*-
+
+import slapos.recipe.addresiliency.renamer
+
+def run(args):
+    renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
+                                                          key_file = args.pop('key_file'),
+                                                          cert_file = args.pop('cert_file'),
+                                                          computer_guid = args.pop('computer_id'),
+                                                          partition_id = args.pop('partition_id'),
+                                                          software_release = args.pop('software'),
+                                                          namebase = args.pop('namebase'))
+
+    renamer.failover()
+
+

slapos/recipe/addresiliency/template/bully.py.in

@@ -8,6 +8,7 @@ import sys
 
 sys.path[:] = %(syspath)s
 
+import slapos
 from slapos import slap as slapmodule
 
 port = 50000
@@ -37,9 +38,11 @@ def rename_broken_and_stop():
     slap.initializeConnection('%(server_url)s',
                               '%(key_file)s',
                               '%(cert_file)s')
-    computer_partition = slap.registerComputerPartition('%(computer_id)s',
-                                                        '%(partition_id)s')
-    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
+    computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s',
+                                                        partition_id='%(partition_id)s')
+    broken = computer_partition.request(software_release='%(software)s',
+                                        software_type='frozen',
+                                        partition_reference='%(namebase)s0')
 
     broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
     broken.stopped()

slapos/recipe/addresiliency/template/bully_new.py.in

 #!%(executable)s
 
+import logging
+import os
 import socket
-import time
 import sys
 import thread
 import time
-import os
 
 sys.path[:] = %(syspath)s
 
 from slapos import slap as slapmodule
+import slapos
 
-port = 50000
-size = 1024
+BASE_PORT = 50000
+SLEEPING_MINS = 2
 
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
 
-def rename_broken_and_stop():
-  try:
+
+
+class Renamer(object):
+    def __init__(self, server_url, key_file, cert_file, computer_guid,
+                 partition_id, software_release, namebase):
+        self.server_url = server_url
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.computer_guid = computer_guid
+        self.partition_id = partition_id
+        self.software_release = software_release
+        self.namebase = namebase
+
+    def _failover(self):
         slap = slapmodule.slap()
-    slap.initializeConnection('%(server_url)s',
-                              '%(key_file)s',
-                              '%(cert_file)s')
-    computer_partition = slap.registerComputerPartition('%(computer_id)s',
-                                                        '%(partition_id)s')
-    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
-
-    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+        slap.initializeConnection(self.server_url,
+                                  self.key_file,
+                                  self.cert_file)
+        computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid,
+                                                            partition_id=self.partition_id)
+        broken = computer_partition.request(software_release=self.software_release,
+                                            software_type='frozen',
+                                            partition_reference=self.namebase+'0')
+
+        broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
         broken.stopped()
-    computer_partition.rename('%(namebase)s0')
-    print 'renaming done\n'
+        computer_partition.rename(self.namebase+'0')
+
+    def failover(self):
+        try:
+            log.info('renaming done')
         except slapos.slap.slap.ServerError:
-    print 'Internal server error\n'
+            log.info('Internal server error')
+
 
 
 ## Leader is always number 0
 
 class ResilientInstance(object):
 
-    def __init__(self, comm):
+    def __init__(self, comm, renamer, confpath):
         self.comm = comm
         self.id = 0
         self.state = 'normal'
         self.halter = 0
-        self.nbComp = nbComp
         self.inElection = False
         self.alive = True
         self.lastPing = time.clock()
 
-        self.mainCanal = self.comm.canal(['ping', 'halt',
-                                          'victory'])
+        self.mainCanal = self.comm.canal(['ping', 'halt', 'victory'])
 
+        self.renamer = renamer
         self.okCanal = self.comm.canal(['ok'])
+        self.confpath = confpath
         self.loadConnectionInfos()
 
-
     def loadConnectionInfos(self):
-        file = open('%(confpath)s', 'r')
+        file = open(self.confpath, 'r')
         params = file.read().split('\n')
         file.close()
         self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
@@ -67,7 +87,8 @@ class ResilientInstance(object):
     ## Needs to be changed to use the master
     def aliveManagement(self):
         while self.alive:
-            time.sleep(30*60)
+            log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS)
+            time.sleep(SLEEPING_MINS*60)
             if self.id == 0:
                 continue
             self.comm.send('ping', 0)
@@ -93,7 +114,7 @@ class ResilientInstance(object):
 
             elif message == 'victory':
                 if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
-                    print '%s thinks %s is the leader\n' % (self.id, sender)
+                    log.info('{} thinks {} is the leader'.format(self.id, sender))
                     self.comm.send('ok', sender)
                 self.state = 'normal'
 
@@ -105,7 +126,7 @@ class ResilientInstance(object):
             self.comm.send('ping', higher)
             message, sender = self.okCanal.get()
             if message:
-                #print '%s is alive (%s)\n' % (higher, self.id)
+                log.info('{} is alive ({})'.format(higher, self.id))
                 self.inElection = False
                 return False
             continue
@@ -114,7 +135,7 @@ class ResilientInstance(object):
             return False
 
         #I should be the new coordinator, halt those below me
-        print 'Should be ME : %s \n' % self.id
+        log.info('Should be ME : {}'.format(self.id))
         self.state = 'election'
         self.halter = self.id
         ups = []
@@ -131,13 +152,13 @@ class ResilientInstance(object):
             message, sender = self.okCanal.get()
             if message:
                 continue
-            print 'Something is wrong... let\'s start over\n'
+            log.info('Something is wrong... let\'s start over')
             return self.election()
         self.state = 'normal'
         self.active = True
-        print '%s Is THE LEADER \n' % self.id
+        log.info('{} Is THE LEADER'.format(self.id))
 
-        rename_broken_and_stop()
+        self.renamer.failover()
 
         self.inElection = False
 
@@ -164,27 +185,24 @@ class FilteredCanal(object):
             self.lock.acquire()
             if self.list:
                 self.lock.release()
-                val = self.list[0]
-                self.list = self.list[1:]
-                return val
+                return self.list.pop(0)
             self.lock.release()
         return [None, None]
 
 
 class Wrapper(object):
 
-    def __init__(self, timeout=20):
-        self.read_pipes = [os.fdopen(x) for x in read_pipes]
-        self.write_pipes = write_pipes
+    def __init__(self, confpath, timeout=20):
         self.canals = []
         self.ips = []
         self.id = 0
         self.timeout = timeout
+        self.confpath = confpath
         self.getConnectionInfos()
         self.socket = None
 
     def getConnectionInfos(self):
-        file = open('%(confpath)s', 'r')
+        file = open(self.confpath, 'r')
         params = file.read().split('\n')
         file.close()
         self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
@@ -193,15 +211,15 @@ class Wrapper(object):
     def start(self):
         self.getConnectionInfos()
         self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        self.socket.bind((self.ips[self.id], port + self.id))
-        s.listen(5)
+        self.socket.bind((self.ips[self.id], BASE_PORT + self.id))
+        self.socket.listen(5)
 
     def send(self, message, number):
         self.getConnectionInfos()
         try:
             s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-            s.connect((self.ips[number], port + number))
-            s.send(message + (' %s\n' % self.id))
+            s.connect((self.ips[number], BASE_PORT + number))
+            s.send(message + (' {}\n'.format(self.id)))
         except (socket.error, socket.herror, socket.gaierror, socket.timeout):
             pass
         finally:
@@ -213,31 +231,48 @@ class Wrapper(object):
         return created
 
     def recv(self):
-        client, _ = s.accept()
+        client, _ = self.socket.accept()
         client_message = client.recv(1024)
         if client_message:
             message, sender = client_message.split()
             for canal in self.canals:
-                canal.append(message, sender)
+                canal.append(message, int(sender))
+
 
 
-wrapper = createWrapper(20)
+def main():
 
-computer = ResilientInstance(wrapper)
+    renamer = Renamer(server_url = '%(server_url)s',
+                      key_file = '%(key_file)s',
+                      cert_file = '%(cert_file)s',
+                      computer_guid = '%(computer_id)s',
+                      partition_id = '%(partition_id)s',
+                      software_release = '%(software)s',
+                      namebase = '%(namebase)s')
 
-#idle waiting for connection infos
-while computer.nbComp < 2 :
+    confpath = '%(confpath)s'
+
+    wrapper = Wrapper(confpath=confpath, timeout=20)
+
+    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
+
+    #idle waiting for connection infos
+    while computer.nbComp < 2 :
         computer.loadConnectionInfos()
         time.sleep(30)
 
-print 'Starting\n'
+    log.info('Starting')
 
-computer.comm.start()
-thread.start_new_thread(computer.listen, ())
-thread.start_new_thread(computer.main, ())
-thread.start_new_thread(computer.aliveManagement, ())
+    computer.comm.start()
+    thread.start_new_thread(computer.listen, ())
+    thread.start_new_thread(computer.main, ())
+    thread.start_new_thread(computer.aliveManagement, ())
 
-while True:
+    while True:
+        # XXX tight loop
         continue
 
 
+if __name__ == '__main__':
+    main()
+

slapos/recipe/dcron.py

@@ -33,26 +33,22 @@ class Recipe(GenericBaseRecipe):
   def install(self):
     self.logger.info("Installing dcron...")
 
-    path_list = []
+    options = self.options
+    script = self.createWrapper(name=options['binary'],
+                                command=options['dcrond-binary'].strip(),
+                                parameters=[
+                                    '-s', options['cron-entries'],
+                                    '-c', options['crontabs'],
+                                    '-t', options['cronstamps'],
+                                    '-f', '-l', '5',
+                                    '-M', options['catcher']
+                                    ])
 
-    cronstamps = self.options['cronstamps']
-    cron_d = self.options['cron-entries']
-    crontabs = self.options['crontabs']
-    catcher = self.options['catcher']
-
-    binary = self.options['binary']
-
-    script = self.createPythonScript(binary,
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['dcrond-binary'].strip(), '-s', cron_d, '-c', crontabs,
-       '-t', cronstamps, '-f', '-l', '5', '-M', catcher]
-      )
-    path_list.append(script)
     self.logger.debug('Main cron executable created at : %r', script)
 
     self.logger.info("dcron successfully installed.")
 
-    return path_list
+    return [script]
 
 
 

slapos/recipe/dropbear.py

@@ -158,6 +158,24 @@ class Client(GenericBaseRecipe):
 
     return [wrapper]
 
+
+def keysplit(s):
+  """
+  Split a string like "ssh-rsa AKLFKJSL..... ssh-rsa AAAASAF...."
+  and return the individual key_type + key strings.
+  """
+  si = iter(s.split(' '))
+  while True:
+    key_type = next(si)
+    try:
+      key_value = next(si)
+    except StopIteration:
+      # odd number of elements, should not happen, yield the last one by itself
+      yield key_type
+      break
+    yield '%s %s' % (key_type, key_value)
+
+
 class AddAuthorizedKey(GenericBaseRecipe):
 
   def install(self):
@@ -167,7 +185,9 @@ class AddAuthorizedKey(GenericBaseRecipe):
     path_list.append(ssh)
 
     authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
-    for key in self.options['key'].split(' '):
+    for key in keysplit(self.options['key']):
+      # XXX key might actually be the string 'None' or 'null'
       authorized_keys.append(key)
 
     return path_list
+

slapos/recipe/equeue.py

@@ -30,17 +30,19 @@ class Recipe(GenericBaseRecipe):
 
   def install(self):
 
-    commandline = [self.options['equeue-binary']]
-    commandline.extend(['--database', self.options['database']])
-    commandline.extend(['-l', self.options['log']])
+    parameters = [
+      '--database', self.options['database'],
+      '-l', self.options['log'],
+    ]
 
     if 'loglevel' in self.options:
-      commandline.extend(['--loglevel', self.options['loglevel']])
+      parameters.extend(['--loglevel', self.options['loglevel']])
 
-    commandline.append(self.options['socket'])
+    parameters.append(self.options['socket'])
+
+    wrapper = self.createWrapper(name=self.options['wrapper'],
+                                 command=self.options['equeue-binary'],
+                                 parameters=parameters)
+
+    return [wrapper]
 
-    return [self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      commandline,
-    )]

slapos/recipe/librecipe/__init__.py

@@ -154,13 +154,6 @@ class BaseSlapRecipe:
     self._writeExecutable(wrapper_path, file_content)
     return wrapper_path
 
-  def createReportRunningWrapper(self, file_content):
-    """Creates report runnig wrapper and returns its path"""
-    report_wrapper_path = os.path.join(self.wrapper_report_directory,
-        'slapreport')
-    self._writeExecutable(report_wrapper_path, file_content)
-    return report_wrapper_path
-
   def substituteTemplate(self, template_location, mapping_dict):
     """Returns template content after substitution"""
     return open(template_location, 'r').read() % mapping_dict

slapos/recipe/librecipe/generic.py

+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
 ##############################################################################
 #
 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
@@ -24,6 +26,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+import io
 import logging
 import os
 import sys
@@ -90,6 +93,21 @@ class GenericBaseRecipe(object):
   def createExecutable(self, name, content, mode=0700):
     return self.createFile(name, content, mode)
 
+  def addLineToFile(self, filepath, line, encoding='utf8'):
+    """Append a single line to a text file, if the line does not exist yet.
+
+    line must be unicode."""
+
+    try:
+      lines = io.open(filepath, 'r', encoding=encoding).readlines()
+    except IOError:
+      lines = []
+
+    if not line in lines:
+      lines.append(line)
+      with io.open(filepath, 'w+', encoding=encoding) as f:
+        f.write(u'\n'.join(lines))
+
   def createPythonScript(self, name, absolute_function, arguments=''):
     """Create a python script using zc.buildout.easy_install.scripts
 
@@ -109,25 +127,31 @@ class GenericBaseRecipe(object):
       path, arguments=arguments)[0]
     return script
 
-  def createWrapper(self, name, command, parameters):
+  def createWrapper(self, name, command, parameters, comments=[], parameters_extra=False):
     """
     Creates a very simple (one command) shell script for process replacement.
     Takes care of quoting.
     """
 
-    q = shlex.quote
-    lines = [
-            '#!/bin/sh',
-            'exec %s' % shlex.quote(command)
-            ]
+    lines = [ '#!/bin/sh' ]
+
+    for comment in comments:
+      lines.append('# %s' % comment)
+
+    lines.append('exec %s' % shlex.quote(command))
 
     for param in parameters:
-      if len(lines[-1]) < 30:
+      if len(lines[-1]) < 40:
         lines[-1] += ' ' + shlex.quote(param)
       else:
         lines[-1] += ' \\'
         lines.append('\t' + shlex.quote(param))
 
+    if parameters_extra:
+        # pass-through further parameters
+        lines[-1] += ' \\'
+        lines.append('\t$@')
+
     content = '\n'.join(lines) + '\n'
     return self.createFile(name, content, 0700)
 

slapos/recipe/mydumper.py

@@ -28,85 +28,76 @@ import subprocess
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
-def dump(args):
-  mydumper_cmd = [args['mydumper']]
-  mydumper_cmd.extend(['-B', args['database']])
 
-  if args['socket'] is not None:
-    mydumper_cmd.extend(['-S', args['socket']])
-  else:
-    mydumper_cmd.extend(['-h', args['host']])
-    mydumper_cmd.etxned(['-P', args['port']])
-
-  mydumper_cmd.extend(['-u', args['user']])
-  if args['password'] is not None:
-    mydumper_cmd.extend(['-p', args['password']])
-
-  if args['compression']:
-    mydumper_cmd.append('--compress')
+def _mydumper_base_cmd(mydumper, database, user, password,
+                       socket=None, host=None, port=None, **kw):
+  cmd = [mydumper]
+  cmd.extend(['-B', database])
 
-  if args['rows'] is not None:
-    mydumper_cmd.extend(['-r', args['rows']])
-
-  mydumper_cmd.extend(['-o', args['directory']])
+  if socket:
+    cmd.extend(['-S', socket])
+  else:
+    cmd.extend(['-h', host])
+    cmd.extend(['-P', port])
 
-  subprocess.check_call(mydumper_cmd)
+  cmd.extend(['-u', user])
+  if password:
+    cmd.extend(['-p', password])
 
+  return cmd
 
 
-def do_import(args):
-  mydumper_cmd = [args['mydumper']]
-  mydumper_cmd.extend(['-B', args['database']])
+def do_export(args):
+  cmd = _mydumper_base_cmd(**args)
 
-  if args['socket'] is not None:
-    mydumper_cmd.extend(['-S', args['socket']])
-  else:
-    mydumper_cmd.extend(['-h', args['host']])
-    mydumper_cmd.etxned(['-P', args['port']])
+  if args['compression']:
+    cmd.append('--compress')
 
-  mydumper_cmd.extend(['-u', args['user']])
-  if args['password'] is not None:
-    mydumper_cmd.extend(['-p', args['password']])
+  if args['rows'] is not None:
+    cmd.extend(['-r', args['rows']])
 
-  mydumper_cmd.append('--overwrite-tables')
+  cmd.extend(['-o', args['directory']])
 
-  mydumper_cmd.extend(['-d', args['directory']])
+  subprocess.check_call(cmd)
 
-  subprocess.check_call(mydumper_cmd)
 
+def do_import(args):
+  cmd = _mydumper_base_cmd(**args)
+  cmd.append('--overwrite-tables')
+  cmd.extend(['-d', args['directory']])
+  subprocess.check_call(cmd)
 
 
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    # Host or socket should be defined
-    try:
-      self.options['host']
-    except:
-      self.options['socket']
-
-    config = dict(database=self.options['database'],
-                  socket=self.options.get('socket'),
-                  host=self.options.get('host'),
-                  port=self.options.get('port', 3306),
-                  directory=self.options['backup-directory'],
-                  user=self.options['user'],
-                  password=self.options.get('password'),
-                 )
-
-    name = __name__
+    config = {
+      'database': self.options['database'],
+      'directory': self.options['backup-directory'],
+      'user': self.options['user'],
+      'password': self.options.get('password'),
+    }
+
+    if self.options.get('host'):
+      config['host'] = self.options['host']
+      config['port'] = self.options.get('port', 3306)
+    elif self.options.get('socket'):
+      config['socket'] = self.options['socket']
+    else:
+      raise ValueError("host or socket must be defined")
+
     if self.optionIsTrue('import', False):
-      config.update(mydumper=self.options['myloader-binary'])
-      name += '.do_import'
+      function = do_import
+      config['mydumper'] = self.options['myloader-binary']
     else:
-      config.update(mydumper=self.options['mydumper-binary'],
-                    compression=self.optionIsTrue('compression', default=False),
-                    rows=self.options.get('rows'),
-                   )
-      name += '.dump'
+      function = do_export
+      config['mydumper'] = self.options['mydumper-binary']
+      config['compression'] = self.optionIsTrue('compression', default=False)
+      config['rows'] = self.options.get('rows')
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-                                      name,
-                                      config)
+    wrapper = self.createPythonScript(name=self.options['wrapper'],
+                                      absolute_function = '%s.%s' % (__name__, function.func_name),
+                                      arguments=config)
 
     return [wrapper]
+

slapos/recipe/notifier.py

@@ -31,62 +31,76 @@ from slapos.recipe.librecipe import GenericBaseRecipe
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    commandline = [self.options['server-binary']]
-    commandline.extend(['--callbacks', self.options['callbacks']])
-    commandline.extend(['--feeds', self.options['feeds']])
-    commandline.extend(['--equeue-socket', self.options['equeue-socket']])
-    commandline.append(self.options['host'])
-    commandline.append(self.options['port'])
+    options = self.options
+    script = self.createWrapper(name=options['wrapper'],
+                                command=options['server-binary'],
+                                parameters=[
+                                   '--callbacks', options['callbacks'],
+                                   '--feeds', options['feeds'],
+                                   '--equeue-socket', options['equeue-socket'],
+                                   options['host'], options['port']
+                                   ],
+                                comments=[
+                                    '',
+                                    'Upon receiving a notification, execute the callback(s).',
+                                    ''])
+    return [script]
 
-    return [self.createPythonScript(self.options['wrapper'],
-                                    'slapos.recipe.librecipe.execute.execute',
-                                    commandline)]
 
 class Callback(GenericBaseRecipe):
 
   def createCallback(self, notification_id, callback):
     callback_id = sha512(notification_id).hexdigest()
-    callback = self.createFile(os.path.join(self.options['callbacks'],
-                                            callback_id),
-                               callback)
-    return callback
+
+    filepath = os.path.join(self.options['callbacks'], callback_id)
+    self.addLineToFile(filepath, callback)
+    return filepath
 
   def install(self):
+    # XXX this path is returned multiple times, one for each callback that has been added.
     return [self.createCallback(self.options['on-notification-id'],
                                 self.options['callback'])]
 
 class Notify(GenericBaseRecipe):
 
-  def createNotifier(self, notifier_binary, executable, wrapper, **kwargs):
-    if not os.path.exists(kwargs['log']):
+  def createNotifier(self, notifier_binary, wrapper, executable,
+                     log, title, notification_url, feed_url):
+
+    if not os.path.exists(log):
       # Just a touch
-      open(kwargs['log'], 'w').close()
+      open(log, 'w').close()
 
-    commandline = [notifier_binary,
-                   '-l', kwargs['log'],
-                   '--title', kwargs['title'],
-                   '--feed', kwargs['feed_url'],
-                   '--notification-url']
+    parameters = [
+            '-l', log,
+            '--title', title,
+            '--feed', feed_url,
+            '--notification-url',
+            ]
+    parameters.extend(notification_url.split(' '))
+    parameters.extend(['--executable', executable])
 
-    commandline.extend(kwargs['notification_url'].split(' '))
-    commandline.extend(['--executable', executable])
+    return self.createWrapper(name=wrapper,
+                              command=notifier_binary,
+                              parameters=parameters,
+                              comments=[
+                                  '',
+                                  'Call an executable and send notification(s).',
+                                  ''])
 
-    return self.createPythonScript(wrapper,
-                                   'slapos.recipe.librecipe.execute.execute',
-                                   [str(i) for i in commandline])
 
   def install(self):
-    feedurl = self.unparseUrl(scheme='http', host=self.options['host'],
+    feed_url = self.unparseUrl(scheme='http', host=self.options['host'],
                                port=self.options['port'],
                                path='/get/%s' % self.options['name'])
 
-    script = self.createNotifier(
-      self.options['notifier-binary'],
-      wrapper=self.options['wrapper'],
-      executable=self.options['executable'],
-      log=os.path.join(self.options['feeds'], self.options['name']),
-      title=self.options['title'],
-      notification_url=self.options['notify'],
-      feed_url=feedurl,
-    )
+    log = os.path.join(self.options['feeds'], self.options['name'])
+
+    options = self.options
+    script = self.createNotifier(notifier_binary=options['notifier-binary'],
+                                 wrapper=options['wrapper'],
+                                 executable=options['executable'],
+                                 log=log,
+                                 title=options['title'],
+                                 notification_url=options['notify'],
+                                 feed_url=feed_url)
     return [script]

slapos/recipe/pbs.py

@@ -24,14 +24,15 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from json import loads as unjson
+
 from hashlib import sha512
-from urlparse import urlparse
+import inspect
+import json
 import os
+import signal
 import subprocess
 import sys
-import signal
-import inspect
+import urlparse
 
 from slapos.recipe.librecipe import GenericSlapRecipe
 from slapos.recipe.dropbear import KnownHostsFile
@@ -43,8 +44,7 @@ from slapos import slap as slapmodule
 def promise(args):
 
   def failed_ssh(partition, ssh):
-    # Bad python 2 syntax, looking forward python 3 to have print(file=)
-    print >> sys.stderr, "SSH Connection failed"
+    sys.stderr.write("SSH Connection failed\n")
     try:
       ssh.terminate()
     except:
@@ -75,16 +75,20 @@ def promise(args):
 
   slap = slapmodule.slap()
   slap.initializeConnection(args['server_url'],
-    key_file=args.get('key_file'), cert_file=args.get('cert_file'))
+                            key_file=args.get('key_file'),
+                            cert_file=args.get('cert_file'))
+
   partition = slap.registerComputerPartition(args['computer_id'],
                                              args['partition_id'])
 
+  ssh = subprocess.Popen([args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
+                         stdin=subprocess.PIPE,
+                         stdout=open(os.devnull, 'w'),
+                         stderr=open(os.devnull, 'w'))
+
   # Rdiff Backup protocol quit command
   quitcommand = 'q' + chr(255) + chr(0) * 7
-  ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
 
-  ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
-                         stdout=open(os.devnull), stderr=open(os.devnull))
   ssh.stdin.write(quitcommand)
   ssh.stdin.flush()
   ssh.stdin.close()
@@ -113,7 +117,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     promise_path = os.path.join(self.options['promises-directory'],
                                 url_hash)
-    parsed_url = urlparse(url)
+    parsed_url = urlparse.urlparse(url)
     promise_dict = self.promise_base_dict.copy()
     promise_dict.update(user=parsed_url.username,
                         host=parsed_url.hostname,
@@ -127,15 +131,17 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
     host = parsed_url.hostname
     known_hosts_file[host] = entry['server-key']
 
-    remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \
+    # XXX use -y because the host might not yet be in the
+    #     trusted hosts file until the next time slapgrid is run.
+
+    remote_schema = '%(ssh)s -y -p %%s %(user)s@%(host)s' % \
       {
         'ssh': self.options['sshclient-binary'],
         'user': parsed_url.username,
         'host': parsed_url.hostname,
       }
 
-    command = [self.options['rdiffbackup-binary']]
-    command.extend(['--remote-schema', remote_schema])
+    parameters = ['--remote-schema', remote_schema]
 
     remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
                                                'path': parsed_url.path}
@@ -144,31 +150,32 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
                                            name_hash)
 
     if entry['type'] == 'push':
-      command.extend(['--restore-as-of', 'now'])
-      command.append('--force')
-      command.extend([local_directory, remote_directory])
+      parameters.extend(['--restore-as-of', 'now'])
+      parameters.append('--force')
+      parameters.extend([local_directory, remote_directory])
+      comments = ['','Push data to a PBS *-import instance.','']
     else:
-      command.extend([remote_directory, local_directory])
+      parameters.extend([remote_directory, local_directory])
+      comments = ['','Pull data from a PBS *-export instance.','']
 
     wrapper_basepath = os.path.join(self.options['wrappers-directory'],
                                     url_hash)
 
-    wrapper_path = wrapper_basepath
     if 'notify' in entry:
-      wrapper_path = '%s_raw' % wrapper_basepath
+      wrapper_path = wrapper_basepath + '_raw'
+    else:
+      wrapper_path = wrapper_basepath
 
-    wrapper = self.createPythonScript(
-       wrapper_path,
-      'slapos.recipe.librecipe.execute.execute',
-      [str(i) for i in command]
-    )
+    wrapper = self.createWrapper(name=wrapper_path,
+                                 command=self.options['rdiffbackup-binary'],
+                                 parameters=parameters,
+                                 comments = comments)
     path_list.append(wrapper)
 
     if 'notify' in entry:
       feed_url = '%s/get/%s' % (self.options['notifier-url'],
                                 entry['notification-id'])
-      wrapper = self.createNotifier(
-        self.options['notifier-binary'],
+      wrapper = self.createNotifier(notifier_binary=self.options['notifier-binary'],
                                     wrapper=wrapper_basepath,
                                     executable=wrapper_path,
                                     log=os.path.join(self.options['feeds'], entry['notification-id']),
@@ -190,40 +197,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     return path_list
 
+
   def _install(self):
     path_list = []
 
-
     if self.optionIsTrue('client', True):
       self.logger.info("Client mode")
 
       slap_connection = self.buildout['slap-connection']
-      self.promise_base_dict = dict(
-        server_url=slap_connection['server-url'],
-        computer_id=slap_connection['computer-id'],
-        cert_file=slap_connection.get('cert-file'),
-        key_file=slap_connection.get('key-file'),
-        partition_id=slap_connection['partition-id'],
-        ssh_client=self.options['sshclient-binary'],
-      )
+      self.promise_base_dict = {
+              'server_url': slap_connection['server-url'],
+              'computer_id': slap_connection['computer-id'],
+              'cert_file': slap_connection.get('cert-file'),
+              'key_file': slap_connection.get('key-file'),
+              'partition_id': slap_connection['partition-id'],
+              'ssh_client': self.options['sshclient-binary'],
+        }
 
-      slaves = unjson(self.options['slave-instance-list'])
+      slaves = json.loads(self.options['slave-instance-list'])
       known_hosts = KnownHostsFile(self.options['known-hosts'])
       with known_hosts:
+        # XXX this API could be cleaner
         for slave in slaves:
           path_list.extend(self.add_slave(slave, known_hosts))
-
-
     else:
-      command = [self.options['rdiffbackup-binary']]
       self.logger.info("Server mode")
-      command.extend(['--restrict', self.options['path']])
-      command.append('--server')
 
-      wrapper = self.createPythonScript(
-        self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.execute',
-        command)
+      wrapper = self.createWrapper(name=self.options['wrapper'],
+                                   command=self.options['rdiffbackup-binary'],
+                                   parameters=[
+                                       '--restrict', self.options['path'],
+                                       '--server'
+                                       ])
       path_list.append(wrapper)
 
     return path_list
+

slapos/recipe/postgres/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import md5
+import os
+import subprocess
+import textwrap
+from zc.buildout import UserError
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+
+class Recipe(GenericBaseRecipe):
+    """\
+    This recipe creates:
+
+        - a Postgres cluster
+        - configuration to allow connections from IPV6 only (or unix socket)
+        - a superuser with provided name and generated password
+        - a database with provided name
+        - a foreground start script in the services directory
+
+    then adds the connection URL to the options.
+    The URL can be used as-is (ie. in sqlalchemy) or by the _urlparse.py recipe.
+    """
+
+    def fetch_ipv6_host(self, options):
+        """
+        Returns a string represtation of ipv6_host.
+        May receive a regular string, a set or a string serialized by buildout.
+        """
+        ipv6_host = options['ipv6_host']
+
+        if isinstance(ipv6_host, set):
+            return ipv6_host.pop()
+        else:
+            return ipv6_host
+
+
+    def _options(self, options):
+        options['password'] = self.generatePassword()
+        options['url'] = 'postgresql://%(user)s:%(password)s@[%(ipv4_host)s]:%(port)s/%(dbname)s' % options
+
+
+    def install(self):
+        pgdata = self.options['pgdata-directory']
+
+        if not os.path.exists(pgdata):
+            self.createCluster()
+            self.createConfig()
+            self.createDatabase()
+            self.createSuperuser()
+            self.createRunScript()
+
+        return [
+                # XXX should we really return something here?
+                # os.path.join(pgdata, 'postgresql.conf')
+                ]
+
+
+    def check_exists(self, path):
+        if not os.path.isfile(path):
+            raise IOError('File not found: %s' % path)
+
+
+    def createCluster(self):
+        initdb_binary = os.path.join(self.options['bin'], 'initdb')
+        self.check_exists(initdb_binary)
+
+        pgdata = self.options['pgdata-directory']
+
+        try:
+            subprocess.check_call([initdb_binary,
+                                   '-D', pgdata,
+                                   '-A', 'ident',
+                                   '-E', 'UTF8',
+                                   ])
+        except subprocess.CalledProcessError:
+            raise UserError('Could not create cluster directory in %s' % pgdata)
+
+
+    def createConfig(self):
+        pgdata = self.options['pgdata-directory']
+
+        with open(os.path.join(pgdata, 'postgresql.conf'), 'wb') as cfg:
+            cfg.write(textwrap.dedent("""\
+                    listen_addresses = '%s,%s'
+                    logging_collector = on
+                    log_rotation_size = 50MB
+                    max_connections = 100
+                    datestyle = 'iso, mdy'
+
+                    lc_messages = 'en_US.UTF-8'
+                    lc_monetary = 'en_US.UTF-8'
+                    lc_numeric = 'en_US.UTF-8'
+                    lc_time = 'en_US.UTF-8'
+                    default_text_search_config = 'pg_catalog.english'
+
+                    unix_socket_directory = '%s'
+                    unix_socket_permissions = 0700
+                    """ % (
+                        self.options['ipv4_host'],
+                        self.fetch_ipv6_host(self.options),
+                        pgdata,
+                        )))
+
+
+        with open(os.path.join(pgdata, 'pg_hba.conf'), 'wb') as cfg:
+            # see http://www.postgresql.org/docs/9.1/static/auth-pg-hba-conf.html
+
+            cfg.write(textwrap.dedent("""\
+                    # TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+                    # "local" is for Unix domain socket connections only (check unix_socket_permissions!)
+                    local   all             all                                     ident
+                    host    all             all             127.0.0.1/32            md5
+                    host    all             all             %s/32                   md5
+                    host    all             all             ::1/128                 md5
+                    host    all             all             %s/128                  md5
+                    """ % (self.options['ipv4_host'], self.fetch_ipv6_host(self.options))))
+
+
+    def createDatabase(self):
+        self.runPostgresCommand(cmd='CREATE DATABASE "%s"' % self.options['dbname'])
+
+
+    def createSuperuser(self):
+        """
+        Creates a Postgres superuser - other than "slapuser#" for use by the application.
+        """
+
+        # http://postgresql.1045698.n5.nabble.com/Algorithm-for-generating-md5-encrypted-password-not-found-in-documentation-td4919082.html
+
+        user = self.options['user']
+        password = self.options['password']
+
+        # encrypt the password to avoid storing in the logs
+        enc_password = 'md5' + md5.md5(password+user).hexdigest()
+
+        self.runPostgresCommand(cmd="""CREATE USER "%s" ENCRYPTED PASSWORD '%s' SUPERUSER""" % (user, enc_password))
+
+
+    def runPostgresCommand(self, cmd):
+        """
+        Executes a command in single-user mode, with no daemon running.
+
+        Multiple commands can be executed by providing newlines,
+        preceeded by backslash, between them.
+        See http://www.postgresql.org/docs/9.1/static/app-postgres.html
+        """
+
+        pgdata = self.options['pgdata-directory']
+        postgres_binary = os.path.join(self.options['bin'], 'postgres')
+
+        try:
+            p = subprocess.Popen([postgres_binary,
+                                  '--single',
+                                  '-D', pgdata,
+                                  'postgres',
+                                  ], stdin=subprocess.PIPE)
+
+            p.communicate(cmd+'\n')
+        except subprocess.CalledProcessError:
+            raise UserError('Could not create database %s' % pgdata)
+
+
+    def createRunScript(self):
+        """
+        Creates a script that runs postgres in the foreground.
+        'exec' is used to allow easy control by supervisor.
+        """
+        content = textwrap.dedent("""\
+                #!/bin/sh
+                exec %(bin)s/postgres \\
+                    -D %(pgdata-directory)s
+                """ % self.options)
+        name = os.path.join(self.options['services'], 'postgres-start')
+        self.createExecutable(name, content=content)
+
+
+
+class ExportRecipe(GenericBaseRecipe):
+
+    def install(self):
+        pgdata = self.options['pgdata-directory']
+        wrapper = self.options['wrapper']
+        self.createBackupScript(wrapper)
+        return [wrapper]
+
+
+    def createBackupScript(self, wrapper):
+        """
+        Create a script to backup the database in 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!/bin/sh
+                umask 077
+                %(bin)s/pg_dump \\
+                        --host=%(pgdata-directory)s \\
+                        --format=custom \\
+                        --file=%(backup-directory)s/database.dump \\
+                        %(dbname)s
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+
+
+class ImportRecipe(GenericBaseRecipe):
+
+    def install(self):
+        pgdata = self.options['pgdata-directory']
+        wrapper = self.options['wrapper']
+        self.createRestoreScript(wrapper)
+        return [wrapper]
+
+
+    def createRestoreScript(self, wrapper):
+        """
+        Create a script to restore the database from 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!/bin/sh
+                %(bin)s/pg_restore \\
+                        --host=%(pgdata-directory)s \\
+                        --dbname=%(dbname)s \\
+                        --clean \\
+                        --no-owner \\
+                        --no-acl \\
+                        %(backup-directory)s/database.dump
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+
+
+

slapos/recipe/request.py

@@ -123,28 +123,41 @@ class Recipe(object):
 
     isSlave = options.get('slave', '').lower() in \
         librecipe.GenericBaseRecipe.TRUE_VALUES
-    self.instance = instance = request(software_url, software_type,
+
+    self._raise_request_exception = None
+    self.instance = None
+    try:
+      self.instance = request(software_url, software_type,
           name, partition_parameter_kw=partition_parameter_kw,
           filter_kw=filter_kw, shared=isSlave)
+      # XXX what is the right way to get a global id?
+      options['instance_guid'] = self.instance.getId()
+    except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady) as exc:
+      self._raise_request_exception = exc
 
     for param in return_parameters:
+      options['connection-%s' % param] = ''
+      if not self.instance:
+        continue
       try:
         options['connection-%s' % param] = str(
-          instance.getConnectionParameter(param))
-      except (slapmodule.NotFoundError, slapmodule.ServerError):
-        options['connection-%s' % param] = ''
+          self.instance.getConnectionParameter(param))
+      except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady):
         if self.failed is None:
           self.failed = param
 
   def install(self):
+    if self._raise_request_exception:
+      raise self._raise_request_exception
+
     if self.failed is not None:
       # Check instance status to know if instance has been deployed
       try:
-        if self.instance.getComputerId() is not None:
+        if self.instance._computer_id is not None:
           status = self.instance.getState()
         else:
           status = 'not ready yet'
-      except (slapmodule.NotFoundError, slapmodule.ServerError):
+      except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady):
         status = 'not ready yet'
       except AttributeError:
         status = 'unknown'
@@ -159,14 +172,17 @@ class Recipe(object):
 
 class RequestOptional(Recipe):
   """
-  Request a SlapOS instance. Won't fail if instance is not ready.
+  Request a SlapOS instance. Won't fail if request failed or is not ready.
   Same as slapos.cookbook:request, but won't raise in case of problem.
   """
   def install(self):
-    if self.failed is not None:
+    if self._raise_request_exception:
+      self.logger.warning('Optional request failed:')
+      self.logger.warning(self._raise_request_exception)
+    elif self.failed is not None:
       # Check instance status to know if instance has been deployed
       try:
-        if self.instance.getComputerId() is not None:
+        if self.instance._computer_id is not None:
           status = self.instance.getState()
         else:
           status = 'not ready yet'

slapos/recipe/slapmonitor/__init__.py

@@ -24,31 +24,31 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
 
-class Recipe(GenericBaseRecipe):
-  """
-  Slapmonitor instance configuration.
-  """
+from slapos.recipe.librecipe import GenericBaseRecipe
 
-  def __init__(self, buildout, name, options):
-    return GenericBaseRecipe.__init__(self, buildout, name, options)
+class MonitorRecipe(GenericBaseRecipe):
 
   def install(self):
-    config = dict(
-      pid_file_path=self.options['pid-file'],
-      database_path=self.options['database-path'],
-      slapmonitor_path = self.options['slapmonitor-path'],
-      shell_path=self.options['shell-path'],
-    )
+    options = self.options
+    script = self.createWrapper(name=options['path'],
+                                command=options['slapmonitor-path'],
+                                parameters=[
+                                    options['pid-file'],
+                                    options['database-path'],
+                                    ])
+    return [script]
 
-    # Runners
-    runner_path = self.createExecutable(
-      self.options['path'],
-      self.substituteTemplate(self.getTemplateFilename('slapmonitor_run.in'),
-                              config))
 
-    return [runner_path]
+class MonitorXMLRecipe(GenericBaseRecipe):
+
+  def install(self):
+    options = self.options
+    script = self.createWrapper(name=options['path'],
+                                command=options['slapmonitor-xml-path'],
+                                parameters=[
+                                    options['database-path'],
+                                    ],
+                                parameters_extra=True)
+    return [script]
 
-  def update(self):
-    pass

slapos/recipe/slapmonitor/template/slapmonitor_run.in

-#!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-#
-exec %(slapmonitor_path)s %(pid_file_path)s %(database_path)s
-

slapos/recipe/slapreport/__init__.py

@@ -24,36 +24,23 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+
 from slapos.recipe.librecipe import GenericBaseRecipe
 
 class Recipe(GenericBaseRecipe):
-  """
-  Slapmonitor instance configuration.
-  """
-
-  def __init__(self, buildout, name, options):
-    return GenericBaseRecipe.__init__(self, buildout, name, options)
 
   def install(self):
-    config = dict(
-      pid_file_path=self.options['pid-file'],
-      consumption_log_path=self.options['consumption-log-path'],
-      database_path=self.options['database-path'],
-      slapreport_path = self.options['slapreport-path'],
-      logbox_ip = self.options['logbox-ip'],
-      logbox_port = self.options['logbox-port'],
-      logbox_user = self.options['logbox-user'],
-      logbox_passwd = self.options['logbox-passwd'],
-      shell_path=self.options['shell-path'],
-    )
-
-    # Runners
-    runner_path = self.createExecutable(
-      self.options['path'],
-      self.substituteTemplate(self.getTemplateFilename('slapreport_run.in'),
-                              config))
-
-    return [runner_path]
+    options = self.options
+    script = self.createWrapper(name=options['path'],
+                                command=options['slapreport-path'],
+                                parameters=[
+                                    options['pid-file'],
+                                    options['consumption-log-path'],
+                                    options['database-path'],
+                                    options['logbox-ip'],
+                                    options['logbox-port'],
+                                    options['logbox-user'],
+                                    options['logbox-passwd'],
+                                    ])
+    return [script]
 
-  def update(self):
-    pass

slapos/recipe/slapreport/template/slapreport_run.in

-#!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-#
-exec %(slapreport_path)s %(pid_file_path)s %(consumption_log_path)s %(database_path)s %(logbox_ip)s %(logbox_port)s %(logbox_user)s %(logbox_passwd)s

software/drupal/software.cfg

@@ -12,7 +12,6 @@ recipe = slapos.recipe.build:download-unpacked
 url = http://ftp.drupal.org/files/projects/drupal-7.16.tar.gz
 md5sum = 352497b2df94b5308e31cb8da020b631
 
-
 [download-patch-hide-dbsetup]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}

software/maarch/instance-custom.cfg.in

+[buildout]
+
+[maarch-configuration]
+recipe = slapos.recipe.maarch:default
+htdocs = $${apache-php:htdocs}
+db_host = $${postgres-urlparse:host}
+db_port = $${postgres-urlparse:port}
+db_dbname = $${postgres-urlparse:path}
+db_username = $${postgres-urlparse:username}
+db_password = $${postgres-urlparse:password}
+language = en
+php_ini_dir = $${directory:php-ini-dir}
+root_docservers = $${buildout:directory}/srv/docservers
+dependency = $${apache-php:recipe}
+

software/maarch/software.cfg

+[buildout]
+
+extends =
+  ../../stack/lapp/buildout.cfg
+
+# += since we need rdiff-backup and friends
+parts +=
+  slapos-cookbook
+  apache-php-postgres
+  pear-modules
+  eggs
+  instance
+  instance-apache-php
+
+#----------------
+#--
+#-- Main application part
+#-- XXX provide a better URL
+
+[application]
+url = http://downloads.sourceforge.net/project/maarch/Maarch%20Entreprise/MaarchEntreprise-1.3.zip?r=http%3A%2F%2Fwww.maarch.org%2Ftelecharger&ts=1347961624&use_mirror=ignum
+md5sum = 5c2c859dee9d0dde3ba959474fd5fc86
+
+
+#----------------
+#--
+#-- We don't need this static configuration file.
+#-- An empty file is provided because it is required by the lapp stack.
+#--
+
+[application-template]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/config.php.in
+md5sum = d41d8cd98f00b204e9800998ecf8427e
+download-only = True
+filename = template.in
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[application-configuration]
+location = config.php
+
+
+#----------------
+#--
+#-- Define parts that will be executed later, in the instance.
+#--
+
+[custom-application-deployment]
+path = ${custom-application-deployment-template:output}
+part-list = maarch-configuration
+
+[custom-application-deployment-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-custom.cfg.in
+output = ${buildout:directory}/instance-custom.cfg
+md5sum = 28f6ad42b73806901ac2cd4ec563d73b
+mode = 0644
+
+
+#----------------
+#--
+#-- Install PHP channels and modules (only if they are not already installed).
+#--
+
+[pear-modules]
+recipe = cp.recipe.cmd
+pear = ${apache-php-postgres:location}/bin/pear
+install_cmd =
+    ${:pear} channel-info pear.maarch.org >/dev/null || ${:pear} channel-discover pear.maarch.org
+    ${:pear} info maarch/CLITools-0.3.1 >/dev/null || ${:pear} install maarch/CLITools-0.3.1
+    ${:pear} info MIME_Type >/dev/null || ${:pear} install MIME_Type
+
+#----------------
+
+

software/mioga/instance.cfg

+[buildout]
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+parts = instance
+
+[instance]
+recipe = ${instance-recipe:egg}:${instance-recipe:module}
+source = ${application:location}
+template = ${application-template:location}/${application-template:filename}
+configuration = ${application-configuration:location}
+
+httpd_binary = ${apache:location}/bin/httpd
+mysql_binary = ${mariadb:location}/bin/mysql
+mysql_install_binary = ${mariadb:location}/bin/mysql_install_db
+mysql_upgrade_binary = ${mariadb:location}/bin/mysql_upgrade
+mysqld_binary = ${mariadb:location}/libexec/mysqld

software/mioga/software.cfg

+[buildout]
+extends = 
+  ../../component/postgresql/buildout.cfg
+  ../../component/apache-perl/buildout.cfg
+  ../../stack/slapos.cfg
+
+[networkcache]
+# Cedric de Saint Martin signature certificate
+signature-certificate-list =
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
+  MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB
+  D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1
+  P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O
+  BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp
+  cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr
+  b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV
+  Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
+  If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
+  -----END CERTIFICATE-----

software/postgres/instance.cfg.in

+[buildout]
+parts =
+  symlinks
+  publish
+  postgres-instance
+
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[instance-parameters]
+# Fetches parameters defined in SlapOS Master for this instance
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+
+[directories]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+etc = $${buildout:directory}/etc
+services = $${directories:etc}/run/
+promises = $${directories:etc}/promise/
+var = $${buildout:directory}/var
+
+
+[symlinks]
+recipe = cns.recipe.symlink
+symlink_target = $${directories:bin}
+symlink_base = ${postgresql:location}/bin
+
+
+[postgres-instance]
+# create cluster, configuration files and a database
+recipe = slapos.cookbook:postgres
+
+# Options
+ipv6_host = $${instance-parameters:ipv6}
+ipv4_host = $${slap-network-information:local-ipv4}
+user = user
+port = 5432
+dbname = db
+# pgdata_directory is created by initdb, and should not exist beforehand.
+pgdata-directory = $${directories:var}/data
+services = $${directories:services}
+bin = $${directories:bin}
+
+
+[publish]
+recipe = slapos.cookbook:publishurl
+url = $${postgres-instance:url}
+
+
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+

software/postgres/software.cfg

+[buildout]
+
+extends = 
+  ../../stack/slapos.cfg
+  ../../component/postgresql/buildout.cfg
+
+parts =
+  eggs
+  slapos-cookbook
+  instance-template
+  postgresql
+
+[instance-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/template.cfg
+#md5sum = 
+mode = 0644
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+    cns.recipe.symlink
+

stack/lamp/apache/instance-apache-backup.cfg → stack/lamp/apache/instance-apache-backup.cfg.in

@@ -16,32 +16,92 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[urls]
-recipe = slapos.cookbook:publish
-url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
-ssh-public-key = $${sshkeys-dropbear:public-key-value}
-ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
-[apache-proxy]
-recipe = slapos.cookbook:apacheproxy
-url = $${slap-parameter:proxy-url}
-pid-file = $${basedirectory:run}/apache.pid
-lock-file = $${basedirectory:run}/apache.lock
-ip = $${slap-network-information:global-ipv6}
-port = 8080
-error-log = $${directory:httpd-log}/error.log
-access-log = $${directory:httpd-log}/access.log
-httpd-conf = $${rootdirectory:etc}/apache.conf
-wrapper = $${basedirectory:services}/apache
+#----------------
+#--
+#-- Creation of all needed directories.
 
-promise = $${basedirectory:promises}/apache
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
 
-httpd-binary = ${apache:location}/bin/httpd
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+ssh = $${rootdirectory:etc}/ssh
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- sshkeys
 
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -60,6 +120,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
+
+#----------------
+#--
+#-- Dropbear.
+
 [dropbear-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
@@ -75,6 +140,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
+
+#----------------
+#--
+#-- rdiff
+
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
 client = false
@@ -82,18 +152,26 @@ path = $${directory:htdocs}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
+
+#----------------
+#--
+#-- Apache Proxy.
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
 
 [logrotate-entry-apache]
 <= logrotate
@@ -106,51 +184,14 @@ sharedscripts = true
 notifempty = true
 create = true
 
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+#----------------
+#--
+#-- Publish instance parameters.
 
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+[urls]
+recipe = slapos.cookbook:publish
+url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
+ssh-public-key = $${sshkeys-dropbear:public-key-value}
+ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-htdocs = $${rootdirectory:srv}/www/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys
-httpd-log = $${basedirectory:log}/apache/

stack/lamp/apache/instance-apache-import.cfg

 [buildout]
-extends = ${template-apache-php:output}
-          ${template-pbsready-export:output}
+extends = ${instance-apache-php:output}
+          ${pbsready-export:output}
 
 parts =
   apache-proxy
@@ -13,6 +13,7 @@ parts =
   dropbear-server
   dropbear-server-pbs-authorized-key
 
+
 [apache-proxy]
 recipe = slapos.cookbook:apacheproxy
 url = $${slap-parameter:proxy-url}
@@ -29,10 +30,103 @@ promise = $${basedirectory:promises}/apache
 
 httpd-binary = ${apache:location}/bin/httpd
 
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+ssh = $${rootdirectory:etc}/ssh
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-proxy:error-log} $${apache-proxy:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+
+#----------------
+#--
+#-- sshkeys
+
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -51,6 +145,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
+
+#----------------
+#--
+#-- Dropbear.
+
 [dropbear-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
@@ -66,6 +165,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
+
+#----------------
+#--
+#-- rdiff
+
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
 client = false
@@ -73,75 +177,3 @@ path = $${directory:htdocs}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
-
-[logrotate-entry-apache]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = apache
-log = $${apache-proxy:error-log} $${apache-proxy:access-log}
-frequency = daily
-rotate-num = 30
-sharedscripts = true
-notifempty = true
-create = true
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-htdocs = $${rootdirectory:srv}/www/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys
-httpd-log = $${basedirectory:log}/apache/

stack/lamp/apache/instance-apache-php.cfg → stack/lamp/apache/instance-apache-php.cfg.in

@@ -10,7 +10,8 @@ parts =
   cron-entry-logrotate
   promise
   slapmonitor
-  slapreport
+  slapmonitor-xml
+
   frontend-promise
   content-promise
   publish-connection-informations
@@ -20,46 +21,98 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
-# Creation of all needed directories
+#----------------
+#--
+#-- Creation of all needed directories.
+
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
 
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-ca-dir = $${rootdirectory:srv}/ssl/
-httpd-log = $${basedirectory:log}/apache/
-php-ini-dir = $${rootdirectory:etc}/php/
-tmp-php = $${rootdirectory:tmp}/php/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-stunnel-conf = $${rootdirectory:etc}/stunnel/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+httpd-log = $${basedirectory:log}/apache
+php-ini-dir = $${rootdirectory:etc}/php
+tmp-php = $${rootdirectory:tmp}/php
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+report = $${rootdirectory:etc}/report
+stunnel-conf = $${rootdirectory:etc}/stunnel
+xml-report = $${rootdirectory:var}/xml_report
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:ca-dir}/requests/
-private = $${directory:ca-dir}/private/
-certs = $${directory:ca-dir}/certs/
-newcerts = $${directory:ca-dir}/newcerts/
-crl = $${directory:ca-dir}/crl/
+requests = $${directory:ca-dir}/requests
+private = $${directory:ca-dir}/private
+certs = $${directory:ca-dir}/certs
+newcerts = $${directory:ca-dir}/newcerts
+crl = $${directory:ca-dir}/crl
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
 
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- Deploy stunnel.
 
-# Deploy stunnel
 [stunnel]
 recipe = slapos.cookbook:stunnel
 client = true
@@ -76,8 +129,22 @@ pid-file = $${basedirectory:run}/stunnel.pid
 wrapper = $${rootdirectory:bin}/raw_stunnel
 post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
 
+[logrotate-entry-stunnel]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = stunnel
+log = $${stunnel:log-file}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+post = $${stunnel:post-rotate-script}
+
+
+#----------------
+#--
+#-- Certificate stuff.
 
-# Certificate stuffs
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${openssl:location}/bin/openssl
@@ -98,7 +165,10 @@ key-file = $${stunnel:key-file}
 cert-file = $${stunnel:cert-file}
 
 
-# Request MariaDB instance and parse its URL
+#----------------
+#--
+#-- Request MariaDB instance and parse its URL.
+
 [request-mariadb]
 <= slap-connection
 recipe = slapos.cookbook:request
@@ -114,7 +184,10 @@ recipe = slapos.cookbook:urlparse
 url = $${request-mariadb:connection-url}
 
 
-# Deploy Apache + PHP application
+#----------------
+#--
+#-- Deploy Apache + PHP application.
+
 [apache-php]
 recipe = slapos.cookbook:apachephp
 source = ${application:location}
@@ -142,21 +215,6 @@ mysql-database = $${mariadb-urlparse:path}
 mysql-host = $${stunnel:local-host}
 mysql-port = $${stunnel:local-port}
 
-
-# Deploy logrotate, cron, configure it
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
-
 [logrotate-entry-apache]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
@@ -168,40 +226,11 @@ sharedscripts = true
 notifempty = true
 create = true
 
-[logrotate-entry-stunnel]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = stunnel
-log = $${stunnel:log-file}
-frequency = daily
-rotate-num = 30
-notifempty = true
-create = true
-post = $${stunnel:post-rotate-script}
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
 
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
+#----------------
+#--
+#-- Request frontend.
 
-
-# Request frontend
 [request-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -215,37 +244,38 @@ return = site_url
 config-custom_domain = $${slap-parameter:domain}
 
 
-# Deploy slapmonitor
+#----------------
+#--
+#-- Deploy slapmonitor.
+
 [slapmonitor]
 recipe = slapos.cookbook:slapmonitor
 pid-file = $${basedirectory:run}/apache.pid
 database-path = $${basedirectory:log}/slapmonitor.db
-shell-path = ${dash:location}/bin/dash
 slapmonitor-path = ${buildout:bin-directory}/slapmonitor
 path = $${basedirectory:services}/slapmonitor
 
-[slapreport]
-recipe = slapos.cookbook:slapreport
-pid-file = $${basedirectory:run}/apache.pid
+[slapmonitor-xml]
+recipe = slapos.cookbook:slapmonitor-xml
 database-path = $${basedirectory:log}/slapmonitor.db
-consumption-log-path = $${basedirectory:log}/instance_consumption.log
-logbox-ip = 87.98.152.12
-logbox-port = 5122
-logbox-user = admin
-logbox-passwd = passer
-shell-path = ${dash:location}/bin/dash
-slapreport-path = ${buildout:bin-directory}/slapreport
-path = $${basedirectory:services}/slapreport
+slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml
+path = $${directory:report}/slapmonitor-xml
 
 
-# Publish all instance parameters (url of instance)
+#----------------
+#--
+#-- Publish instance parameters.
+
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
 backend_url = $${apache-php:url}
 url = $${request-frontend:connection-site_url}
 
 
-# Deploy promises scripts
+#----------------
+#--
+#-- Deploy promises scripts.
+
 [promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${basedirectory:promises}/apache
@@ -266,6 +296,9 @@ url = $${request-frontend:connection-site_url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
+
+
+
 [slap-parameter]
 # Default value if no domain is specified
 domain =
@@ -274,3 +307,4 @@ logbox-ip =
 logbox-port =
 logbox-user =
 logbox-passwd =
+

stack/lamp/buildout.cfg

@@ -10,15 +10,17 @@ parts =
   rdiff-backup
   dropbear
   eggs
-  instance-recipe-egg
-  template
+  instance
 
-  template-apache-php
-  template-mariadb
+  instance-apache-php
+  instance-mariadb
 
 #Contains the importer and exporter recipes for mariadb
-  template-mariadb-import
-  template-mariadb-export
+  instance-mariadb-import
+  instance-mariadb-export
+
+  instance-default-root
+
 
 extends =
   ../resilient/buildout.cfg
@@ -48,79 +50,85 @@ extends =
 # Compile dir is for plugins, there's no plugin in LAMP
 keep-compile-dir = false
 
-[instance-recipe]
-egg = slapos.cookbook
-module = lamp.request
-
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-eggs = ${instance-recipe:egg}
-
 [application]
 recipe = hexagonit.recipe.download
 #If provided tarball does not containt top directory this option shall be changed to false
 strip-top-level-dir = true
 
-[template]
+
+#----------------
+#-- Instance-level buildout profiles.
+
+[instance]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-output = ${buildout:directory}/template.cfg
-md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+md5sum = 38bdcf0a8263d4a19bd6a35c0cd00340
 mode = 0644
 
-[template-apache-php]
+[instance-apache-php]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/apache/instance-apache-php.cfg
-output = ${buildout:directory}/template-apache-php.cfg
-md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
+output = ${buildout:directory}/instance-apache-php.cfg
+md5sum = fc29d853dcd0802dd61c60b09e898c11
 mode = 0644
 
-[template-apache-backup]
+[instance-apache-backup]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg
-output = ${buildout:directory}/template-apache-backup.cfg
-md5sum = cfb77ac8785e0d125a785f69a5339014
+url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
+output = ${buildout:directory}/instance-apache-backup.cfg
+md5sum = 48f969d82319a9d145570f5f0fd27672
 mode = 0644
 
 [template-resilient-lamp]
 recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance-resilient.cfg
-rendered = ${buildout:directory}/template-resilient.cfg
+template = ${:_profile_base_location_}/template-resilient.cfg.in
+rendered = ${buildout:directory}/instance-resilient.cfg
 
-context = key templateapache template-apache-php:output
+context = key templateapache instance-apache-php:output
           key dropbear dropbear:location
           key buildout buildout:bin-directory
 
 import-list = file parts template-parts:destination
               file replicated template-replicated:destination
 
-md5sum = 03aafcba5c626a4a1bd180d71007be1e
+md5sum = 5605ad8896c2718854bf26148c4ae940
 mode = 0644
  
 
-[template-mariadb]
+[instance-mariadb]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg
-output = ${buildout:directory}/template-mariadb.cfg
-md5sum = fa9dc10efbcf61119f4cbab37c741322
+url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg.in
+output = ${buildout:directory}/instance-mariadb.cfg
+md5sum = ba8dd08dfd5e6a9dc614693d066eb21d
 mode = 0644
 
-[template-mariadb-import]
+[instance-mariadb-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg
-output = ${buildout:directory}/template-mariadb-import.cfg
-md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg.in
+output = ${buildout:directory}/instance-mariadb-import.cfg
+md5sum = ea43b8ed38a55a11b027fc283c0e718a
 mode = 0644
 
-[template-mariadb-export]
+[instance-mariadb-export]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg
-output = ${buildout:directory}/template-mariadb-export.cfg
-md5sum = 4b7dec765265b27c8235419b82ca7b02
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg.in
+output = ${buildout:directory}/instance-mariadb-export.cfg
+md5sum = 685c8abf0f487c72273846002ec631a0
 mode = 0644
 
 
-# Dummy parts in case no application configuration file is needed
+[instance-default-root]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-default-root.cfg.in
+output = ${buildout:directory}/instance-default-root.cfg
+md5sum = 53c9020f7a0b5203f976e069e455787b
+mode = 0644
+
+
+#----------------
+#-- Dummy parts in case no application configuration file is needed
+
 [application-template]
 filename =
 location =
@@ -128,6 +136,8 @@ location =
 [application-configuration]
 location =
 
+#----------------
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =

stack/lamp/instance-default-root.cfg.in

+[buildout]
+parts = request-apache
+
+[request-apache]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = $${slap-connection:software-release-url}
+software-type = apache
+name = Apache
+

stack/lamp/instance.cfg → stack/lamp/instance.cfg.in

 [buildout]
 
-extends = 
-  ${template-switchsoftware:output}
-
 parts =
   switch_softwaretype
 
@@ -12,12 +9,19 @@ offline = true
 
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = ${template-apache-php:output}
+default = ${instance-apache-php:output}
 resilient = ${template-resilient-lamp:rendered}
-mariadb = ${template-mariadb:output}
-mariadb-import = ${template-mariadb-import:output}
-mariadb-export = ${template-mariadb-export:output}
+mariadb = ${instance-mariadb:output}
+mariadb-import = ${instance-mariadb-import:output}
+mariadb-export = ${instance-mariadb-export:output}
+apache-backup = ${instance-apache-backup:output}
+
+# To prepend an empty root instance:
+#default = ${instance-default-root:output}
+#apache = ${instance-apache-php:output}
+
+#frozen creates a syntax error, meaning it can keep its data.
+#It's dirty as hell, it needs to be replaced.
+frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
-apache-backup = ${template-apache-backup:output}
-frozen = ${template-frozen:output}
 

stack/lamp/mariadb/instance-mariadb-export.cfg → stack/lamp/mariadb/instance-mariadb-export.cfg.in

 [buildout]
-extends = ${template-mariadb:output}
-          ${template-pbsready-export:output}
+extends = ${instance-mariadb:output}
+          ${pbsready-export:output}
 
 parts += mariadb
 

stack/lamp/mariadb/instance-mariadb-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg.in

 [buildout]
-extends = ${template-mariadb:output}
-          ${template-pbsready-import:output}
+extends = ${instance-mariadb:output}
+          ${pbsready-import:output}
 
 parts += mariadb
 

stack/lamp/mariadb/instance-mariadb.cfg → stack/lamp/mariadb/instance-mariadb.cfg.in

@@ -12,111 +12,80 @@ parts =
   cron
   cron-entry-logrotate
   slapmonitor
-  slapreport
+  slapmonitor-xml
 
 gzip-binary = ${gzip:location}/bin/gzip
 
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[urls]
-recipe = slapos.cookbook:publish
-url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
-ip = $${slap-network-information:global-ipv6}
-
-[mariadb]
-recipe = slapos.cookbook:mysql
 
-# Options
-recovering = false
-user = user
-port = 3306
-ip = $${slap-network-information:local-ipv4}
-database = db
+#----------------
+#--
+#-- Creation of all needed directories.
 
-# Paths
-wrapper = $${basedirectory:services}/mariadb
-update-wrapper = $${basedirectory:services}/mariadb_update
-logrotate-post = $${rootdirectory:bin}/mariadb_post_logrotate
-data-directory = $${directory:mariadb-data}
-pid-file = $${basedirectory:run}/mariadb.pid
-socket = $${basedirectory:run}/mariadb.sock
-error-log = $${basedirectory:log}/mariadb_error.log
-conf-file = $${rootdirectory:etc}/mariadb.cnf
-promise = $${basedirectory:promises}/mysql
-
-# Binary information
-mysql-base-directory = ${mariadb:location}
-mysql-binary = ${mariadb:location}/bin/mysql
-mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
-mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
-mysqld-binary = ${mariadb:location}/bin/mysqld
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
 
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+script = $${rootdirectory:etc}/script
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
 
-[slapmonitor]
-recipe = slapos.cookbook:slapmonitor
-pid-file = $${basedirectory:run}/mariadb.pid
-database-path = $${basedirectory:log}/slapmonitor.db
-shell-path = ${dash:location}/bin/dash
-slapmonitor-path = ${buildout:bin-directory}/slapmonitor
-path = $${basedirectory:services}/slapmonitor
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+mariadb-data = $${rootdirectory:srv}/mariadb
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+report = $${rootdirectory:etc}/report
+stunnel-conf = $${rootdirectory:etc}/stunnel
+xml-report = $${rootdirectory:var}/xml_report
+
+
+#----------------
+#--
+#-- Deploy cron.
 
-[slapreport]
-recipe = slapos.cookbook:slapreport
-pid-file = $${basedirectory:run}/mariadb.pid
-consumption-log-path = $${basedirectory:log}/instance_consumption.log
-database-path = $${basedirectory:log}/slapmonitor.db
-logbox-ip = 87.98.152.12
-logbox-port = 5122
-logbox-user = admin
-logbox-passwd = passer
-shell-path = ${dash:location}/bin/dash
-slapreport-path = ${buildout:bin-directory}/slapreport
-path = $${basedirectory:services}/slapreport
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
 
-[certificate-authority]
-recipe = slapos.cookbook:certificate_authority
-openssl-binary = ${openssl:location}/bin/openssl
-ca-dir = $${directory:ca-dir}
-requests-directory = $${cadirectory:requests}
-wrapper = $${basedirectory:services}/ca
-ca-private = $${cadirectory:private}
-ca-certs = $${cadirectory:certs}
-ca-newcerts = $${cadirectory:newcerts}
-ca-crl = $${cadirectory:crl}
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
 
-[cadirectory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:ca-dir}/requests/
-private = $${directory:ca-dir}/private/
-certs = $${directory:ca-dir}/certs/
-newcerts = $${directory:ca-dir}/newcerts/
-crl = $${directory:ca-dir}/crl/
 
-[ca-stunnel]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-executable = $${stunnel:wrapper}
-wrapper = $${basedirectory:services}/stunnel
-key-file = $${stunnel:key-file}
-cert-file = $${stunnel:cert-file}
+#----------------
+#--
+#-- Deploy logrotate.
 
-[stunnel]
-recipe = slapos.cookbook:stunnel
-stunnel-binary = ${stunnel:location}/bin/stunnel
-wrapper = $${rootdirectory:bin}/stunnel
-log-file = $${basedirectory:log}/stunnel.log
-config-file = $${directory:stunnel-conf}/stunnel.conf
-key-file = $${directory:stunnel-conf}/stunnel.key
-cert-file = $${directory:stunnel-conf}/stunnel.crt
-pid-file = $${basedirectory:run}/stunnel.pid
-local-host = $${mariadb:ip}
-local-port = $${mariadb:port}
-remote-host = $${slap-network-information:global-ipv6}
-remote-port = 6446
-client = false
-post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
 
 [logrotate]
 recipe = slapos.cookbook:logrotate
@@ -164,53 +133,119 @@ rotate-num = 30
 notifempty = true
 create = true
 
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
+#----------------
+#--
+#-- Deploy stunnel.
 
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
+[stunnel]
+recipe = slapos.cookbook:stunnel
+stunnel-binary = ${stunnel:location}/bin/stunnel
+wrapper = $${rootdirectory:bin}/stunnel
+log-file = $${basedirectory:log}/stunnel.log
+config-file = $${directory:stunnel-conf}/stunnel.conf
+key-file = $${directory:stunnel-conf}/stunnel.key
+cert-file = $${directory:stunnel-conf}/stunnel.crt
+pid-file = $${basedirectory:run}/stunnel.pid
+local-host = $${mariadb:ip}
+local-port = $${mariadb:port}
+remote-host = $${slap-network-information:global-ipv6}
+remote-port = 6446
+client = false
+post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
 
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
+#----------------
+#--
+#-- Certificate stuff.
 
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-script = $${rootdirectory:etc}/script/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
 
-[directory]
+[cadirectory]
 recipe = slapos.cookbook:mkdirectory
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-ca-dir = $${rootdirectory:srv}/ssl/
-mariadb-data = $${rootdirectory:srv}/mariadb/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-stunnel-conf = $${rootdirectory:etc}/stunnel/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+
+#----------------
+#--
+#-- Creates a MariaDB configuration file, and a database.
+
+[mariadb]
+recipe = slapos.cookbook:mysql
+
+# Options
+recovering = false
+user = user
+port = 3306
+ip = $${slap-network-information:local-ipv4}
+database = db
+
+# Paths
+wrapper = $${basedirectory:services}/mariadb
+update-wrapper = $${basedirectory:services}/mariadb_update
+logrotate-post = $${rootdirectory:bin}/mariadb_post_logrotate
+data-directory = $${directory:mariadb-data}
+pid-file = $${basedirectory:run}/mariadb.pid
+socket = $${basedirectory:run}/mariadb.sock
+error-log = $${basedirectory:log}/mariadb_error.log
+conf-file = $${rootdirectory:etc}/mariadb.cnf
+promise = $${basedirectory:promises}/mysql
+
+# Binary information
+mysql-base-directory = ${mariadb:location}
+mysql-binary = ${mariadb:location}/bin/mysql
+mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
+mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
+mysqld-binary = ${mariadb:location}/bin/mysqld
+
+
+#----------------
+#--
+#-- Deploy slapmonitor.
+
+[slapmonitor]
+recipe = slapos.cookbook:slapmonitor
+pid-file = $${basedirectory:run}/mariadb.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-path = ${buildout:bin-directory}/slapmonitor
+path = $${basedirectory:services}/slapmonitor
+
+[slapmonitor-xml]
+recipe = slapos.cookbook:slapmonitor-xml
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml
+path = $${directory:report}/slapmonitor-xml
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[urls]
+recipe = slapos.cookbook:publish
+url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
+ip = $${slap-network-information:global-ipv6}
+
 
 [slap-parameter]
 #Default  value if no ssh parameters specified

stack/lamp/instance-resilient.cfg → stack/lamp/template-resilient.cfg.in

@@ -6,7 +6,8 @@
 extends =
    {{templateapache}}
 
-parts =
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
   {{ parts.replicate("mariadb","3") }}
   request-apache-backup-1
   request-apache-backup-2
@@ -44,8 +45,8 @@ slave = false
 
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = ${directory:sshkeys}/requests/
-keys = ${directory:sshkeys}/keys/
+requests = ${directory:sshkeys}/requests
+keys = ${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -121,6 +122,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url}
 config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
 config-frequency = 30 * * * *
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
 
 [request-pull-backup-server-apache-2]
 <= request-pbs-common
@@ -134,6 +137,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url}
 config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
 config-frequency = 30 * * * *
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
 
 
 [request-pull-backup-server-apache-backup-1]
@@ -146,6 +151,8 @@ config-type = push
 config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
 config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
 
 [request-pull-backup-server-apache-backup-2]
 <= request-pbs-common
@@ -157,8 +164,10 @@ config-type = push
 config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
 config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
 
 
 [directory]
-ssh = ${rootdirectory:etc}/ssh/
+ssh = ${rootdirectory:etc}/ssh
 sshkeys = ${rootdirectory:srv}/sshkeys

stack/lapp/README.txt

+
+LAPP stack
+==========
+
+This fork of the LAMP stack provides:
+
+ - a Postgres instance, with an empty database and a 'postgres' superuser.
+   Log rotation is handled by Postgres itself.
+
+ - symlinks to all the postgres binaries, usable through unix socket
+   with no further authentication, or through ipv6
+
+ - a psycopg2 (postgres driver) egg
+
+ - configuration for a maarch instance (this part should be brought outside the stack)
+

stack/lapp/apache/instance-apache-backup.cfg.in

+[buildout]
+
+parts =
+  urls
+  apache-proxy
+  logrotate
+  logrotate-entry-apache
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  sshkeys-dropbear
+  dropbear-server
+  dropbear-server-pbs-authorized-key
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+ssh = $${rootdirectory:etc}/ssh
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- sshkeys
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = $${dropbear-server:wrapper}
+public-key = $${dropbear-server:rsa-keyfile}.pub
+private-key = $${dropbear-server:rsa-keyfile}
+wrapper = $${basedirectory:services}/sshd
+
+
+#----------------
+#--
+#-- Dropbear.
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = $${slap-network-information:global-ipv6}
+port = 2222
+home = $${directory:ssh}
+wrapper = $${rootdirectory:bin}/raw_sshd
+shell = $${rdiff-backup-server:wrapper}
+rsa-keyfile = $${directory:ssh}/server_key.rsa
+dropbear-binary = ${dropbear:location}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = $${slap-parameter:authorized-key}
+
+
+#----------------
+#--
+#-- rdiff
+
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:htdocs}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+
+
+#----------------
+#--
+#-- Apache Proxy.
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-proxy:error-log} $${apache-proxy:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[urls]
+recipe = slapos.cookbook:publish
+url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
+ssh-public-key = $${sshkeys-dropbear:public-key-value}
+ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
+

stack/lapp/apache/instance-apache-php.cfg.in

+[buildout]
+extends = ${custom-application-deployment:path}
+
+parts =
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  promise
+  slapmonitor
+  slapmonitor-xml
+
+  frontend-promise
+  content-promise
+  publish-connection-informations
+  ${custom-application-deployment:part-list}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+httpd-log = $${basedirectory:log}/apache
+php-ini-dir = $${rootdirectory:etc}/php
+tmp-php = $${rootdirectory:tmp}/php
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+report = $${rootdirectory:etc}/report
+stunnel-conf = $${rootdirectory:etc}/stunnel
+xml-report = $${rootdirectory:var}/xml_report
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests
+private = $${directory:ca-dir}/private
+certs = $${directory:ca-dir}/certs
+newcerts = $${directory:ca-dir}/newcerts
+crl = $${directory:ca-dir}/crl
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- Deploy stunnel.
+
+[stunnel]
+recipe = slapos.cookbook:stunnel
+client = true
+stunnel-binary = ${stunnel:location}/bin/stunnel
+remote-host = $${postgres-urlparse:host}
+remote-port = $${postgres-urlparse:port}
+local-host = $${slap-network-information:local-ipv4}
+local-port = 33060
+log-file = $${basedirectory:log}/stunnel.log
+config-file = $${directory:stunnel-conf}/stunnel.conf
+key-file = $${directory:stunnel-conf}/stunnel.key
+cert-file = $${directory:stunnel-conf}/stunnel.crt
+pid-file = $${basedirectory:run}/stunnel.pid
+wrapper = $${rootdirectory:bin}/raw_stunnel
+post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
+
+[logrotate-entry-stunnel]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = stunnel
+log = $${stunnel:log-file}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+post = $${stunnel:post-rotate-script}
+
+
+#----------------
+#--
+#-- Certificate stuff.
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+
+#----------------
+#--
+#-- Request Postgres instance and parse its URL.
+
+[request-postgres]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Postgres
+software-url = $${slap-connection:software-release-url}
+software-type = postgres
+return = url
+sla = computer_guid
+sla-computer_guid = $${slap-connection:computer-id}
+
+[postgres-urlparse]
+recipe = slapos.cookbook:urlparse
+url = $${request-postgres:connection-url}
+
+
+#----------------
+#--
+#-- Deploy Apache + PHP application.
+#-- Despite the names of mysql-* parameters, they are not really specific to mysql.
+
+[apache-php]
+recipe = slapos.cookbook:apachephp
+source = ${application:location}
+template = ${application-template:location}/${application-template:filename}
+configuration = ${application-configuration:location}
+
+htdocs = $${rootdirectory:srv}/www/
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+url = http://[$${:ip}]:$${:port}/
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+php-ini-dir = $${directory:php-ini-dir}
+tmp-dir = $${directory:tmp-php}
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+mysql-username = $${postgres-urlparse:username}
+mysql-password = $${postgres-urlparse:password}
+mysql-database = $${postgres-urlparse:path}
+mysql-host = $${stunnel:local-host}
+mysql-port = $${stunnel:local-port}
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-php:error-log} $${apache-php:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+
+#----------------
+#--
+#-- Request frontend.
+
+[request-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url custom_domain
+config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
+return = site_url
+config-custom_domain = $${slap-parameter:domain}
+
+
+#----------------
+#--
+#-- Deploy slapmonitor.
+
+[slapmonitor]
+recipe = slapos.cookbook:slapmonitor
+pid-file = $${basedirectory:run}/apache.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-path = ${buildout:bin-directory}/slapmonitor
+path = $${basedirectory:services}/slapmonitor
+
+[slapmonitor-xml]
+recipe = slapos.cookbook:slapmonitor-xml
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml
+path = $${directory:report}/slapmonitor-xml
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = $${apache-php:url}
+url = $${request-frontend:connection-site_url}
+
+
+#----------------
+#--
+#-- Deploy promises scripts.
+
+[promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/apache
+hostname = $${apache-php:ip}
+port = $${apache-php:port}
+
+[frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/frontend
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+[content-promise]
+recipe = slapos.cookbook:check_page_content
+path = $${basedirectory:promises}/content
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+
+
+
+[slap-parameter]
+# Default value if no domain is specified
+domain =
+# Default value if no ssh parameter is specified
+logbox-ip =
+logbox-port =
+logbox-user =
+logbox-passwd =
+

stack/lapp/buildout.cfg

+[buildout]
+parts =
+  apache-php-postgres
+  rdiff-backup
+  dropbear
+  eggs
+  instance
+  psycopg2
+
+  instance-apache-php
+  instance-postgres
+
+#Contains the importer and exporter recipes for postgres
+  instance-postgres-import
+  instance-postgres-export
+
+  instance-default-root
+
+
+extends =
+  ../resilient/buildout.cfg
+  ../../component/apache/buildout.cfg
+  ../../component/apache-php/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/postgresql/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
+  ../../component/stunnel/buildout.cfg
+  ../../component/dropbear/buildout.cfg
+  ../slapos.cfg
+
+
+#----------------
+#-- Application-specific part (maarch, etc.)
+
+[application]
+recipe = hexagonit.recipe.download
+strip-top-level-dir = true
+
+
+#----------------
+#-- Instance-level buildout profiles.
+
+[instance]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+md5sum = 1aaf3ea7b14e09e66904bdb80e3cfe2f
+mode = 0644
+
+[instance-apache-php]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
+output = ${buildout:directory}/instance-apache-php.cfg
+md5sum = bed286b680bd8cd494da080cdc229f1e
+mode = 0644
+
+[instance-apache-backup]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
+output = ${buildout:directory}/instance-apache-backup.cfg
+md5sum = 48f969d82319a9d145570f5f0fd27672
+mode = 0644
+
+[template-resilient-lapp]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/template-resilient.cfg.in
+rendered = ${buildout:directory}/instance-resilient.cfg
+
+context = key templateapache instance-apache-php:output
+          key dropbear dropbear:location
+          key buildout buildout:bin-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = 525f50e60d0a96557a552de6afa4ab88
+mode = 0644
+ 
+
+[instance-postgres]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/postgres/instance-postgres.cfg.in
+output = ${buildout:directory}/instance-postgres.cfg
+md5sum = c5cd2a644fcd8450bc5d13bf53ec9f7d
+mode = 0644
+
+[instance-postgres-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/postgres/instance-postgres-import.cfg.in
+output = ${buildout:directory}/instance-postgres-import.cfg
+md5sum = 1989ba2164dd5f79793a04e0a02ea515
+mode = 0644
+
+[instance-postgres-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/postgres/instance-postgres-export.cfg.in
+output = ${buildout:directory}/instance-postgres-export.cfg
+md5sum = 7bce31bc22a731a8fc6119aee96586f5
+mode = 0644
+
+
+#----------------
+#-- Postgres driver for Python recipes.
+
+[psycopg2-env]
+PATH = ${postgresql:location}/bin:%(PATH)s
+
+
+[psycopg2]
+recipe = zc.recipe.egg:custom
+egg = psycopg2
+define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
+environment = psycopg2-env
+include-dirs =
+    ${postgresql:location}/include
+library-dirs =
+    ${postgresql:location}/lib
+rpath = 
+    ${postgresql:location}/lib
+
+[instance-default-root]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-default-root.cfg.in
+output = ${buildout:directory}/instance-default-root.cfg
+md5sum = 53c9020f7a0b5203f976e069e455787b
+mode = 0644
+
+#----------------
+#--
+#-- Optional part allowing applications using this stack to run a custom
+#-- instance.cfg at the end of Apache/PHP instance deployment.
+#-- To use it in your application, just override those two parameters, like:
+
+[custom-application-deployment]
+# path = /path/to/instance-custom.cfg
+# part-list = part1 part2
+# See software/maarch/software.cfg for an example.
+path =
+part-list =
+
+#----------------
+#-- Dummy parts in case no application configuration file is needed
+
+[application-template]
+filename =
+location =
+
+[application-configuration]
+location =
+
+#----------------
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${psycopg2:egg}
+  slapos.toolbox
+  cns.recipe.symlink
+
+
+[versions]
+meld3 = 0.6.10
+

stack/lapp/instance-default-root.cfg.in

+[buildout]
+parts = request-apache
+
+[request-apache]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = $${slap-connection:software-release-url}
+software-type = apache
+name = Apache
+

stack/resilient/switchsoftware.cfg → stack/lapp/instance.cfg.in

 [buildout]
+
 parts =
   switch_softwaretype
 
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
-pull-backup = ${template-pull-backup:output}
+default = ${instance-default-root:output}
+apache = ${instance-apache-php:output}
+resilient = ${template-resilient-lapp:rendered}
+postgres = ${instance-postgres:output}
+postgres-import = ${instance-postgres-import:output}
+postgres-export = ${instance-postgres-export:output}
+apache-backup = ${instance-apache-backup:output}
+
 #frozen creates a syntax error, meaning it can keep its data.
 #It's dirty as hell, it needs to be replaced.
-frozen = ${template-frozen:output}
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}
+

stack/lapp/postgres/instance-postgres-export.cfg.in

+[buildout]
+extends = ${instance-postgres:output}
+          ${pbsready-export:output}
+
+
+parts +=
+  urls
+  postgres-instance
+  postgres-promise
+
+
+[exporter]
+recipe = slapos.cookbook:postgres.export
+wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter
+bin = $${postgres-instance:bin}
+pgdata-directory = $${postgres-instance:pgdata-directory}
+backup-directory = $${postgres-instance:backup-directory}
+dbname = $${postgres-instance:dbname}

stack/lapp/postgres/instance-postgres-import.cfg.in

+[buildout]
+extends = ${instance-postgres:output}
+          ${pbsready-import:output}
+
+
+parts +=
+  postgres-instance
+
+[importer]
+recipe = slapos.cookbook:postgres.import
+wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-importer
+bin = $${postgres-instance:bin}
+pgdata-directory = $${postgres-instance:pgdata-directory}
+backup-directory = $${postgres-instance:backup-directory}
+dbname = $${postgres-instance:dbname}

stack/lapp/postgres/instance-postgres.cfg.in

+[buildout]
+parts =
+  urls
+  postgres-instance
+  postgres-promise
+  stunnel
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-stunnel
+  logrotate-entry-cron
+  cron
+  cron-entry-logrotate
+  slapmonitor
+  slapmonitor-xml
+
+gzip-binary = ${gzip:location}/bin/gzip
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+script = $${rootdirectory:etc}/script
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+logrotate-backup = $${basedirectory:backup}/logrotate
+report = $${rootdirectory:etc}/report
+stunnel-conf = $${rootdirectory:etc}/stunnel
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+xml-report = $${rootdirectory:var}/xml_report
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = $${buildout:gzip-binary}
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-stunnel]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = stunnel
+log = $${stunnel:log-file}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+post = $${stunnel:post-rotate-script}
+
+[logrotate-entry-cron]
+<= logrotate
+recipe =slapos.cookbook:logrotate.d
+name = crond
+log = $${cron-simplelogger:log}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+
+#----------------
+#--
+#-- Deploy stunnel.
+#-- XXX This is actually not needed with Postgres.
+
+[stunnel]
+recipe = slapos.cookbook:stunnel
+stunnel-binary = ${stunnel:location}/bin/stunnel
+wrapper = $${rootdirectory:bin}/stunnel
+log-file = $${basedirectory:log}/stunnel.log
+config-file = $${directory:stunnel-conf}/stunnel.conf
+key-file = $${directory:stunnel-conf}/stunnel.key
+cert-file = $${directory:stunnel-conf}/stunnel.crt
+pid-file = $${basedirectory:run}/stunnel.pid
+local-host = $${postgres-instance:ipv4_host}
+local-port = $${postgres-instance:port}
+remote-host = $${slap-network-information:global-ipv6}
+remote-port = 6446
+client = false
+post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
+
+
+#----------------
+#--
+#-- Certificate stuff.
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+#----------------
+#--
+#-- Creates a Postgres cluster, configuration files, and a database.
+
+[postgres-instance]
+recipe = slapos.cookbook:postgres
+ipv6_host = $${slap-network-information:global-ipv6}
+user = postgres
+port = 5432
+dbname = db
+# XXX the next line is required by stunnel, not by us
+ipv4_host = $${slap-network-information:local-ipv4}
+# pgdata_directory is created by initdb, and should not exist beforehand.
+pgdata-directory = $${rootdirectory:var}/data
+backup-directory = $${basedirectory:backup}/postgres
+services = $${basedirectory:services}
+bin = $${rootdirectory:bin}
+
+dependency-symlinks = $${symlinks:recipe}
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+#----------------
+#--
+#-- Creates symlinks from the instance to the software release.
+
+[symlinks]
+recipe = cns.recipe.symlink
+symlink_target = $${rootdirectory:bin}
+symlink_base = ${postgresql:location}/bin
+
+
+#----------------
+#--
+#-- Deploy slapmonitor.
+
+[slapmonitor]
+recipe = slapos.cookbook:slapmonitor
+pid-file = $${postgres-instance:pgdata-directory}/postmaster.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-path = ${buildout:bin-directory}/slapmonitor
+path = $${basedirectory:services}/slapmonitor
+
+[slapmonitor-xml]
+recipe = slapos.cookbook:slapmonitor-xml
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml
+path = $${directory:report}/slapmonitor-xml
+
+
+#----------------
+#--
+#-- Deploy promise scripts.
+
+[postgres-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/postgres
+hostname = $${slap-network-information:global-ipv6}
+port = $${postgres-instance:port}
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[urls]
+recipe = slapos.cookbook:publish
+url = $${postgres-instance:url}
+ip = $${slap-network-information:global-ipv6}
+
+#----------------
+#--
+#-- Fetches parameters defined in SlapOS Master for this instance
+
+[instance-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+

stack/lapp/template-resilient.cfg.in

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+
+extends =
+   {{templateapache}}
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("postgres","3") }}
+  request-apache-backup-1
+  request-apache-backup-2
+
+  request-pull-backup-server-apache-1
+  request-pull-backup-server-apache-backup-1
+
+  request-pull-backup-server-apache-2
+  request-pull-backup-server-apache-backup-2
+
+  publish-connection-informations
+  apache-php
+  stunnel
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  dropbear-server
+  sshkeys-authority
+  dropbear-server-pbs-authorized-key
+
+  request-pull-backup-server
+
+{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }}
+
+
+[request-pull-backup-server]
+<= request-pbs-common
+name = PBS (Pull Backup Server)
+return = ssh-key notification-url feeds-url
+slave = false
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = ${directory:sshkeys}/requests
+keys = ${directory:sshkeys}/keys
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = ${sshkeys-directory:requests}
+keys-directory = ${sshkeys-directory:keys}
+wrapper = ${basedirectory:services}/sshkeys_authority
+keygen-binary = {{dropbear}}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = ${dropbear-server:wrapper}
+public-key = ${dropbear-server:rsa-keyfile}.pub
+private-key = ${dropbear-server:rsa-keyfile}
+wrapper = ${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = ${slap-network-information:global-ipv6}
+port = 2222
+home = ${directory:ssh}
+wrapper = ${rootdirectory:bin}/raw_sshd
+shell = ${rdiff-backup-server:wrapper}
+rsa-keyfile = ${directory:ssh}/server_key.rsa
+dropbear-binary = {{dropbear}}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = ${request-pull-backup-server:connection-ssh-key}
+
+[rdiff-backup-server]
+<= apache-php
+recipe = slapos.cookbook:pbs
+client = false
+path = ${apache-php:htdocs}
+wrapper = ${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = {{buildout}}/rdiff-backup
+
+[request-apache-backup-1]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 1
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-apache-backup-2]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 2
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-pull-backup-server-apache-1]
+<= request-pbs-common
+name = PBS pulling from Apache 1
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+[request-pull-backup-server-apache-2]
+<= request-pbs-common
+name = PBS pulling from Apache 2
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+
+[request-pull-backup-server-apache-backup-1]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-1:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-1:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-1:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+[request-pull-backup-server-apache-backup-2]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-2:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-2:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-2:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+
+[directory]
+ssh = ${rootdirectory:etc}/ssh
+sshkeys = ${rootdirectory:srv}/sshkeys

stack/resilient/buildout.cfg

 [buildout]
 
 parts =
-#Templates needed to setup automatic backup
-  template-pbsready
-  template-pbsready-import
-  template-pbsready-export
+  pbsready
+  pbsready-import
+  pbsready-export
   template-replicated
   template-parts
-
-#Frozen is the state used to not destroy a broken instance's content
-  template-frozen
-
+  instance-frozen
   template-resilient
-  template-switchsoftware
 
 
-[template-pbsready]
+#----------------
+#--
+#-- Profiles needed to setup automated backup and recovery.
+#--
+
+[pbsready]
+# Common parts for pbsready-import and pbsready-export.
+# Provides rdiff-backup, notification queue, ssh authentication,
+# dropbear server, and the bully script.
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pbsready.cfg
-output = ${buildout:directory}/template-pbsready.cfg
-md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99
+url = ${:_profile_base_location_}/pbsready.cfg.in
+output = ${buildout:directory}/pbsready.cfg
+md5sum = a2edaadfe652b4b131626b4801768f40
 mode = 0644
 
-[template-pbsready-import]
+[pbsready-import]
+# An import instance has an importer script, which is called
+# by the parent PBS instance when the dump content is propagated.
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pbsready-import.cfg
-output = ${buildout:directory}/template-pbsready-import.cfg
-md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4
+url = ${:_profile_base_location_}/pbsready-import.cfg.in
+output = ${buildout:directory}/pbsready-import.cfg
+md5sum = eda0c1574d8991f4f9e08e3707c2b04b
 mode = 0644
 
-[template-pbsready-export]
+[pbsready-export]
+# An export instance has an exporter script, and communicates
+# to parent PBS instances to deliver the exported dump.
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pbsready-export.cfg
-output = ${buildout:directory}/template-pbsready-export.cfg
-md5sum = 29d36aac2008b173cb9ce5da9e88c0fa
+url = ${:_profile_base_location_}/pbsready-export.cfg.in
+output = ${buildout:directory}/pbsready-export.cfg
+md5sum = dd56f9c74e580475a17a9afb1d220390
 mode = 0644
 
 [template-pull-backup]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pull-backup.cfg
-output = ${buildout:directory}/template-pull-backup.cfg
-md5sum = f88cc9192a63c88f83a9e5191075534e
+url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
+output = ${buildout:directory}/instance-pull-backup.cfg
+md5sum = 18b88cd012e886fbaa457b03928c2d10
 mode = 0644
 
 [template-replicated]
 recipe = slapos.recipe.download
-url = ${:_profile_base_location_}/template-replicated.cfg
-md5sum = 61780842ccda1600a102456b0da69ac9
+url = ${:_profile_base_location_}/template-replicated.cfg.in
+md5sum = 63b5649f3cf1c9a77315382793d9593f
 mode = 0644
-destination = ${buildout:directory}/template-replicated.cfg
+destination = ${buildout:directory}/template-replicated.cfg.in
 
 [template-parts]
 recipe = slapos.recipe.download
-url = ${:_profile_base_location_}/template-parts.cfg
-md5sum = f5fc27235725f05fdbde76a78ebc363e
+url = ${:_profile_base_location_}/template-parts.cfg.in
+md5sum = c942f82552fcb42fc74a5f896e0cd5f3
 mode = 0644
-destination = ${buildout:directory}/template-parts.cfg
+destination = ${buildout:directory}/template-parts.cfg.in
 
-[template-frozen]
+[instance-frozen]
+# When an instance is detected as broken, its software type is changed to "frozen".
+# On the next run of slapgrid-cp, the buildout profile is replaced by instance-frozen.cfg,
+# which will run without removing any content because it raises an error.
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-frozen.cfg
-output = ${buildout:directory}/template-frozen.cfg
+url = ${:_profile_base_location_}/instance-frozen.cfg.in
+output = ${buildout:directory}/instance-frozen.cfg
 
 [template-resilient]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/resilient.cfg
+url = ${:_profile_base_location_}/resilient.cfg.in
 output = ${buildout:directory}/resilient.cfg
 md5sum = 59e74d290d623de2c1e147e48f284fba
 mode = 0644
 
-[template-switchsoftware]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/switchsoftware.cfg
-output = ${buildout:directory}/switchsoftware.cfg
-md5sum = c94a0ed85fce2e72254ae956dce7e40d
-mode = 0644
\ No newline at end of file

stack/resilient/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg.in

@@ -3,9 +3,9 @@
 parts =
   connection-dict
   pbs
+  logrotate
   cron
   cron-entry-logrotate
-    logrotate
   sshkeys-authority
   sshkeys-dropbear
 
@@ -13,14 +13,51 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[connection-dict]
-recipe = slapos.cookbook:publish
-ssh-key = $${sshkeys-dropbear:public-key-value}
-notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
-feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+home = $${buildout:directory}/home
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+var = $${buildout:directory}/var
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+ssh-home = $${rootdirectory:home}/ssh
+notifier = $${rootdirectory:etc}/notifier
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronoutput = $${basedirectory:log}/cron-ouput
+pbs-backup = $${basedirectory:backup}/pbs
+sshkeys = $${rootdirectory:srv}/sshkeys
+pbs-wrappers = $${rootdirectory:bin}/pbs
+dot-ssh = $${basedirectory:ssh-home}/.ssh
+notifier-feeds = $${basedirectory:notifier}/feeds
+notifier-callbacks = $${basedirectory:notifier}/callbacks
 
 
-## sets up the equeue for the notifier
+
+#----------------
+#--
+#-- Set up the equeue and notifier.
+
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
@@ -29,7 +66,8 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
-
+# notifier.notify adds the [exporter, notifier] to the execution queue
+# notifier.notify.callback sets up a callback
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -42,40 +80,20 @@ wrapper = $${basedirectory:services}/notifier
 server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
 
-## Dropbear Client to provide ssh 
-[dropbear-client]
-recipe = slapos.cookbook:dropbear.client
-dbclient-binary = ${dropbear:location}/bin/dbclient
-wrapper = $${rootdirectory:bin}/ssh
-home = $${basedirectory:ssh-home}
-identity-file = $${basedirectory:ssh-home}/id_rsa
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-wrapper = $${basedirectory:services}/sshkeys_authority
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = pbs
-type = rsa
-executable = $${dropbear-client:wrapper}
-public-key = $${dropbear-client:identity-file}.pub
-private-key = $${dropbear-client:identity-file}
-wrapper = $${rootdirectory:bin}/do_backup
 
+#----------------
+#--
+#-- The pull-backup-server contains every backup (incremental)
+#-- to prevent a corrupt dump from destroying everything.
 
-## The pull-backup-server contains every backup (incremental).
-## to prevent a corrupt dump from destroying everything.
 [pbs]
 <= notifier
 recipe = slapos.cookbook:pbs
@@ -91,10 +109,9 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port}/
 slave-instance-list = $${slap-parameter:slave_instance_list}
 
 
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
+#----------------
+#--
+#-- Deploy cron.
 
 [cron]
 recipe = slapos.cookbook:cron
@@ -105,6 +122,16 @@ cronstamps = $${directory:cronstamps}
 catcher = $${cron-simplelogger:wrapper}
 binary = $${basedirectory:services}/crond
 
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
 [cron-entry-logrotate]
 <= cron
 recipe = slapos.cookbook:cron.d
@@ -115,7 +142,7 @@ command = $${logrotate:wrapper}
 [logrotate]
 recipe = slapos.cookbook:logrotate
 # Binaries
-logrotate-binary = ${logrotate:location}/sbin/logrotate
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
 gzip-binary = ${gzip:location}/bin/gzip
 gunzip-binary = ${gzip:location}/bin/gunzip
 # Directories
@@ -125,15 +152,7 @@ logrotate-entries = $${directory:logrotate-entries}
 backup = $${directory:logrotate-backup}
 state-file = $${rootdirectory:srv}/logrotate.status
 
-[logrotate-entry-equeue]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = equeue
-log = $${equeue:log}
-frequency = daily
-rotate-num = 30
-
-[logrotate-entry-equeue]
+[logrotate-entry-cron]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
 name = cron
@@ -142,45 +161,61 @@ frequency = daily
 rotate-num = 30
 
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+#----------------
+#--
+#-- sshkeys
 
-[basedirectory]
+[sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-cronoutput = $${basedirectory:log}/cron-ouput/
-pbs-backup = $${basedirectory:backup}/pbs/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-sshkeys = $${rootdirectory:srv}/sshkeys
-pbs-wrappers = $${rootdirectory:bin}/pbs/
-dot-ssh = $${basedirectory:ssh-home}/.ssh/
-notifier-feeds = $${basedirectory:notifier}/feeds/
-notifier-callbacks = $${basedirectory:notifier}/callbacks/
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
 
-[basedirectory]
-ssh-home = $${rootdirectory:home}/ssh
-notifier = $${rootdirectory:etc}/notifier/
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = pbs
+type = rsa
+executable = $${dropbear-client:wrapper}
+public-key = $${dropbear-client:identity-file}.pub
+private-key = $${dropbear-client:identity-file}
+wrapper = $${rootdirectory:bin}/do_backup
 
-[rootdirectory]
-home = $${buildout:directory}/home/
 
+#----------------
+#--
+#-- Dropbear.
+
+[dropbear-client]
+recipe = slapos.cookbook:dropbear.client
+dbclient-binary = ${dropbear:location}/bin/dbclient
+wrapper = $${rootdirectory:bin}/ssh
+home = $${basedirectory:ssh-home}
+identity-file = $${basedirectory:ssh-home}/id_rsa
+
+
+#----------------
+#--
+#-- Slave instance list (empty default).
 
-# Default values
 [slap-parameter]
 slave_instance_list = []
+
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[connection-dict]
+recipe = slapos.cookbook:publish
+ssh-key = $${sshkeys-dropbear:public-key-value}
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
+

stack/resilient/instance-pbsready-export.cfg → stack/resilient/pbsready-export.cfg.in

 [buildout]
-extends = ${template-pbsready:output}
+
+extends = ${pbsready:output}
 
 parts += cron-entry-backup
 
 [urls]
 notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
 
-#notify launches executable, and once it's done, notifies the pull-backup-servers.
 [notifier-exporter]
+# notifier.notify launches an (exporter) executable, and when finished,
+# notifies the the pull-backup-servers.
 <= notifier
 recipe = slapos.cookbook:notifier.notify
 name = exporter
@@ -16,8 +18,9 @@ executable = $${exporter:wrapper}
 wrapper = $${rootdirectory:bin}/exporter
 notify = $${slap-parameter:notify}
 
-#adds the exporter to cron
 [cron-entry-backup]
+# Schedule the periodic database dump.
+# Through notifications, this triggers (one or more) incremental backups on PBS instances.
 <= cron
 recipe = slapos.cookbook:cron.d
 name = backup

stack/resilient/instance-pbsready-import.cfg → stack/resilient/pbsready-import.cfg.in

 [buildout]
 
-extends = ${template-pbsready:output}
+extends = ${pbsready:output}
 
 parts += import-on-notification
 
 [urls]
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 
-#Launches callback, when a notification is received
 [import-on-notification]
+# notifier.callback runs a script when a notification (sent by a parent PBS)
+# is received
 <= notifier
 recipe = slapos.cookbook:notifier.callback
 on-notification-id = $${slap-parameter:on-notification}

stack/resilient/instance-pbsready.cfg → stack/resilient/pbsready.cfg.in

 [buildout]
 
-parts =
+parts +=
   resiliency
   urls
   stunnel
@@ -18,21 +18,99 @@ parts =
   dropbear-server-pbs-authorized-key
   notifier
 
-# adds the resiliency script for the bully algorithm
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[basedirectory]
+services = $${rootdirectory:etc}/run
+cache = $${rootdirectory:var}/cache
+notifier = $${rootdirectory:etc}/notifier
+
+[directory]
+backup = $${basedirectory:backup}/$${slap-parameter:namebase}
+ssh = $${rootdirectory:etc}/ssh/
+sshkeys = $${rootdirectory:srv}/sshkeys
+notifier-feeds = $${basedirectory:notifier}/feeds
+notifier-callbacks = $${basedirectory:notifier}/callbacks
+
+
+#----------------
+#--
+#-- Resiliency script for the bully algorithm
+
 [resiliency]
+# If enable-bully-service is true, the scripts will be run automatically.
+# If false, they can be run with bin/bullly for all the PBSReady instances.
+enable-bully-service = False
 recipe = slapos.cookbook:addresiliency
-script = $${basedirectory:script}
-run = $${basedirectory:services}
+wrapper-bully = bully
+wrapper-takeover = takeover
+services = $${basedirectory:services}
+bin = $${rootdirectory:bin}
+etc = $${rootdirectory:etc}
+
+
+#----------------
+#--
+#-- Sets up an rdiff-backup server (with a dropbear server for ssh)
 
-# sets up an rdiff-backup server (with a dropbear server for ssh)
 [urls]
 ssh-public-key = $${sshkeys-dropbear:public-key-value}
 ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:backup}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+
+
+#----------------
+#--
+#-- Set up the equeue and notifier.
+
+[equeue]
+recipe = slapos.cookbook:equeue
+socket = $${basedirectory:run}/equeue.sock
+log = $${basedirectory:log}/equeue.log
+database = $${rootdirectory:srv}/equeue.db
+wrapper = $${basedirectory:services}/equeue
+equeue-binary = ${buildout:bin-directory}/equeue
+
+# notifier.notify adds the [exporter, notifier] to the execution queue
+# notifier.notify.callback sets up a callback
+[notifier]
+recipe = slapos.cookbook:notifier
+feeds = $${directory:notifier-feeds}
+callbacks = $${directory:notifier-callbacks}
+id-file = $${rootdirectory:etc}/notifier.id
+equeue-socket = $${equeue:socket}
+host = $${slap-network-information:global-ipv6}
+port = 8080
+wrapper = $${basedirectory:services}/notifier
+server-binary = ${buildout:bin-directory}/pubsubserver
+notifier-binary = ${buildout:bin-directory}/pubsubnotifier
+
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
+
+
+#----------------
+#--
+#-- sshkeys
+
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -51,6 +129,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
+
+#----------------
+#--
+#-- Dropbear.
+
 [dropbear-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
@@ -66,57 +149,5 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:backup}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-
-## Sets up the execution queue for the notifier
-[logrotate-entry-equeue]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = equeue
-log = $${equeue:log}
-frequency = daily
-rotate-num = 30
-
-[equeue]
-recipe = slapos.cookbook:equeue
-socket = $${basedirectory:run}/equeue.sock
-log = $${basedirectory:log}/equeue.log
-database = $${rootdirectory:srv}/equeue.db
-wrapper = $${basedirectory:services}/equeue
-equeue-binary = ${buildout:bin-directory}/equeue
-
-
-## notifier.notify adds the [exporter, notifier] to the execution queue
-## notifier.notify.callback sets up a callback
-[notifier]
-recipe = slapos.cookbook:notifier
-feeds = $${directory:notifier-feeds}
-callbacks = $${directory:notifier-callbacks}
-id-file = $${rootdirectory:etc}/notifier.id
-equeue-socket = $${equeue:socket}
-host = $${slap-network-information:global-ipv6}
-port = 8080
-wrapper = $${basedirectory:services}/notifier
-server-binary = ${buildout:bin-directory}/pubsubserver
-notifier-binary = ${buildout:bin-directory}/pubsubnotifier
-
 
-[basedirectory]
-script = $${rootdirectory:etc}/script/
-services = $${rootdirectory:etc}/run/
-cache = $${rootdirectory:var}/cache/
-notifier = $${rootdirectory:etc}/notifier/
 
-[directory]
-backup = $${basedirectory:backup}/$${slap-parameter:namebase}
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys
-notifier-feeds = $${basedirectory:notifier}/feeds/
-notifier-callbacks = $${basedirectory:notifier}/callbacks/
-script = $${basedirectory:script}

stack/resilient/template-replicated.cfg → stack/resilient/template-replicated.cfg.in

@@ -2,8 +2,6 @@
 
 ## Tells the Backupable recipe that we want a backup
 [resilient]
-config-script = bully.py
-config-wrapper = bully
 config-namebase = {{namebase}}
 
 ## Every request is double to provide the 3 IPs.
@@ -15,14 +13,11 @@ software-type = {{typeexport}}
 name = {{namebase}}0
 return = url ssh-public-key ssh-url notification-id ip
 
-config = number script wrapper authorized-key notify ip-list namebase
-
+config = number authorized-key notify ip-list namebase
+config-number = 0
 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
-
 config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
-
 config-ip-list =
-config-number = 0
 
 {% for id in range(1,nbbackup|int) %}
 
@@ -37,14 +32,13 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{typeimport}}
 return = url ssh-public-key ssh-url notification-url ip
 
-config = number script wrapper authorized-key on-notification ip-list namebase
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
 
+config = number authorized-key on-notification ip-list namebase
+config-number = {{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
 config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
-pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
-
 config-ip-list =
-config-number = {{id}}
 
 {% endfor %}
 
@@ -60,19 +54,15 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
 recipe = slapos.cookbook:request
 name = {{namebase}}0
 
-config = number script wrapper authorized-key notify ip-list namebase
-
 software-url = ${slap-connection:software-release-url}
 software-type = {{typeexport}}
 return = url ssh-public-key ssh-url notification-id ip
 
-
+config = number authorized-key notify ip-list namebase
+config-number = 0
 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
 config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
 
-
-config-number=0
-
 {% for id in range(1,nbbackup|int) %}
 [request-{{namebase}}-pseudo-replicating-{{id}}-2]
 <= slap-connection
@@ -87,14 +77,13 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{typeimport}}
 return = url ssh-public-key ssh-url notification-url
 
-config = number script wrapper authorized-key on-notification ip-list namebase
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
 
+config = number authorized-key on-notification ip-list namebase
+config-number = {{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
 config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
-pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
 
-
-config-number = {{id}}
 {% endfor %}
 
 
@@ -135,6 +124,8 @@ config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
 config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
 config-title = Pulling from {{namebase}}
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
 
 [request-pull-backup-server-{{namebase}}-backup-{{id}}]
 <= request-pbs-common
@@ -149,6 +140,8 @@ config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-noti
 config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
 config-title = Pushing to {{namebase}} backup {{id}}
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
 {% endfor %}
 
 {% endmacro %}