Merge branch 'lapp' into lapp-resilient2

slapos/recipe/addresiliency/__init__.py

@@ -26,56 +26,55 @@
 ##############################################################################
 from slapos.recipe.librecipe import GenericSlapRecipe
 
-import sys
 import os
 
 
 class Recipe(GenericSlapRecipe):
-  """ This class provides the installation of the resilience
-      script on the partition.
-  """
+    """ This class provides the installation of the resilience
+        script on the partition.
+    """
 
-  def _install(self):
-    path_list = []
-    self_id = int(self.parameter_dict['number'])
-    ip = self.parameter_dict['ip-list'].split(' ')
-    print 'Creating bully script with  ips : %s\n' % ip
-    slap_connection = self.buildout['slap-connection']
+    def _install(self):
+        path_list = []
+
+        confpath = os.path.join(self.options['etc'], 'bully.conf')
+
+        ip_list = self.parameter_dict['ip-list']
+        print 'Creating bully configuration with ips : %s\n' % ip_list
+
+        conf = self.createFile(confpath,
+                               self.substituteTemplate(
+                               self.getTemplateFilename('bully.conf.in'),
+                               {
+                                   'self_id': int(self.parameter_dict['number']),
+                                   'ip_list': ip_list
+                                   }
+                               ))
+        path_list.append(conf)
+
+        slap_connection = self.buildout['slap-connection']
+
+        if self.optionIsTrue('enable-bully-service', default=False):
+            wrapper_dir = self.options['services']
+        else:
+            wrapper_dir = self.options['bin']
+
+        wrapper = self.createPythonScript(
+            name=os.path.join(wrapper_dir, self.parameter_dict['wrapper']),
+            absolute_function='slapos.recipe.addresiliency.bully.run',
+            arguments={
+                'confpath': confpath,
+                'server_url': slap_connection['server-url'],
+                'key_file': slap_connection.get('key-file'),
+                'cert_file': slap_connection.get('cert-file'),
+                'computer_id': slap_connection['computer-id'],
+                'partition_id': slap_connection['partition-id'],
+                'software': slap_connection['software-release-url'],
+                'namebase': self.parameter_dict['namebase'],
+            })
+
+        path_list.append(wrapper)
+
+        return path_list
 
-    path_conf = os.path.join(self.options['script'], 'conf.in')
-    path_bully = os.path.join(self.options['script'], self.parameter_dict['script'])
-    path_bully_new = os.path.join(self.options['script'], 'new.py')
-    path_run = os.path.join(self.options['run'], self.parameter_dict['wrapper'])
-    print 'paths: %s\n%s\n' % (path_run, path_bully)
-    bully_conf = dict(self_id=self_id,
-                      ip_list=ip,
-                      executable=sys.executable,
-                      syspath=sys.path,
-                      server_url=slap_connection['server-url'],
-                      key_file=slap_connection.get('key-file'),
-                      cert_file=slap_connection.get('cert-file'),
-                      computer_id=slap_connection['computer-id'],
-                      partition_id=slap_connection['partition-id'],
-                      software=slap_connection['software-release-url'],
-                      namebase=self.parameter_dict['namebase'],
-                      confpath=path_conf)
-    try:
-      conf = self.createFile(path_conf,
-                             self.substituteTemplate(
-                             self.getTemplateFilename('conf.in.in'),
-                             bully_conf))
-      path_list.append(conf)
-      script = self.createExecutable(path_bully,
-                                     self.substituteTemplate(
-                                     self.getTemplateFilename('bully.py.in'),
-                                     bully_conf))
-      path_list.append(script)
 
-      wrapper = self.createPythonScript(
-          path_run,
-          'slapos.recipe.librecipe.execute.execute',
-          [path_bully])
-      path_list.append(wrapper)
-    except IOError:
-      pass
-    return path_list

slapos/recipe/addresiliency/bully.py

+# -*- coding: utf-8 -*-
+
+import logging
+import Queue
+import socket
+import thread
+import time
+
+from slapos import slap as slapmodule
+import slapos
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+BASE_PORT = 50000
+SLEEPING_MINS = 2           # XXX was 30, increase after testing
+
+MSG_PING = 'ping'
+MSG_HALT = 'halt'
+MSG_VICTORY = 'victory'
+
+MSG_OK = 'ok'
+
+STATE_NORMAL = 'normal'
+STATE_WAITINGCONFIRM = 'waitingConfirm'
+STATE_ELECTION = 'election'
+STATE_REORGANIZATION = 'reorganization'
+
+
+
+
+class Renamer(object):
+    def __init__(self, server_url, key_file, cert_file, computer_guid,
+                 partition_id, software_release, namebase):
+        self.server_url = server_url
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.computer_guid = computer_guid
+        self.partition_id = partition_id
+        self.software_release = software_release
+        self.namebase = namebase
+
+
+    def _failover(self):
+        """\
+        This method does
+
+        - retrieve the broken computer partition
+        - change its reference to 'broken-...' and its software type to 'frozen'
+        - retrieve the winner computer partition (attached to this process)
+        - change its reference and software type to replace the broken one
+
+        Then, after running slapgrid-cp a few times, the winner takes over and
+        a new cp is created to replace it as an importer.
+        """
+
+        # TODO: replace hardcoded strings with values from the API
+
+        slap = slapmodule.slap()
+        slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
+
+        # partition that will take over.
+        cp_winner = slap.registerComputerPartition(computer_guid=self.computer_guid,
+                                                   partition_id=self.partition_id)
+        # XXX although we can already rename cp_winner, to change its software type we need to
+        # get hold of the root cp as well
+
+        root_partition_id = 'slappart0'       # XXX hardcoded. what's the API for this?
+
+        cp_root = slap.registerComputerPartition(computer_guid=self.computer_guid,
+                                                 partition_id=root_partition_id)
+
+        cp_exporter_ref = self.namebase + '0'       # this is ok. the boss is always number zero.
+
+        # partition to be deactivated
+        cp_broken = cp_root.request(software_release=self.software_release,
+                                    software_type='frozen',
+                                    partition_reference=cp_exporter_ref)
+
+        broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
+
+        # XXX can we retrieve and log the old reference name?
+        log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
+
+        cp_broken.rename(new_name=broken_new_ref)
+
+        cp_broken.stopped()
+
+        log.debug("Renaming {}: {}".format(cp_broken.getId(), cp_exporter_ref))
+
+        # update software type and name for the partition that will take over
+
+        cp_winner_ref = self.namebase + '2'     # XXX hardcoded. what's the API for this?
+
+        cp_root.request(software_release=self.software_release,
+                        software_type=self.namebase+'-export',
+                        partition_reference=cp_winner_ref).rename(new_name=cp_exporter_ref)
+
+
+
+    def failover(self):
+        try:
+            self._failover()
+            log.info('Renaming done')
+        except slapos.slap.slap.ServerError:
+            log.info('Internal server error')
+
+
+
+## Leader is always number 0
+
+class ResilientInstance(object):
+
+    def __init__(self, comm, renamer, confpath):
+        self.comm = comm
+        self.participant_id = 0
+        self.state = STATE_NORMAL
+        self.halter_id = 0
+        self.inElection = False
+        self.alive = True
+
+        self.mainCanal = self.comm.create_canal([MSG_PING, MSG_HALT, MSG_VICTORY])
+
+        self.renamer = renamer
+        self.okCanal = self.comm.create_canal([MSG_OK])
+        self.confpath = confpath
+        self.loadConnectionInfo()
+
+    def loadConnectionInfo(self):
+        params = open(self.confpath, 'r').readlines()
+        self.total_participants = len(params[0].split())
+        new_id = int(params[1])
+        if self.participant_id != new_id:
+            self.halter_id = new_id
+            self.participant_id = new_id 
+
+    ## Needs to be changed to use the master
+    def aliveManagement(self):
+        while self.alive:
+            log.info('XXX sleeping for %d minutes' % SLEEPING_MINS)
+            time.sleep(SLEEPING_MINS*60)
+            if self.participant_id == 0:
+                continue
+            self.comm.send(MSG_PING, 0)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            self.election()
+
+    def listen(self):
+        while self.alive:
+            self.comm.recv()
+
+    def main(self):
+        while self.alive:
+            message, sender = self.mainCanal.get()
+            if message == MSG_PING:
+                self.comm.send(MSG_OK, sender)
+
+            elif message == MSG_HALT:
+                self.state = STATE_WAITINGCONFIRM
+                self.halter_id = int(sender)
+                self.comm.send(MSG_OK, sender)
+
+            elif message == MSG_VICTORY:
+                if int(sender) == self.halter_id and self.state == STATE_WAITINGCONFIRM:
+                    log.info('{} thinks {} is the leader'.format(self.participant_id, sender))
+                    self.comm.send(MSG_OK, sender)
+                self.state = STATE_NORMAL
+
+    def election(self):
+        self.inElection = True
+        self.loadConnectionInfo()
+        # Check if I'm the highest instance alive
+        for higher in range(self.participant_id + 1, self.total_participants):
+            self.comm.send(MSG_PING, higher)
+            message, sender = self.okCanal.get()
+            if message:
+                log.info('{} is alive ({})'.format(higher, self.participant_id))
+                self.inElection = False
+                return False
+            continue
+
+        if not self.alive:
+            return False
+
+        # I should be the new coordinator, halt those below me
+        log.info('Should be ME : {}'.format(self.participant_id))
+        self.state = STATE_ELECTION
+        self.halter_id = self.participant_id
+        ups = []
+        for lower in range(self.participant_id):
+            self.comm.send(MSG_HALT, lower)
+            message, sender = self.okCanal.get()
+            if message:
+                ups.append(lower)
+
+        #Broadcast Victory
+        self.state = STATE_REORGANIZATION
+        for up in ups:
+            self.comm.send(MSG_VICTORY, up)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            log.info('Something is wrong... let\'s start over')
+            return self.election()
+        self.state = STATE_NORMAL
+        self.active = True
+        log.info('{} Is THE LEADER'.format(self.participant_id))
+
+        self.renamer.failover()
+
+        self.inElection = False
+
+        return True
+
+
+
+class FilteredCanal(object):
+
+    def __init__(self, accept, timeout):
+        self.accept = accept
+        self.queue = Queue.Queue()
+        self.timeout = timeout
+
+    def append(self, message, sender):
+        if message in self.accept:
+            self.queue.put([message, sender])
+
+    def get(self):
+        try:
+            return self.queue.get(timeout=self.timeout)
+        except Queue.Empty:
+            return [None, None]
+
+
+
+class Wrapper(object):
+
+    def __init__(self, confpath, timeout=20):
+        self.canals = []
+        self.ips = []
+        self.participant_id = 0
+        self.timeout = timeout
+        self.confpath = confpath
+        self.getConnectionInfo()
+        self.socket = None
+
+    def getConnectionInfo(self):
+        params = open(self.confpath, 'r').readlines()
+        self.ips = params[0].split()
+        self.participant_id = int(params[1])
+
+    def start(self):
+        self.getConnectionInfo()
+        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        self.socket.bind((self.ips[self.participant_id], BASE_PORT + self.participant_id))
+        self.socket.listen(5)
+
+    def send(self, message, number):
+        self.getConnectionInfo()
+        try:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            s.connect((self.ips[number], BASE_PORT + number))
+            s.send(message + (' {}\n'.format(self.participant_id)))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+            pass
+        finally:
+            s.close()
+
+    def create_canal(self, accept):
+        created = FilteredCanal(accept, self.timeout)
+        self.canals.append(created)
+        return created
+
+    def recv(self):
+        client, _ = self.socket.accept()
+        client_message = client.recv(1024)
+        if client_message:
+            message, sender = client_message.split()
+            for canal in self.canals:
+                canal.append(message, int(sender))
+
+
+
+
+def run(args):
+    confpath = args.pop('confpath')
+
+    renamer = Renamer(server_url = args.pop('server_url'),
+                      key_file = args.pop('key_file'),
+                      cert_file = args.pop('cert_file'),
+                      computer_guid = args.pop('computer_id'),
+                      partition_id = args.pop('partition_id'),
+                      software_release = args.pop('software'),
+                      namebase = args.pop('namebase'))
+
+    if args:
+        raise ValueError('Unknown arguments: %s' % ', '.join(args))
+
+    wrapper = Wrapper(confpath=confpath, timeout=20)
+
+    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
+
+    # idle waiting for connection infos
+    while computer.total_participants < 2:
+        computer.loadConnectionInfo()
+        time.sleep(30)
+
+    log.info('Starting')
+
+    computer.comm.start()
+    thread.start_new_thread(computer.listen, ())
+    thread.start_new_thread(computer.aliveManagement, ())
+
+    computer.main()
+

slapos/recipe/addresiliency/template/bully.py.in

@@ -8,6 +8,7 @@ import sys
 
 sys.path[:] = %(syspath)s
 
+import slapos
 from slapos import slap as slapmodule
 
 port = 50000
@@ -37,9 +38,11 @@ def rename_broken_and_stop():
     slap.initializeConnection('%(server_url)s',
                               '%(key_file)s',
                               '%(cert_file)s')
-    computer_partition = slap.registerComputerPartition('%(computer_id)s',
-                                                        '%(partition_id)s')
-    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
+    computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s',
+                                                        partition_id='%(partition_id)s')
+    broken = computer_partition.request(software_release='%(software)s',
+                                        software_type='frozen',
+                                        partition_reference='%(namebase)s0')
 
     broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
     broken.stopped()

slapos/recipe/addresiliency/template/bully_new.py.in

 #!%(executable)s
 
+import logging
+import os
 import socket
-import time
 import sys
 import thread
 import time
-import os
 
 sys.path[:] = %(syspath)s
 
 from slapos import slap as slapmodule
+import slapos
+
+BASE_PORT = 50000
+SLEEPING_MINS = 2
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+
+class Renamer(object):
+    def __init__(self, server_url, key_file, cert_file, computer_guid,
+                 partition_id, software_release, namebase):
+        self.server_url = server_url
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.computer_guid = computer_guid
+        self.partition_id = partition_id
+        self.software_release = software_release
+        self.namebase = namebase
+
+    def _failover(self):
+        slap = slapmodule.slap()
+        slap.initializeConnection(self.server_url,
+                                  self.key_file,
+                                  self.cert_file)
+        computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid,
+                                                            partition_id=self.partition_id)
+        broken = computer_partition.request(software_release=self.software_release,
+                                            software_type='frozen',
+                                            partition_reference=self.namebase+'0')
+
+        broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+        broken.stopped()
+        computer_partition.rename(self.namebase+'0')
+
+    def failover(self):
+        try:
+            log.info('renaming done')
+        except slapos.slap.slap.ServerError:
+            log.info('Internal server error')
 
-port = 50000
-size = 1024
-
-
-def rename_broken_and_stop():
-  try:
-    slap = slapmodule.slap()
-    slap.initializeConnection('%(server_url)s',
-                              '%(key_file)s',
-                              '%(cert_file)s')
-    computer_partition = slap.registerComputerPartition('%(computer_id)s',
-                                                        '%(partition_id)s')
-    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
-
-    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
-    broken.stopped()
-    computer_partition.rename('%(namebase)s0')
-    print 'renaming done\n'
-  except slapos.slap.slap.ServerError:
-    print 'Internal server error\n'
 
 
 ## Leader is always number 0
 
 class ResilientInstance(object):
 
-    def __init__(self, comm):
+    def __init__(self, comm, renamer, confpath):
         self.comm = comm
         self.id = 0
         self.state = 'normal'
         self.halter = 0
-        self.nbComp = nbComp
         self.inElection = False
         self.alive = True
         self.lastPing = time.clock()
 
-        self.mainCanal = self.comm.canal(['ping', 'halt',
-                                          'victory'])
+        self.mainCanal = self.comm.canal(['ping', 'halt', 'victory'])
 
+        self.renamer = renamer
         self.okCanal = self.comm.canal(['ok'])
+        self.confpath = confpath
         self.loadConnectionInfos()
 
-
     def loadConnectionInfos(self):
-        file = open('%(confpath)s', 'r')
+        file = open(self.confpath, 'r')
         params = file.read().split('\n')
         file.close()
         self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
@@ -67,7 +87,8 @@ class ResilientInstance(object):
     ## Needs to be changed to use the master
     def aliveManagement(self):
         while self.alive:
-            time.sleep(30*60)
+            log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS)
+            time.sleep(SLEEPING_MINS*60)
             if self.id == 0:
                 continue
             self.comm.send('ping', 0)
@@ -84,7 +105,7 @@ class ResilientInstance(object):
         while self.alive:
             message, sender = self.mainCanal.get()
             if message == 'ping':
-                    self.comm.send('ok', sender)
+                self.comm.send('ok', sender)
 
             elif message == 'halt':
                 self.state = 'waitingConfirm'
@@ -93,7 +114,7 @@ class ResilientInstance(object):
 
             elif message == 'victory':
                 if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
-                    print '%s thinks %s is the leader\n' % (self.id, sender)
+                    log.info('{} thinks {} is the leader'.format(self.id, sender))
                     self.comm.send('ok', sender)
                 self.state = 'normal'
 
@@ -105,7 +126,7 @@ class ResilientInstance(object):
             self.comm.send('ping', higher)
             message, sender = self.okCanal.get()
             if message:
-                #print '%s is alive (%s)\n' % (higher, self.id)
+                log.info('{} is alive ({})'.format(higher, self.id))
                 self.inElection = False
                 return False
             continue
@@ -114,7 +135,7 @@ class ResilientInstance(object):
             return False
 
         #I should be the new coordinator, halt those below me
-        print 'Should be ME : %s \n' % self.id
+        log.info('Should be ME : {}'.format(self.id))
         self.state = 'election'
         self.halter = self.id
         ups = []
@@ -131,13 +152,13 @@ class ResilientInstance(object):
             message, sender = self.okCanal.get()
             if message:
                 continue
-            print 'Something is wrong... let\'s start over\n'
+            log.info('Something is wrong... let\'s start over')
             return self.election()
         self.state = 'normal'
         self.active = True
-        print '%s Is THE LEADER \n' % self.id
+        log.info('{} Is THE LEADER'.format(self.id))
 
-        rename_broken_and_stop()
+        self.renamer.failover()
 
         self.inElection = False
 
@@ -164,27 +185,24 @@ class FilteredCanal(object):
             self.lock.acquire()
             if self.list:
                 self.lock.release()
-                val = self.list[0]
-                self.list = self.list[1:]
-                return val
+                return self.list.pop(0)
             self.lock.release()
         return [None, None]
 
 
 class Wrapper(object):
 
-    def __init__(self, timeout=20):
-        self.read_pipes = [os.fdopen(x) for x in read_pipes]
-        self.write_pipes = write_pipes
+    def __init__(self, confpath, timeout=20):
         self.canals = []
         self.ips = []
         self.id = 0
         self.timeout = timeout
+        self.confpath = confpath
         self.getConnectionInfos()
         self.socket = None
 
     def getConnectionInfos(self):
-        file = open('%(confpath)s', 'r')
+        file = open(self.confpath, 'r')
         params = file.read().split('\n')
         file.close()
         self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
@@ -193,17 +211,17 @@ class Wrapper(object):
     def start(self):
         self.getConnectionInfos()
         self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        self.socket.bind((self.ips[self.id], port + self.id))
-        s.listen(5)
+        self.socket.bind((self.ips[self.id], BASE_PORT + self.id))
+        self.socket.listen(5)
 
     def send(self, message, number):
         self.getConnectionInfos()
         try:
             s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-            s.connect((self.ips[number], port + number))
-            s.send(message + (' %s\n' % self.id))
+            s.connect((self.ips[number], BASE_PORT + number))
+            s.send(message + (' {}\n'.format(self.id)))
         except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-          pass
+            pass
         finally:
             s.close()
 
@@ -213,31 +231,48 @@ class Wrapper(object):
         return created
 
     def recv(self):
-        client, _ = s.accept()
+        client, _ = self.socket.accept()
         client_message = client.recv(1024)
         if client_message:
             message, sender = client_message.split()
             for canal in self.canals:
-                canal.append(message, sender)
+                canal.append(message, int(sender))
+
+
+
+def main():
+
+    renamer = Renamer(server_url = '%(server_url)s',
+                      key_file = '%(key_file)s',
+                      cert_file = '%(cert_file)s',
+                      computer_guid = '%(computer_id)s',
+                      partition_id = '%(partition_id)s',
+                      software_release = '%(software)s',
+                      namebase = '%(namebase)s')
+
+    confpath = '%(confpath)s'
 
+    wrapper = Wrapper(confpath=confpath, timeout=20)
 
-wrapper = createWrapper(20)
+    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
 
-computer = ResilientInstance(wrapper)
+    #idle waiting for connection infos
+    while computer.nbComp < 2 :
+        computer.loadConnectionInfos()
+        time.sleep(30)
 
-#idle waiting for connection infos
-while computer.nbComp < 2 :
-    computer.loadConnectionInfos()
-    time.sleep(30)
+    log.info('Starting')
 
-print 'Starting\n'
+    computer.comm.start()
+    thread.start_new_thread(computer.listen, ())
+    thread.start_new_thread(computer.main, ())
+    thread.start_new_thread(computer.aliveManagement, ())
 
-computer.comm.start()
-thread.start_new_thread(computer.listen, ())
-thread.start_new_thread(computer.main, ())
-thread.start_new_thread(computer.aliveManagement, ())
+    while True:
+        # XXX tight loop
+        continue
 
-while True:
-    continue
 
+if __name__ == '__main__':
+    main()
 

slapos/recipe/dcron.py

@@ -33,26 +33,22 @@ class Recipe(GenericBaseRecipe):
   def install(self):
     self.logger.info("Installing dcron...")
 
-    path_list = []
+    options = self.options
+    script = self.createWrapper(name=options['binary'],
+                                command=options['dcrond-binary'].strip(),
+                                parameters=[
+                                    '-s', options['cron-entries'],
+                                    '-c', options['crontabs'],
+                                    '-t', options['cronstamps'],
+                                    '-f', '-l', '5',
+                                    '-M', options['catcher']
+                                    ])
 
-    cronstamps = self.options['cronstamps']
-    cron_d = self.options['cron-entries']
-    crontabs = self.options['crontabs']
-    catcher = self.options['catcher']
-
-    binary = self.options['binary']
-
-    script = self.createPythonScript(binary,
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['dcrond-binary'].strip(), '-s', cron_d, '-c', crontabs,
-       '-t', cronstamps, '-f', '-l', '5', '-M', catcher]
-      )
-    path_list.append(script)
     self.logger.debug('Main cron executable created at : %r', script)
 
     self.logger.info("dcron successfully installed.")
 
-    return path_list
+    return [script]
 
 
 

slapos/recipe/dropbear.py

@@ -158,6 +158,24 @@ class Client(GenericBaseRecipe):
 
     return [wrapper]
 
+
+def keysplit(s):
+  """
+  Split a string like "ssh-rsa AKLFKJSL..... ssh-rsa AAAASAF...."
+  and return the individual key_type + key strings.
+  """
+  si = iter(s.split(' '))
+  while True:
+    key_type = next(si)
+    try:
+      key_value = next(si)
+    except StopIteration:
+      # odd number of elements, should not happen, yield the last one by itself
+      yield key_type
+      break
+    yield '%s %s' % (key_type, key_value)
+
+
 class AddAuthorizedKey(GenericBaseRecipe):
 
   def install(self):
@@ -167,7 +185,9 @@ class AddAuthorizedKey(GenericBaseRecipe):
     path_list.append(ssh)
 
     authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
-    for key in self.options['key'].split(' '):
+    for key in keysplit(self.options['key']):
+      # XXX key might actually be the string 'None' or 'null'
       authorized_keys.append(key)
 
     return path_list
+

slapos/recipe/equeue.py

@@ -30,17 +30,19 @@ class Recipe(GenericBaseRecipe):
 
   def install(self):
 
-    commandline = [self.options['equeue-binary']]
-    commandline.extend(['--database', self.options['database']])
-    commandline.extend(['-l', self.options['log']])
+    parameters = [
+      '--database', self.options['database'],
+      '-l', self.options['log'],
+    ]
 
     if 'loglevel' in self.options:
-      commandline.extend(['--loglevel', self.options['loglevel']])
+      parameters.extend(['--loglevel', self.options['loglevel']])
 
-    commandline.append(self.options['socket'])
+    parameters.append(self.options['socket'])
+
+    wrapper = self.createWrapper(name=self.options['wrapper'],
+                                 command=self.options['equeue-binary'],
+                                 parameters=parameters)
+
+    return [wrapper]
 
-    return [self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      commandline,
-    )]

slapos/recipe/librecipe/generic.py

+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
 ##############################################################################
 #
 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
@@ -24,6 +26,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+import io
 import logging
 import os
 import sys
@@ -90,6 +93,21 @@ class GenericBaseRecipe(object):
   def createExecutable(self, name, content, mode=0700):
     return self.createFile(name, content, mode)
 
+  def addLineToFile(self, filepath, line, encoding='utf8'):
+    """Append a single line to a text file, if the line does not exist yet.
+    
+    line must be unicode."""
+
+    try:
+      lines = io.open(filepath, 'r', encoding=encoding).readlines()
+    except IOError:
+      lines = []
+
+    if not line in lines:
+      lines.append(line)
+      with io.open(filepath, 'w+', encoding=encoding) as f:
+        f.write(u'\n'.join(lines))
+
   def createPythonScript(self, name, absolute_function, arguments=''):
     """Create a python script using zc.buildout.easy_install.scripts
 
@@ -115,7 +133,6 @@ class GenericBaseRecipe(object):
     Takes care of quoting.
     """
 
-    q = shlex.quote
     lines = [
             '#!/bin/sh',
             'exec %s' % shlex.quote(command)

slapos/recipe/mydumper.py

@@ -28,85 +28,76 @@ import subprocess
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
-def dump(args):
-  mydumper_cmd = [args['mydumper']]
-  mydumper_cmd.extend(['-B', args['database']])
 
-  if args['socket'] is not None:
-    mydumper_cmd.extend(['-S', args['socket']])
-  else:
-    mydumper_cmd.extend(['-h', args['host']])
-    mydumper_cmd.etxned(['-P', args['port']])
-
-  mydumper_cmd.extend(['-u', args['user']])
-  if args['password'] is not None:
-    mydumper_cmd.extend(['-p', args['password']])
-
-  if args['compression']:
-    mydumper_cmd.append('--compress')
+def _mydumper_base_cmd(mydumper, database, user, password,
+                       socket=None, host=None, port=None, **kw):
+  cmd = [mydumper]
+  cmd.extend(['-B', database])
 
-  if args['rows'] is not None:
-    mydumper_cmd.extend(['-r', args['rows']])
-
-  mydumper_cmd.extend(['-o', args['directory']])
+  if socket:
+    cmd.extend(['-S', socket])
+  else:
+    cmd.extend(['-h', host])
+    cmd.extend(['-P', port])
 
-  subprocess.check_call(mydumper_cmd)
+  cmd.extend(['-u', user])
+  if password:
+    cmd.extend(['-p', password])
 
+  return cmd
 
 
-def do_import(args):
-  mydumper_cmd = [args['mydumper']]
-  mydumper_cmd.extend(['-B', args['database']])
+def do_export(args):
+  cmd = _mydumper_base_cmd(**args)
 
-  if args['socket'] is not None:
-    mydumper_cmd.extend(['-S', args['socket']])
-  else:
-    mydumper_cmd.extend(['-h', args['host']])
-    mydumper_cmd.etxned(['-P', args['port']])
+  if args['compression']:
+    cmd.append('--compress')
 
-  mydumper_cmd.extend(['-u', args['user']])
-  if args['password'] is not None:
-    mydumper_cmd.extend(['-p', args['password']])
+  if args['rows'] is not None:
+    cmd.extend(['-r', args['rows']])
 
-  mydumper_cmd.append('--overwrite-tables')
+  cmd.extend(['-o', args['directory']])
 
-  mydumper_cmd.extend(['-d', args['directory']])
+  subprocess.check_call(cmd)
 
-  subprocess.check_call(mydumper_cmd)
 
+def do_import(args):
+  cmd = _mydumper_base_cmd(**args)
+  cmd.append('--overwrite-tables')
+  cmd.extend(['-d', args['directory']])
+  subprocess.check_call(cmd)
 
 
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    # Host or socket should be defined
-    try:
-      self.options['host']
-    except:
-      self.options['socket']
-
-    config = dict(database=self.options['database'],
-                  socket=self.options.get('socket'),
-                  host=self.options.get('host'),
-                  port=self.options.get('port', 3306),
-                  directory=self.options['backup-directory'],
-                  user=self.options['user'],
-                  password=self.options.get('password'),
-                 )
-
-    name = __name__
+    config = {
+      'database': self.options['database'],
+      'directory': self.options['backup-directory'],
+      'user': self.options['user'],
+      'password': self.options.get('password'),
+    }
+
+    if self.options.get('host'):
+      config['host'] = self.options['host']
+      config['port'] = self.options.get('port', 3306)
+    elif self.options.get('socket'):
+      config['socket'] = self.options['socket']
+    else:
+      raise ValueError("host or socket must be defined")
+
     if self.optionIsTrue('import', False):
-      config.update(mydumper=self.options['myloader-binary'])
-      name += '.do_import'
+      function = do_import
+      config['mydumper'] = self.options['myloader-binary']
     else:
-      config.update(mydumper=self.options['mydumper-binary'],
-                    compression=self.optionIsTrue('compression', default=False),
-                    rows=self.options.get('rows'),
-                   )
-      name += '.dump'
+      function = do_export
+      config['mydumper'] = self.options['mydumper-binary']
+      config['compression'] = self.optionIsTrue('compression', default=False)
+      config['rows'] = self.options.get('rows')
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-                                      name,
-                                      config)
+    wrapper = self.createPythonScript(name=self.options['wrapper'],
+                                      absolute_function = '%s.%s' % (__name__, function.func_name),
+                                      arguments=config)
 
     return [wrapper]
+

slapos/recipe/notifier.py

@@ -31,62 +31,68 @@ from slapos.recipe.librecipe import GenericBaseRecipe
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    commandline = [self.options['server-binary']]
-    commandline.extend(['--callbacks', self.options['callbacks']])
-    commandline.extend(['--feeds', self.options['feeds']])
-    commandline.extend(['--equeue-socket', self.options['equeue-socket']])
-    commandline.append(self.options['host'])
-    commandline.append(self.options['port'])
+    options = self.options
+    script = self.createWrapper(name=options['wrapper'],
+                                command=options['server-binary'],
+                                parameters=[
+                                   '--callbacks', options['callbacks'],
+                                   '--feeds', options['feeds'],
+                                   '--equeue-socket', options['equeue-socket'],
+                                   options['host'], options['port']
+                                   ])
+    return [script]
 
-    return [self.createPythonScript(self.options['wrapper'],
-                                    'slapos.recipe.librecipe.execute.execute',
-                                    commandline)]
 
 class Callback(GenericBaseRecipe):
 
   def createCallback(self, notification_id, callback):
     callback_id = sha512(notification_id).hexdigest()
-    callback = self.createFile(os.path.join(self.options['callbacks'],
-                                            callback_id),
-                               callback)
-    return callback
+
+    filepath = os.path.join(self.options['callbacks'], callback_id)
+    self.addLineToFile(filepath, callback)
+    return filepath
 
   def install(self):
+    # XXX this path is returned multiple times, one for each callback that has been added.
     return [self.createCallback(self.options['on-notification-id'],
                                 self.options['callback'])]
 
 class Notify(GenericBaseRecipe):
 
-  def createNotifier(self, notifier_binary, executable, wrapper, **kwargs):
-    if not os.path.exists(kwargs['log']):
+  def createNotifier(self, notifier_binary, wrapper, executable,
+                     log, title, notification_url, feed_url):
+
+    if not os.path.exists(log):
       # Just a touch
-      open(kwargs['log'], 'w').close()
+      open(log, 'w').close()
 
-    commandline = [notifier_binary,
-                   '-l', kwargs['log'],
-                   '--title', kwargs['title'],
-                   '--feed', kwargs['feed_url'],
-                   '--notification-url']
+    parameters = [
+            '-l', log,
+            '--title', title,
+            '--feed', feed_url,
+            '--notification-url',
+            ]
+    parameters.extend(notification_url.split(' '))
+    parameters.extend(['--executable', executable])
 
-    commandline.extend(kwargs['notification_url'].split(' '))
-    commandline.extend(['--executable', executable])
+    return self.createWrapper(name=wrapper,
+                              command=notifier_binary,
+                              parameters=parameters)
 
-    return self.createPythonScript(wrapper,
-                                   'slapos.recipe.librecipe.execute.execute',
-                                   [str(i) for i in commandline])
 
   def install(self):
-    feedurl = self.unparseUrl(scheme='http', host=self.options['host'],
-                              port=self.options['port'],
-                              path='/get/%s' % self.options['name'])
-
-    script = self.createNotifier(
-      self.options['notifier-binary'],
-      wrapper=self.options['wrapper'],
-      executable=self.options['executable'],
-      log=os.path.join(self.options['feeds'], self.options['name']),
-      title=self.options['title'],
-      notification_url=self.options['notify'],
-      feed_url=feedurl,
-    )
+    feed_url = self.unparseUrl(scheme='http', host=self.options['host'],
+                               port=self.options['port'],
+                               path='/get/%s' % self.options['name'])
+
+    log = os.path.join(self.options['feeds'], self.options['name'])
+
+    options = self.options
+    script = self.createNotifier(notifier_binary=options['notifier-binary'],
+                                 wrapper=options['wrapper'],
+                                 executable=options['executable'],
+                                 log=log,
+                                 title=options['title'],
+                                 notification_url=options['notify'],
+                                 feed_url=feed_url)
     return [script]

slapos/recipe/pbs.py

@@ -24,14 +24,15 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from json import loads as unjson
+
 from hashlib import sha512
-from urlparse import urlparse
+import inspect
+import json
 import os
+import signal
 import subprocess
 import sys
-import signal
-import inspect
+import urlparse
 
 from slapos.recipe.librecipe import GenericSlapRecipe
 from slapos.recipe.dropbear import KnownHostsFile
@@ -43,8 +44,7 @@ from slapos import slap as slapmodule
 def promise(args):
 
   def failed_ssh(partition, ssh):
-    # Bad python 2 syntax, looking forward python 3 to have print(file=)
-    print >> sys.stderr, "SSH Connection failed"
+    sys.stderr.write("SSH Connection failed\n")
     try:
       ssh.terminate()
     except:
@@ -75,16 +75,20 @@ def promise(args):
 
   slap = slapmodule.slap()
   slap.initializeConnection(args['server_url'],
-    key_file=args.get('key_file'), cert_file=args.get('cert_file'))
+                            key_file=args.get('key_file'),
+                            cert_file=args.get('cert_file'))
+
   partition = slap.registerComputerPartition(args['computer_id'],
                                              args['partition_id'])
 
+  ssh = subprocess.Popen([args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
+                         stdin=subprocess.PIPE,
+                         stdout=open(os.devnull, 'w'),
+                         stderr=open(os.devnull, 'w'))
+
   # Rdiff Backup protocol quit command
   quitcommand = 'q' + chr(255) + chr(0) * 7
-  ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
 
-  ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
-                         stdout=open(os.devnull), stderr=open(os.devnull))
   ssh.stdin.write(quitcommand)
   ssh.stdin.flush()
   ssh.stdin.close()
@@ -113,7 +117,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     promise_path = os.path.join(self.options['promises-directory'],
                                 url_hash)
-    parsed_url = urlparse(url)
+    parsed_url = urlparse.urlparse(url)
     promise_dict = self.promise_base_dict.copy()
     promise_dict.update(user=parsed_url.username,
                         host=parsed_url.hostname,
@@ -134,8 +138,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
         'host': parsed_url.hostname,
       }
 
-    command = [self.options['rdiffbackup-binary']]
-    command.extend(['--remote-schema', remote_schema])
+    parameters = ['--remote-schema', remote_schema]
 
     remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
                                                'path': parsed_url.path}
@@ -144,38 +147,36 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
                                            name_hash)
 
     if entry['type'] == 'push':
-      command.extend(['--restore-as-of', 'now'])
-      command.append('--force')
-      command.extend([local_directory, remote_directory])
+      parameters.extend(['--restore-as-of', 'now'])
+      parameters.append('--force')
+      parameters.extend([local_directory, remote_directory])
     else:
-      command.extend([remote_directory, local_directory])
+      parameters.extend([remote_directory, local_directory])
 
     wrapper_basepath = os.path.join(self.options['wrappers-directory'],
                                     url_hash)
 
-    wrapper_path = wrapper_basepath
     if 'notify' in entry:
-      wrapper_path = '%s_raw' % wrapper_basepath
+      wrapper_path = wrapper_basepath + '_raw'
+    else:
+      wrapper_path = wrapper_basepath
 
-    wrapper = self.createPythonScript(
-       wrapper_path,
-      'slapos.recipe.librecipe.execute.execute',
-      [str(i) for i in command]
-    )
+    wrapper = self.createWrapper(name=wrapper_path,
+                                 command=self.options['rdiffbackup-binary'],
+                                 parameters=parameters)
     path_list.append(wrapper)
 
     if 'notify' in entry:
       feed_url = '%s/get/%s' % (self.options['notifier-url'],
                                 entry['notification-id'])
-      wrapper = self.createNotifier(
-        self.options['notifier-binary'],
-        wrapper=wrapper_basepath,
-        executable=wrapper_path,
-        log=os.path.join(self.options['feeds'], entry['notification-id']),
-        title=entry.get('title', 'Untitled'),
-        notification_url=entry['notify'],
-        feed_url=feed_url,
-      )
+      wrapper = self.createNotifier(notifier_binary=self.options['notifier-binary'],
+                                    wrapper=wrapper_basepath,
+                                    executable=wrapper_path,
+                                    log=os.path.join(self.options['feeds'], entry['notification-id']),
+                                    title=entry.get('title', 'Untitled'),
+                                    notification_url=entry['notify'],
+                                    feed_url=feed_url,
+                                  )
       path_list.append(wrapper)
       #self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference'])
 
@@ -190,40 +191,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     return path_list
 
+
   def _install(self):
     path_list = []
 
-
     if self.optionIsTrue('client', True):
       self.logger.info("Client mode")
 
       slap_connection = self.buildout['slap-connection']
-      self.promise_base_dict = dict(
-        server_url=slap_connection['server-url'],
-        computer_id=slap_connection['computer-id'],
-        cert_file=slap_connection.get('cert-file'),
-        key_file=slap_connection.get('key-file'),
-        partition_id=slap_connection['partition-id'],
-        ssh_client=self.options['sshclient-binary'],
-      )
-
-      slaves = unjson(self.options['slave-instance-list'])
+      self.promise_base_dict = {
+              'server_url': slap_connection['server-url'],
+              'computer_id': slap_connection['computer-id'],
+              'cert_file': slap_connection.get('cert-file'),
+              'key_file': slap_connection.get('key-file'),
+              'partition_id': slap_connection['partition-id'],
+              'ssh_client': self.options['sshclient-binary'],
+        }
+
+      slaves = json.loads(self.options['slave-instance-list'])
       known_hosts = KnownHostsFile(self.options['known-hosts'])
       with known_hosts:
+        # XXX this API could be cleaner
         for slave in slaves:
           path_list.extend(self.add_slave(slave, known_hosts))
-
-
     else:
-      command = [self.options['rdiffbackup-binary']]
       self.logger.info("Server mode")
-      command.extend(['--restrict', self.options['path']])
-      command.append('--server')
 
-      wrapper = self.createPythonScript(
-        self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.execute',
-        command)
+      wrapper = self.createWrapper(name=self.options['wrapper'],
+                                   command=self.options['rdiffbackup-binary'],
+                                   parameters=[
+                                       '--restrict', self.options['path'],
+                                       '--server'
+                                       ])
       path_list.append(wrapper)
 
     return path_list
+

slapos/recipe/request.py

@@ -123,20 +123,30 @@ class Recipe(object):
 
     isSlave = options.get('slave', '').lower() in \
         librecipe.GenericBaseRecipe.TRUE_VALUES
-    self.instance = instance = request(software_url, software_type,
+    self.instance = request(software_url, software_type,
       name, partition_parameter_kw=partition_parameter_kw,
       filter_kw=filter_kw, shared=isSlave)
 
+    self._raise_resource_not_ready = None
+    try:
+        # XXX what is the right way to get a global id?
+        options['instance_guid'] = self.instance.getId()
+    except slapmodule.ResourceNotReady as exc:
+        self._raise_resource_not_ready = exc
+
     for param in return_parameters:
       try:
         options['connection-%s' % param] = str(
-          instance.getConnectionParameter(param))
+          self.instance.getConnectionParameter(param))
       except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady):
         options['connection-%s' % param] = ''
         if self.failed is None:
           self.failed = param
 
   def install(self):
+    if self._raise_resource_not_ready:
+      raise slapmodule.ResourceNotReady(self._resource_not_ready)
+
     if self.failed is not None:
       # Check instance status to know if instance has been deployed
       try:

software/drupal/software.cfg

@@ -12,7 +12,6 @@ recipe = slapos.recipe.build:download-unpacked
 url = http://ftp.drupal.org/files/projects/drupal-7.16.tar.gz
 md5sum = 352497b2df94b5308e31cb8da020b631
 
-
 [download-patch-hide-dbsetup]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}

stack/lamp/apache/instance-apache-backup.cfg → stack/lamp/apache/instance-apache-backup.cfg.in

@@ -16,32 +16,92 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[urls]
-recipe = slapos.cookbook:publish
-url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
-ssh-public-key = $${sshkeys-dropbear:public-key-value}
-ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
-[apache-proxy]
-recipe = slapos.cookbook:apacheproxy
-url = $${slap-parameter:proxy-url}
-pid-file = $${basedirectory:run}/apache.pid
-lock-file = $${basedirectory:run}/apache.lock
-ip = $${slap-network-information:global-ipv6}
-port = 8080
-error-log = $${directory:httpd-log}/error.log
-access-log = $${directory:httpd-log}/access.log
-httpd-conf = $${rootdirectory:etc}/apache.conf
-wrapper = $${basedirectory:services}/apache
+#----------------
+#--
+#-- Creation of all needed directories.
 
-promise = $${basedirectory:promises}/apache
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
 
-httpd-binary = ${apache:location}/bin/httpd
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+ssh = $${rootdirectory:etc}/ssh
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- sshkeys
 
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -60,6 +120,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
+
+#----------------
+#--
+#-- Dropbear.
+
 [dropbear-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
@@ -75,6 +140,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
+
+#----------------
+#--
+#-- rdiff
+
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
 client = false
@@ -82,18 +152,26 @@ path = $${directory:htdocs}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
+
+#----------------
+#--
+#-- Apache Proxy.
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
 
 [logrotate-entry-apache]
 <= logrotate
@@ -106,51 +184,14 @@ sharedscripts = true
 notifempty = true
 create = true
 
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+#----------------
+#--
+#-- Publish instance parameters.
 
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+[urls]
+recipe = slapos.cookbook:publish
+url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
+ssh-public-key = $${sshkeys-dropbear:public-key-value}
+ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-htdocs = $${rootdirectory:srv}/www/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys
-httpd-log = $${basedirectory:log}/apache/

stack/lamp/apache/instance-apache-import.cfg

 [buildout]
-extends = ${template-apache-php:output}
-          ${template-pbsready-export:output}
+extends = ${instance-apache-php:output}
+          ${pbsready-export:output}
 
 parts =
   apache-proxy
@@ -13,6 +13,7 @@ parts =
   dropbear-server
   dropbear-server-pbs-authorized-key
 
+
 [apache-proxy]
 recipe = slapos.cookbook:apacheproxy
 url = $${slap-parameter:proxy-url}
@@ -29,10 +30,103 @@ promise = $${basedirectory:promises}/apache
 
 httpd-binary = ${apache:location}/bin/httpd
 
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+ssh = $${rootdirectory:etc}/ssh
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-proxy:error-log} $${apache-proxy:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+
+#----------------
+#--
+#-- sshkeys
+
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -51,6 +145,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
+
+#----------------
+#--
+#-- Dropbear.
+
 [dropbear-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
@@ -66,6 +165,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
+
+#----------------
+#--
+#-- rdiff
+
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
 client = false
@@ -73,75 +177,3 @@ path = $${directory:htdocs}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
-
-[logrotate-entry-apache]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = apache
-log = $${apache-proxy:error-log} $${apache-proxy:access-log}
-frequency = daily
-rotate-num = 30
-sharedscripts = true
-notifempty = true
-create = true
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-htdocs = $${rootdirectory:srv}/www/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys
-httpd-log = $${basedirectory:log}/apache/

stack/lamp/apache/instance-apache-php.cfg → stack/lamp/apache/instance-apache-php.cfg.in

@@ -20,46 +20,96 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
-# Creation of all needed directories
+#----------------
+#--
+#-- Creation of all needed directories.
+
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
 
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-ca-dir = $${rootdirectory:srv}/ssl/
-httpd-log = $${basedirectory:log}/apache/
-php-ini-dir = $${rootdirectory:etc}/php/
-tmp-php = $${rootdirectory:tmp}/php/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-stunnel-conf = $${rootdirectory:etc}/stunnel/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
+ca-dir = $${rootdirectory:srv}/ssl
+httpd-log = $${basedirectory:log}/apache
+php-ini-dir = $${rootdirectory:etc}/php
+tmp-php = $${rootdirectory:tmp}/php
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+stunnel-conf = $${rootdirectory:etc}/stunnel
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:ca-dir}/requests/
-private = $${directory:ca-dir}/private/
-certs = $${directory:ca-dir}/certs/
-newcerts = $${directory:ca-dir}/newcerts/
-crl = $${directory:ca-dir}/crl/
+requests = $${directory:ca-dir}/requests
+private = $${directory:ca-dir}/private
+certs = $${directory:ca-dir}/certs
+newcerts = $${directory:ca-dir}/newcerts
+crl = $${directory:ca-dir}/crl
+
 
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- Deploy stunnel.
 
-# Deploy stunnel
 [stunnel]
 recipe = slapos.cookbook:stunnel
 client = true
@@ -76,8 +126,22 @@ pid-file = $${basedirectory:run}/stunnel.pid
 wrapper = $${rootdirectory:bin}/raw_stunnel
 post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
 
+[logrotate-entry-stunnel]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = stunnel
+log = $${stunnel:log-file}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+post = $${stunnel:post-rotate-script}
+
+
+#----------------
+#--
+#-- Certificate stuff.
 
-# Certificate stuffs
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${openssl:location}/bin/openssl
@@ -98,7 +162,10 @@ key-file = $${stunnel:key-file}
 cert-file = $${stunnel:cert-file}
 
 
-# Request MariaDB instance and parse its URL
+#----------------
+#--
+#-- Request MariaDB instance and parse its URL.
+
 [request-mariadb]
 <= slap-connection
 recipe = slapos.cookbook:request
@@ -114,7 +181,10 @@ recipe = slapos.cookbook:urlparse
 url = $${request-mariadb:connection-url}
 
 
-# Deploy Apache + PHP application
+#----------------
+#--
+#-- Deploy Apache + PHP application.
+
 [apache-php]
 recipe = slapos.cookbook:apachephp
 source = ${application:location}
@@ -142,21 +212,6 @@ mysql-database = $${mariadb-urlparse:path}
 mysql-host = $${stunnel:local-host}
 mysql-port = $${stunnel:local-port}
 
-
-# Deploy logrotate, cron, configure it
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
-
 [logrotate-entry-apache]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
@@ -168,40 +223,11 @@ sharedscripts = true
 notifempty = true
 create = true
 
-[logrotate-entry-stunnel]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = stunnel
-log = $${stunnel:log-file}
-frequency = daily
-rotate-num = 30
-notifempty = true
-create = true
-post = $${stunnel:post-rotate-script}
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
 
+#----------------
+#--
+#-- Request frontend.
 
-# Request frontend
 [request-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -215,7 +241,10 @@ return = site_url
 config-custom_domain = $${slap-parameter:domain}
 
 
-# Deploy slapmonitor
+#----------------
+#--
+#-- Deploy slapmonitor.
+
 [slapmonitor]
 recipe = slapos.cookbook:slapmonitor
 pid-file = $${basedirectory:run}/apache.pid
@@ -238,14 +267,20 @@ slapreport-path = ${buildout:bin-directory}/slapreport
 path = $${basedirectory:services}/slapreport
 
 
-# Publish all instance parameters (url of instance)
+#----------------
+#--
+#-- Publish all instance parameters (url of instance).
+
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
 backend_url = $${apache-php:url}
 url = $${request-frontend:connection-site_url}
 
 
-# Deploy promises scripts
+#----------------
+#--
+#-- Deploy promises scripts.
+
 [promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${basedirectory:promises}/apache
@@ -266,6 +301,9 @@ url = $${request-frontend:connection-site_url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
+
+
+
 [slap-parameter]
 # Default value if no domain is specified
 domain =
@@ -274,3 +312,4 @@ logbox-ip =
 logbox-port =
 logbox-user =
 logbox-passwd =
+

stack/lamp/buildout.cfg

@@ -13,13 +13,16 @@ parts =
   instance-recipe-egg
   template
 
-  template-apache-php
+  instance-apache-php
   template-mariadb
 
 #Contains the importer and exporter recipes for mariadb
   template-mariadb-import
   template-mariadb-export
 
+  instance-default-root
+
+
 extends =
   ../resilient/buildout.cfg
   ../../component/mariadb/buildout.cfg
@@ -61,66 +64,80 @@ recipe = hexagonit.recipe.download
 #If provided tarball does not containt top directory this option shall be changed to false
 strip-top-level-dir = true
 
+
+#----------------
+#-- Instance-level buildout profiles.
+
 [template]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-output = ${buildout:directory}/template.cfg
-md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+md5sum = 9e6a4adaa6b5ac923ad16e637bf5170c
 mode = 0644
 
-[template-apache-php]
+[instance-apache-php]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/apache/instance-apache-php.cfg
-output = ${buildout:directory}/template-apache-php.cfg
-md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
+output = ${buildout:directory}/instance-apache-php.cfg
+md5sum = dec36ec4fe54d87f4a5e5b8573f72fb7
 mode = 0644
 
-[template-apache-backup]
+[instance-apache-backup]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg
-output = ${buildout:directory}/template-apache-backup.cfg
-md5sum = cfb77ac8785e0d125a785f69a5339014
+url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
+output = ${buildout:directory}/instance-apache-backup.cfg
+md5sum = 48f969d82319a9d145570f5f0fd27672
 mode = 0644
 
 [template-resilient-lamp]
 recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance-resilient.cfg
-rendered = ${buildout:directory}/template-resilient.cfg
+template = ${:_profile_base_location_}/template-resilient.cfg.in
+rendered = ${buildout:directory}/instance-resilient.cfg
 
-context = key templateapache template-apache-php:output
+context = key templateapache instance-apache-php:output
           key dropbear dropbear:location
           key buildout buildout:bin-directory
 
 import-list = file parts template-parts:destination
               file replicated template-replicated:destination
 
-md5sum = 03aafcba5c626a4a1bd180d71007be1e
+md5sum = 5151295af3994499e2be6d68938787d3
 mode = 0644
  
 
 [template-mariadb]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg
-output = ${buildout:directory}/template-mariadb.cfg
-md5sum = fa9dc10efbcf61119f4cbab37c741322
+url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg.in
+output = ${buildout:directory}/instance-mariadb.cfg
+md5sum = aa33c843f04753a34154467522f0c65e
 mode = 0644
 
 [template-mariadb-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg
-output = ${buildout:directory}/template-mariadb-import.cfg
-md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg.in
+output = ${buildout:directory}/instance-mariadb-import.cfg
+md5sum = 24a9286f1aed04f7393c92f251cbe0c9
 mode = 0644
 
 [template-mariadb-export]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg
-output = ${buildout:directory}/template-mariadb-export.cfg
-md5sum = 4b7dec765265b27c8235419b82ca7b02
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg.in
+output = ${buildout:directory}/instance-mariadb-export.cfg
+md5sum = 96e915bca045e3ff62a3f8fc6e692685
+mode = 0644
+
+
+[instance-default-root]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-default-root.cfg.in
+output = ${buildout:directory}/instance-default-root.cfg
+md5sum = 53c9020f7a0b5203f976e069e455787b
 mode = 0644
 
 
-# Dummy parts in case no application configuration file is needed
+#----------------
+#-- Dummy parts in case no application configuration file is needed
+
 [application-template]
 filename =
 location =
@@ -128,6 +145,8 @@ location =
 [application-configuration]
 location =
 
+#----------------
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =

stack/lamp/instance-default-root.cfg.in

+[buildout]
+parts = request-apache
+
+[request-apache]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = $${slap-connection:software-release-url}
+software-type = apache
+name = Apache
+

stack/lamp/instance.cfg → stack/lamp/instance.cfg.in

 [buildout]
 
-extends = 
-  ${template-switchsoftware:output}
-
 parts =
   switch_softwaretype
 
@@ -12,12 +9,16 @@ offline = true
 
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = ${template-apache-php:output}
+default = ${instance-default-root:output}
+apache = ${instance-apache-php:output}
 resilient = ${template-resilient-lamp:rendered}
 mariadb = ${template-mariadb:output}
 mariadb-import = ${template-mariadb-import:output}
 mariadb-export = ${template-mariadb-export:output}
+apache-backup = ${instance-apache-backup:output}
+
+#frozen creates a syntax error, meaning it can keep its data.
+#It's dirty as hell, it needs to be replaced.
+frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
-apache-backup = ${template-apache-backup:output}
-frozen = ${template-frozen:output}
 

stack/lamp/mariadb/instance-mariadb-export.cfg → stack/lamp/mariadb/instance-mariadb-export.cfg.in

 [buildout]
 extends = ${template-mariadb:output}
-          ${template-pbsready-export:output}
+          ${pbsready-export:output}
 
 parts += mariadb
 
@@ -12,4 +12,4 @@ socket = $${mariadb:socket}
 user = root
 mydumper-binary = ${mydumper:location}/bin/mydumper
 database = $${mariadb:database}
-import = false
\ No newline at end of file
+import = false

stack/lamp/mariadb/instance-mariadb-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg.in

 [buildout]
 extends = ${template-mariadb:output}
-          ${template-pbsready-import:output}
+          ${pbsready-import:output}
 
 parts += mariadb
 

stack/lamp/mariadb/instance-mariadb.cfg → stack/lamp/mariadb/instance-mariadb.cfg.in

@@ -16,107 +16,74 @@ parts =
 
 gzip-binary = ${gzip:location}/bin/gzip
 
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[urls]
-recipe = slapos.cookbook:publish
-url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
-ip = $${slap-network-information:global-ipv6}
 
-[mariadb]
-recipe = slapos.cookbook:mysql
+#----------------
+#--
+#-- Creation of all needed directories.
 
-# Options
-recovering = false
-user = user
-port = 3306
-ip = $${slap-network-information:local-ipv4}
-database = db
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
 
-# Paths
-wrapper = $${basedirectory:services}/mariadb
-update-wrapper = $${basedirectory:services}/mariadb_update
-logrotate-post = $${rootdirectory:bin}/mariadb_post_logrotate
-data-directory = $${directory:mariadb-data}
-pid-file = $${basedirectory:run}/mariadb.pid
-socket = $${basedirectory:run}/mariadb.sock
-error-log = $${basedirectory:log}/mariadb_error.log
-conf-file = $${rootdirectory:etc}/mariadb.cnf
-promise = $${basedirectory:promises}/mysql
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+script = $${rootdirectory:etc}/script
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
 
-# Binary information
-mysql-base-directory = ${mariadb:location}
-mysql-binary = ${mariadb:location}/bin/mysql
-mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
-mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
-mysqld-binary = ${mariadb:location}/bin/mysqld
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+mariadb-data = $${rootdirectory:srv}/mariadb
+logrotate-backup = $${basedirectory:backup}/logrotate
+stunnel-conf = $${rootdirectory:etc}/stunnel
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
 
 
-[slapmonitor]
-recipe = slapos.cookbook:slapmonitor
-pid-file = $${basedirectory:run}/mariadb.pid
-database-path = $${basedirectory:log}/slapmonitor.db
-shell-path = ${dash:location}/bin/dash
-slapmonitor-path = ${buildout:bin-directory}/slapmonitor
-path = $${basedirectory:services}/slapmonitor
+#----------------
+#--
+#-- Deploy cron.
 
-[slapreport]
-recipe = slapos.cookbook:slapreport
-pid-file = $${basedirectory:run}/mariadb.pid
-consumption-log-path = $${basedirectory:log}/instance_consumption.log
-database-path = $${basedirectory:log}/slapmonitor.db
-logbox-ip = 87.98.152.12
-logbox-port = 5122
-logbox-user = admin
-logbox-passwd = passer
-shell-path = ${dash:location}/bin/dash
-slapreport-path = ${buildout:bin-directory}/slapreport
-path = $${basedirectory:services}/slapreport
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
 
-[certificate-authority]
-recipe = slapos.cookbook:certificate_authority
-openssl-binary = ${openssl:location}/bin/openssl
-ca-dir = $${directory:ca-dir}
-requests-directory = $${cadirectory:requests}
-wrapper = $${basedirectory:services}/ca
-ca-private = $${cadirectory:private}
-ca-certs = $${cadirectory:certs}
-ca-newcerts = $${cadirectory:newcerts}
-ca-crl = $${cadirectory:crl}
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
 
-[cadirectory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:ca-dir}/requests/
-private = $${directory:ca-dir}/private/
-certs = $${directory:ca-dir}/certs/
-newcerts = $${directory:ca-dir}/newcerts/
-crl = $${directory:ca-dir}/crl/
 
-[ca-stunnel]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-executable = $${stunnel:wrapper}
-wrapper = $${basedirectory:services}/stunnel
-key-file = $${stunnel:key-file}
-cert-file = $${stunnel:cert-file}
+#----------------
+#--
+#-- Deploy logrotate.
 
-[stunnel]
-recipe = slapos.cookbook:stunnel
-stunnel-binary = ${stunnel:location}/bin/stunnel
-wrapper = $${rootdirectory:bin}/stunnel
-log-file = $${basedirectory:log}/stunnel.log
-config-file = $${directory:stunnel-conf}/stunnel.conf
-key-file = $${directory:stunnel-conf}/stunnel.key
-cert-file = $${directory:stunnel-conf}/stunnel.crt
-pid-file = $${basedirectory:run}/stunnel.pid
-local-host = $${mariadb:ip}
-local-port = $${mariadb:port}
-remote-host = $${slap-network-information:global-ipv6}
-remote-port = 6446
-client = false
-post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
 
 [logrotate]
 recipe = slapos.cookbook:logrotate
@@ -164,53 +131,127 @@ rotate-num = 30
 notifempty = true
 create = true
 
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
+#----------------
+#--
+#-- Deploy stunnel.
 
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
+[stunnel]
+recipe = slapos.cookbook:stunnel
+stunnel-binary = ${stunnel:location}/bin/stunnel
+wrapper = $${rootdirectory:bin}/stunnel
+log-file = $${basedirectory:log}/stunnel.log
+config-file = $${directory:stunnel-conf}/stunnel.conf
+key-file = $${directory:stunnel-conf}/stunnel.key
+cert-file = $${directory:stunnel-conf}/stunnel.crt
+pid-file = $${basedirectory:run}/stunnel.pid
+local-host = $${mariadb:ip}
+local-port = $${mariadb:port}
+remote-host = $${slap-network-information:global-ipv6}
+remote-port = 6446
+client = false
+post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
 
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
+#----------------
+#--
+#-- Certificate stuff.
 
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-script = $${rootdirectory:etc}/script/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
 
-[directory]
+[cadirectory]
 recipe = slapos.cookbook:mkdirectory
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-ca-dir = $${rootdirectory:srv}/ssl/
-mariadb-data = $${rootdirectory:srv}/mariadb/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-stunnel-conf = $${rootdirectory:etc}/stunnel/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+
+#----------------
+#--
+#-- Creates a MariaDB configuration file, and a database.
+
+[mariadb]
+recipe = slapos.cookbook:mysql
+
+# Options
+recovering = false
+user = user
+port = 3306
+ip = $${slap-network-information:local-ipv4}
+database = db
+
+# Paths
+wrapper = $${basedirectory:services}/mariadb
+update-wrapper = $${basedirectory:services}/mariadb_update
+logrotate-post = $${rootdirectory:bin}/mariadb_post_logrotate
+data-directory = $${directory:mariadb-data}
+pid-file = $${basedirectory:run}/mariadb.pid
+socket = $${basedirectory:run}/mariadb.sock
+error-log = $${basedirectory:log}/mariadb_error.log
+conf-file = $${rootdirectory:etc}/mariadb.cnf
+promise = $${basedirectory:promises}/mysql
+
+# Binary information
+mysql-base-directory = ${mariadb:location}
+mysql-binary = ${mariadb:location}/bin/mysql
+mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
+mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
+mysqld-binary = ${mariadb:location}/bin/mysqld
+
+
+#----------------
+#--
+#-- Deploy slapmonitor.
+
+[slapmonitor]
+recipe = slapos.cookbook:slapmonitor
+pid-file = $${basedirectory:run}/mariadb.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+shell-path = ${dash:location}/bin/dash
+slapmonitor-path = ${buildout:bin-directory}/slapmonitor
+path = $${basedirectory:services}/slapmonitor
+
+[slapreport]
+recipe = slapos.cookbook:slapreport
+pid-file = $${basedirectory:run}/mariadb.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+consumption-log-path = $${basedirectory:log}/instance_consumption.log
+logbox-ip = 87.98.152.12
+logbox-port = 5122
+logbox-user = admin
+logbox-passwd = passer
+shell-path = ${dash:location}/bin/dash
+slapreport-path = ${buildout:bin-directory}/slapreport
+path = $${basedirectory:services}/slapreport
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[urls]
+recipe = slapos.cookbook:publish
+url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
+ip = $${slap-network-information:global-ipv6}
+
 
 [slap-parameter]
 #Default  value if no ssh parameters specified

stack/lamp/instance-resilient.cfg → stack/lamp/template-resilient.cfg.in

@@ -44,8 +44,8 @@ slave = false
 
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = ${directory:sshkeys}/requests/
-keys = ${directory:sshkeys}/keys/
+requests = ${directory:sshkeys}/requests
+keys = ${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -121,6 +121,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url}
 config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
 config-frequency = 30 * * * *
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
 
 [request-pull-backup-server-apache-2]
 <= request-pbs-common
@@ -134,6 +136,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url}
 config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
 config-frequency = 30 * * * *
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
 
 
 [request-pull-backup-server-apache-backup-1]
@@ -146,6 +150,8 @@ config-type = push
 config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
 config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
 
 [request-pull-backup-server-apache-backup-2]
 <= request-pbs-common
@@ -157,8 +163,10 @@ config-type = push
 config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
 config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
 
 
 [directory]
-ssh = ${rootdirectory:etc}/ssh/
+ssh = ${rootdirectory:etc}/ssh
 sshkeys = ${rootdirectory:srv}/sshkeys

stack/resilient/buildout.cfg

 [buildout]
 
 parts =
-#Templates needed to setup automatic backup
-  template-pbsready
-  template-pbsready-import
-  template-pbsready-export
+  pbsready
+  pbsready-import
+  pbsready-export
   template-replicated
   template-parts
-
-#Frozen is the state used to not destroy a broken instance's content
-  template-frozen
-
+  instance-frozen
   template-resilient
-  template-switchsoftware
 
 
-[template-pbsready]
+#----------------
+#--
+#-- Profiles needed to setup automated backup and recovery.
+#--
+
+[pbsready]
+# Common parts for pbsready-import and pbsready-export.
+# Provides rdiff-backup, notification queue, ssh authentication,
+# dropbear server, and the bully script.
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pbsready.cfg
-output = ${buildout:directory}/template-pbsready.cfg
-md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99
+url = ${:_profile_base_location_}/pbsready.cfg.in
+output = ${buildout:directory}/pbsready.cfg
+md5sum = b6102416d000cae81dd2b06268946ea9
 mode = 0644
 
-[template-pbsready-import]
+[pbsready-import]
+# An import instance has an importer script, which is called
+# by the parent PBS instance when the dump content is propagated.
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pbsready-import.cfg
-output = ${buildout:directory}/template-pbsready-import.cfg
-md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4
+url = ${:_profile_base_location_}/pbsready-import.cfg.in
+output = ${buildout:directory}/pbsready-import.cfg
+md5sum = eda0c1574d8991f4f9e08e3707c2b04b
 mode = 0644
 
-[template-pbsready-export]
+[pbsready-export]
+# An export instance has an exporter script, and communicates
+# to parent PBS instances to deliver the exported dump.
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pbsready-export.cfg
-output = ${buildout:directory}/template-pbsready-export.cfg
-md5sum = 29d36aac2008b173cb9ce5da9e88c0fa
+url = ${:_profile_base_location_}/pbsready-export.cfg.in
+output = ${buildout:directory}/pbsready-export.cfg
+md5sum = dd56f9c74e580475a17a9afb1d220390
 mode = 0644
 
 [template-pull-backup]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pull-backup.cfg
-output = ${buildout:directory}/template-pull-backup.cfg
-md5sum = f88cc9192a63c88f83a9e5191075534e
+url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
+output = ${buildout:directory}/instance-pull-backup.cfg
+md5sum = 18b88cd012e886fbaa457b03928c2d10
 mode = 0644
 
 [template-replicated]
 recipe = slapos.recipe.download
-url = ${:_profile_base_location_}/template-replicated.cfg
-md5sum = 61780842ccda1600a102456b0da69ac9
+url = ${:_profile_base_location_}/template-replicated.cfg.in
+md5sum = 1017d919dbf41904f04f5c17dcb574fa
 mode = 0644
-destination = ${buildout:directory}/template-replicated.cfg
+destination = ${buildout:directory}/template-replicated.cfg.in
 
 [template-parts]
 recipe = slapos.recipe.download
-url = ${:_profile_base_location_}/template-parts.cfg
-md5sum = f5fc27235725f05fdbde76a78ebc363e
+url = ${:_profile_base_location_}/template-parts.cfg.in
+md5sum = c942f82552fcb42fc74a5f896e0cd5f3
 mode = 0644
-destination = ${buildout:directory}/template-parts.cfg
+destination = ${buildout:directory}/template-parts.cfg.in
 
-[template-frozen]
+[instance-frozen]
+# When an instance is detected as broken, its software type is changed to "frozen".
+# On the next run of slapgrid-cp, the buildout profile is replaced by instance-frozen.cfg,
+# which will run without removing any content because it raises an error.
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-frozen.cfg
-output = ${buildout:directory}/template-frozen.cfg
+url = ${:_profile_base_location_}/instance-frozen.cfg.in
+output = ${buildout:directory}/instance-frozen.cfg
 
 [template-resilient]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/resilient.cfg
+url = ${:_profile_base_location_}/resilient.cfg.in
 output = ${buildout:directory}/resilient.cfg
 md5sum = 59e74d290d623de2c1e147e48f284fba
 mode = 0644
 
-[template-switchsoftware]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/switchsoftware.cfg
-output = ${buildout:directory}/switchsoftware.cfg
-md5sum = c94a0ed85fce2e72254ae956dce7e40d
-mode = 0644
\ No newline at end of file

stack/resilient/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg.in

 [buildout]
 
 parts =
-    connection-dict
-    pbs
-    cron
-    cron-entry-logrotate
-    logrotate
-    sshkeys-authority
-    sshkeys-dropbear
+  connection-dict
+  pbs
+  logrotate
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  sshkeys-dropbear
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[connection-dict]
-recipe = slapos.cookbook:publish
-ssh-key = $${sshkeys-dropbear:public-key-value}
-notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
-feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
 
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+home = $${buildout:directory}/home
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+var = $${buildout:directory}/var
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+ssh-home = $${rootdirectory:home}/ssh
+notifier = $${rootdirectory:etc}/notifier
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronoutput = $${basedirectory:log}/cron-ouput
+pbs-backup = $${basedirectory:backup}/pbs
+sshkeys = $${rootdirectory:srv}/sshkeys
+pbs-wrappers = $${rootdirectory:bin}/pbs
+dot-ssh = $${basedirectory:ssh-home}/.ssh
+notifier-feeds = $${basedirectory:notifier}/feeds
+notifier-callbacks = $${basedirectory:notifier}/callbacks
+
+
+
+#----------------
+#--
+#-- Set up the equeue and notifier.
 
-## sets up the equeue for the notifier
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
@@ -29,7 +66,8 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
-
+# notifier.notify adds the [exporter, notifier] to the execution queue
+# notifier.notify.callback sets up a callback
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -42,40 +80,20 @@ wrapper = $${basedirectory:services}/notifier
 server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
 
-## Dropbear Client to provide ssh 
-[dropbear-client]
-recipe = slapos.cookbook:dropbear.client
-dbclient-binary = ${dropbear:location}/bin/dbclient
-wrapper = $${rootdirectory:bin}/ssh
-home = $${basedirectory:ssh-home}
-identity-file = $${basedirectory:ssh-home}/id_rsa
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-wrapper = $${basedirectory:services}/sshkeys_authority
 
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = pbs
-type = rsa
-executable = $${dropbear-client:wrapper}
-public-key = $${dropbear-client:identity-file}.pub
-private-key = $${dropbear-client:identity-file}
-wrapper = $${rootdirectory:bin}/do_backup
+#----------------
+#--
+#-- The pull-backup-server contains every backup (incremental)
+#-- to prevent a corrupt dump from destroying everything.
 
-
-## The pull-backup-server contains every backup (incremental).
-## to prevent a corrupt dump from destroying everything.
 [pbs]
 <= notifier
 recipe = slapos.cookbook:pbs
@@ -91,10 +109,9 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port}/
 slave-instance-list = $${slap-parameter:slave_instance_list}
 
 
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
+#----------------
+#--
+#-- Deploy cron.
 
 [cron]
 recipe = slapos.cookbook:cron
@@ -105,6 +122,16 @@ cronstamps = $${directory:cronstamps}
 catcher = $${cron-simplelogger:wrapper}
 binary = $${basedirectory:services}/crond
 
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
 [cron-entry-logrotate]
 <= cron
 recipe = slapos.cookbook:cron.d
@@ -115,7 +142,7 @@ command = $${logrotate:wrapper}
 [logrotate]
 recipe = slapos.cookbook:logrotate
 # Binaries
-logrotate-binary = ${logrotate:location}/sbin/logrotate
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
 gzip-binary = ${gzip:location}/bin/gzip
 gunzip-binary = ${gzip:location}/bin/gunzip
 # Directories
@@ -125,15 +152,7 @@ logrotate-entries = $${directory:logrotate-entries}
 backup = $${directory:logrotate-backup}
 state-file = $${rootdirectory:srv}/logrotate.status
 
-[logrotate-entry-equeue]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = equeue
-log = $${equeue:log}
-frequency = daily
-rotate-num = 30
-
-[logrotate-entry-equeue]
+[logrotate-entry-cron]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
 name = cron
@@ -142,45 +161,61 @@ frequency = daily
 rotate-num = 30
 
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+#----------------
+#--
+#-- sshkeys
 
-[basedirectory]
+[sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-cronoutput = $${basedirectory:log}/cron-ouput/
-pbs-backup = $${basedirectory:backup}/pbs/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-sshkeys = $${rootdirectory:srv}/sshkeys
-pbs-wrappers = $${rootdirectory:bin}/pbs/
-dot-ssh = $${basedirectory:ssh-home}/.ssh/
-notifier-feeds = $${basedirectory:notifier}/feeds/
-notifier-callbacks = $${basedirectory:notifier}/callbacks/
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
 
-[basedirectory]
-ssh-home = $${rootdirectory:home}/ssh
-notifier = $${rootdirectory:etc}/notifier/
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = pbs
+type = rsa
+executable = $${dropbear-client:wrapper}
+public-key = $${dropbear-client:identity-file}.pub
+private-key = $${dropbear-client:identity-file}
+wrapper = $${rootdirectory:bin}/do_backup
 
-[rootdirectory]
-home = $${buildout:directory}/home/
 
+#----------------
+#--
+#-- Dropbear.
+
+[dropbear-client]
+recipe = slapos.cookbook:dropbear.client
+dbclient-binary = ${dropbear:location}/bin/dbclient
+wrapper = $${rootdirectory:bin}/ssh
+home = $${basedirectory:ssh-home}
+identity-file = $${basedirectory:ssh-home}/id_rsa
+
+
+#----------------
+#--
+#-- Slave instance list (empty default).
 
-# Default values
 [slap-parameter]
 slave_instance_list = []
+
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[connection-dict]
+recipe = slapos.cookbook:publish
+ssh-key = $${sshkeys-dropbear:public-key-value}
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
+

stack/resilient/instance-pbsready-export.cfg → stack/resilient/pbsready-export.cfg.in

 [buildout]
-extends = ${template-pbsready:output}
+
+extends = ${pbsready:output}
 
 parts += cron-entry-backup
 
 [urls]
 notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
 
-#notify launches executable, and once it's done, notifies the pull-backup-servers.
 [notifier-exporter]
+# notifier.notify launches an (exporter) executable, and when finished,
+# notifies the the pull-backup-servers.
 <= notifier
 recipe = slapos.cookbook:notifier.notify
 name = exporter
@@ -16,8 +18,9 @@ executable = $${exporter:wrapper}
 wrapper = $${rootdirectory:bin}/exporter
 notify = $${slap-parameter:notify}
 
-#adds the exporter to cron
 [cron-entry-backup]
+# Schedule the periodic database dump.
+# Through notifications, this triggers (one or more) incremental backups on PBS instances.
 <= cron
 recipe = slapos.cookbook:cron.d
 name = backup

stack/resilient/instance-pbsready-import.cfg → stack/resilient/pbsready-import.cfg.in

 [buildout]
 
-extends = ${template-pbsready:output}
+extends = ${pbsready:output}
 
 parts += import-on-notification
 
 [urls]
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 
-#Launches callback, when a notification is received
 [import-on-notification]
+# notifier.callback runs a script when a notification (sent by a parent PBS)
+# is received
 <= notifier
 recipe = slapos.cookbook:notifier.callback
 on-notification-id = $${slap-parameter:on-notification}

stack/resilient/instance-pbsready.cfg → stack/resilient/pbsready.cfg.in

@@ -18,21 +18,97 @@ parts =
   dropbear-server-pbs-authorized-key
   notifier
 
-# adds the resiliency script for the bully algorithm
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[basedirectory]
+services = $${rootdirectory:etc}/run
+cache = $${rootdirectory:var}/cache
+notifier = $${rootdirectory:etc}/notifier
+
+[directory]
+backup = $${basedirectory:backup}/$${slap-parameter:namebase}
+ssh = $${rootdirectory:etc}/ssh/
+sshkeys = $${rootdirectory:srv}/sshkeys
+notifier-feeds = $${basedirectory:notifier}/feeds
+notifier-callbacks = $${basedirectory:notifier}/callbacks
+
+
+#----------------
+#--
+#-- Resiliency script for the bully algorithm
+
 [resiliency]
+# If enable-bully-service is true, the scripts will be run automatically.
+# If false, they can be run with bin/bullly for all the PBSReady instances.
+enable-bully-service = False
 recipe = slapos.cookbook:addresiliency
-script = $${basedirectory:script}
-run = $${basedirectory:services}
+services = $${basedirectory:services}
+bin = $${rootdirectory:bin}
+etc = $${rootdirectory:etc}
+
+
+#----------------
+#--
+#-- Sets up an rdiff-backup server (with a dropbear server for ssh)
 
-# sets up an rdiff-backup server (with a dropbear server for ssh)
 [urls]
 ssh-public-key = $${sshkeys-dropbear:public-key-value}
 ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:backup}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+
+
+#----------------
+#--
+#-- Set up the equeue and notifier.
+
+[equeue]
+recipe = slapos.cookbook:equeue
+socket = $${basedirectory:run}/equeue.sock
+log = $${basedirectory:log}/equeue.log
+database = $${rootdirectory:srv}/equeue.db
+wrapper = $${basedirectory:services}/equeue
+equeue-binary = ${buildout:bin-directory}/equeue
+
+# notifier.notify adds the [exporter, notifier] to the execution queue
+# notifier.notify.callback sets up a callback
+[notifier]
+recipe = slapos.cookbook:notifier
+feeds = $${directory:notifier-feeds}
+callbacks = $${directory:notifier-callbacks}
+id-file = $${rootdirectory:etc}/notifier.id
+equeue-socket = $${equeue:socket}
+host = $${slap-network-information:global-ipv6}
+port = 8080
+wrapper = $${basedirectory:services}/notifier
+server-binary = ${buildout:bin-directory}/pubsubserver
+notifier-binary = ${buildout:bin-directory}/pubsubnotifier
+
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
+
+
+#----------------
+#--
+#-- sshkeys
+
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -51,6 +127,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
+
+#----------------
+#--
+#-- Dropbear.
+
 [dropbear-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
@@ -66,57 +147,5 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:backup}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-
-## Sets up the execution queue for the notifier
-[logrotate-entry-equeue]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = equeue
-log = $${equeue:log}
-frequency = daily
-rotate-num = 30
-
-[equeue]
-recipe = slapos.cookbook:equeue
-socket = $${basedirectory:run}/equeue.sock
-log = $${basedirectory:log}/equeue.log
-database = $${rootdirectory:srv}/equeue.db
-wrapper = $${basedirectory:services}/equeue
-equeue-binary = ${buildout:bin-directory}/equeue
-
-
-## notifier.notify adds the [exporter, notifier] to the execution queue
-## notifier.notify.callback sets up a callback
-[notifier]
-recipe = slapos.cookbook:notifier
-feeds = $${directory:notifier-feeds}
-callbacks = $${directory:notifier-callbacks}
-id-file = $${rootdirectory:etc}/notifier.id
-equeue-socket = $${equeue:socket}
-host = $${slap-network-information:global-ipv6}
-port = 8080
-wrapper = $${basedirectory:services}/notifier
-server-binary = ${buildout:bin-directory}/pubsubserver
-notifier-binary = ${buildout:bin-directory}/pubsubnotifier
-
 
-[basedirectory]
-script = $${rootdirectory:etc}/script/
-services = $${rootdirectory:etc}/run/
-cache = $${rootdirectory:var}/cache/
-notifier = $${rootdirectory:etc}/notifier/
 
-[directory]
-backup = $${basedirectory:backup}/$${slap-parameter:namebase}
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys
-notifier-feeds = $${basedirectory:notifier}/feeds/
-notifier-callbacks = $${basedirectory:notifier}/callbacks/
-script = $${basedirectory:script}

stack/resilient/switchsoftware.cfg

-[buildout]
-parts =
-  switch_softwaretype
-
-[switch_softwaretype]
-recipe = slapos.cookbook:softwaretype
-pull-backup = ${template-pull-backup:output}
-#frozen creates a syntax error, meaning it can keep its data.
-#It's dirty as hell, it needs to be replaced.
-frozen = ${template-frozen:output}

stack/resilient/template-parts.cfg → stack/resilient/template-parts.cfg.in

@@ -18,4 +18,4 @@
   
 
 
-  {% endmacro %}
\ No newline at end of file
+  {% endmacro %}

stack/resilient/template-replicated.cfg → stack/resilient/template-replicated.cfg.in

@@ -16,13 +16,10 @@ name = {{namebase}}0
 return = url ssh-public-key ssh-url notification-id ip
 
 config = number script wrapper authorized-key notify ip-list namebase
-
+config-number = 0
 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
-
 config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
-
 config-ip-list =
-config-number = 0
 
 {% for id in range(1,nbbackup|int) %}
 
@@ -37,14 +34,13 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{typeimport}}
 return = url ssh-public-key ssh-url notification-url ip
 
-config = number script wrapper authorized-key on-notification ip-list namebase
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
 
+config = number script wrapper authorized-key on-notification ip-list namebase
+config-number = {{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
 config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
-pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
-
 config-ip-list =
-config-number = {{id}}
 
 {% endfor %}
 
@@ -60,19 +56,15 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
 recipe = slapos.cookbook:request
 name = {{namebase}}0
 
-config = number script wrapper authorized-key notify ip-list namebase
-
 software-url = ${slap-connection:software-release-url}
 software-type = {{typeexport}}
 return = url ssh-public-key ssh-url notification-id ip
 
-
+config = number script wrapper authorized-key notify ip-list namebase
+config-number = 0
 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
 config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
 
-
-config-number=0
-
 {% for id in range(1,nbbackup|int) %}
 [request-{{namebase}}-pseudo-replicating-{{id}}-2]
 <= slap-connection
@@ -87,14 +79,13 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{typeimport}}
 return = url ssh-public-key ssh-url notification-url
 
-config = number script wrapper authorized-key on-notification ip-list namebase
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
 
+config = number script wrapper authorized-key on-notification ip-list namebase
+config-number = {{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
 config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
-pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
-
 
-config-number = {{id}}
 {% endfor %}
 
 
@@ -135,6 +126,8 @@ config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
 config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
 config-title = Pulling from {{namebase}}
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
 
 [request-pull-backup-server-{{namebase}}-backup-{{id}}]
 <= request-pbs-common
@@ -149,6 +142,8 @@ config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-noti
 config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
 config-title = Pushing to {{namebase}} backup {{id}}
 slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
 {% endfor %}
 
-{% endmacro %}
\ No newline at end of file
+{% endmacro %}