Merge Thimothée's work on resiliency

This is a squash merge of 'resiliency-testing' branch when
the HEAD of this branch was f725c81d7d69.

The merge was done by Antoine Catton.

setup.py

@@ -41,6 +41,7 @@ setup(name=name,
       zip_safe=True,
       entry_points={
         'zc.buildout': [
+          'addresiliency = slapos.recipe.addres:Recipe',
           'agent = slapos.recipe.agent:Recipe',
           'apache.frontend = slapos.recipe.apache_frontend:Recipe',
           'apachephp = slapos.recipe.apachephp:Recipe',
@@ -60,6 +61,7 @@ setup(name=name,
           'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
           'dropbear.client = slapos.recipe.dropbear:Client',
           'dropbear = slapos.recipe.dropbear:Recipe',
+		      'dumpmdb = slapos.recipe.dumpmdb:Recipe',
           'duplicity = slapos.recipe.duplicity:Recipe',
           'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
           'equeue = slapos.recipe.equeue:Recipe',
@@ -83,6 +85,7 @@ setup(name=name,
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
           'helloworld = slapos.recipe.helloworld:Recipe',
+          'importmdb = slapos.recipe.importmdb:Recipe',
           'java = slapos.recipe.java:Recipe',
           'kumofs = slapos.recipe.kumofs:Recipe',
           'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',

slapos/recipe/addres/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+import sys
+import os
+
+
+class Recipe(GenericBaseRecipe):
+  """ This class provides the installation of the resilience
+      script on the partition.
+  """
+
+  def install(self):
+    path_list = []
+    param_dict = self.getComputerPartitionInstanceParameterDict()
+    self_id = int(param_dict['number'])
+    ip = param_dict['ip-list'].split(' ')
+    print 'Creating bully script with  ips : %s\n' % ip
+    slap_connection = self.buildout['slap-connection']
+
+    path_conf = os.path.join(self.options['script'], 'conf.in')
+    path_bully = os.path.join(self.options['script'], param_dict['script'])
+    path_bully_new = os.path.join(self.options['script'], 'new.py')
+    path_run = os.path.join(self.options['run'], param_dict['wrapper'])
+    print 'paths: %s\n%s\n' % (path_run, path_bully)
+    bully_conf = dict(self_id=self_id,
+                      ip_list=ip,
+                      executable=sys.executable,
+                      syspath=sys.path,
+                      server_url=slap_connection['server-url'],
+                      key_file=slap_connection.get('key-file'),
+                      cert_file=slap_connection.get('cert-file'),
+                      computer_id=slap_connection['computer-id'],
+                      partition_id=slap_connection['partition-id'],
+                      software=slap_connection['software-release-url'],
+                      namebase=param_dict['namebase'],
+                      confpath=path_conf)
+    try:
+      conf = self.createFile(path_conf,
+                             self.substituteTemplate(
+                             self.getTemplateFilename('conf.in.in'),
+                             bully_conf))
+      path_list.append(conf)
+      script = self.createExecutable(path_bully,
+                                     self.substituteTemplate(
+                                     self.getTemplateFilename('bully.py.in'),
+                                     bully_conf))
+      path_list.append(script)
+
+      # for testing purposes only
+      scriptNew = self.createExecutable(path_bully_new,
+                                  self.substituteTemplate(
+                                  self.getTemplateFilename('bully_new.py.in'),
+                                  bully_conf))
+      path_list.append(scriptNew)
+
+      wrapper = self.createPythonScript(
+          path_run,
+          'slapos.recipe.librecipe.execute.execute',
+          [path_bully])
+      path_list.append(wrapper)
+    except IOError:
+      pass
+    return path_list

slapos/recipe/addres/template/bully.py.in

+#!%(executable)s
+
+import select
+import socket
+import threading
+import time
+import sys
+
+sys.path[:] = %(syspath)s
+
+from slapos import slap as slapmodule
+
+port = 50000
+size = 1024
+wait = True
+
+
+def loadConnectionInfos():
+  connectionInfos = {}
+  file = open('%(confpath)s', 'r')
+  params = file.read().split('\n')
+  file.close()
+  ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
+  connectionInfos['self_id'] = int(params[1])
+
+  connectionInfos['server_list'] = \
+      [(i, ip_list[i]) for i in range(len(ip_list))]
+
+  connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
+
+  return connectionInfos
+
+
+def rename_broken_and_stop():
+  try:
+    slap = slapmodule.slap()
+    slap.initializeConnection('%(server_url)s',
+                              '%(key_file)s',
+                              '%(cert_file)s')
+    computer_partition = slap.registerComputerPartition('%(computer_id)s',
+                                                        '%(partition_id)s')
+    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
+
+    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+    broken.stopped()
+    computer_partition.rename('%(namebase)s0')
+    print 'renaming done\n'
+  except slapos.slap.slap.ServerError:
+    print 'Internal server error\n'
+
+def election():
+  global wait
+  connection = loadConnectionInfos()
+  message = "%%s, %%s" %% (connection['self_id'], "Election")
+  victory = True
+
+  for (remote_id, addr) in connection['server_list']:
+    if remote_id > connection['self_id']:
+      try:
+        s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        s.connect((addr, port + remote_id))
+        s.send(message)
+        reply = s.recv(size)
+        if reply == "%%s, %%s" %% (remote_id, "Alive"):
+          victory = False
+      except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+        pass
+      finally:
+        s.close()
+
+  if victory:
+    wait = True
+    for (remote_id, addr) in connection['server_list']:
+      if remote_id < connection['self_id']:
+        try:
+          s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+          s.connect((addr, port + remote_id))
+          s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+          pass
+        finally:
+          s.close()
+    rename_broken_and_stop()
+
+
+def failure_detect():
+  global wait
+  connection = loadConnectionInfos()
+  while True:
+    time.sleep(30)
+    if wait:
+      print 'waiting 30 minutes\n'
+      time.sleep(30 * 60)
+      wait = False
+    if not connection['server_list'][0]:
+      continue
+    (remote_id, addr) = connection['server_list'][0]
+    try:
+      s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+      s.connect((addr, port + remote_id))
+      s.close()
+    except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+      s.close()
+      election()
+
+failure_detect_thread = threading.Thread(target=failure_detect)
+failure_detect_thread.start()
+
+connection = loadConnectionInfos()
+s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+s.bind((connection['self_ip'], port + connection['self_id']))
+s.listen(5)
+#election()
+while True:
+  force_election = False
+  client, _ = s.accept()
+  client_message = client.recv(1024)
+  if client_message:
+    client_id, message = client_message.split(', ')
+    client_id = eval(client_id)
+    if message == "Victory":
+      wait = True
+      print "%%s wins" %% client_id
+    elif message == "Election":
+      print "%%s starts an election" %% client_id
+      if client_id < connection['self_id']:
+        client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
+        force_election = True
+  client.close()
+  if force_election:
+    election()

slapos/recipe/addres/template/bully_new.py.in

+#!%(executable)s
+
+import socket
+import time
+import sys
+import thread
+import time
+import os
+
+sys.path[:] = %(syspath)s
+
+from slapos import slap as slapmodule
+
+port = 50000
+size = 1024
+
+
+def rename_broken_and_stop():
+  try:
+    slap = slapmodule.slap()
+    slap.initializeConnection('%(server_url)s',
+                              '%(key_file)s',
+                              '%(cert_file)s')
+    computer_partition = slap.registerComputerPartition('%(computer_id)s',
+                                                        '%(partition_id)s')
+    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
+
+    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+    broken.stopped()
+    computer_partition.rename('%(namebase)s0')
+    print 'renaming done\n'
+  except slapos.slap.slap.ServerError:
+    print 'Internal server error\n'
+
+
+## Leader is always number 0
+
+class ResilientInstance(object):
+
+    def __init__(self, comm):
+        self.comm = comm
+        self.id = 0
+        self.state = 'normal'
+        self.halter = 0
+        self.nbComp = nbComp
+        self.inElection = False
+        self.alive = True
+        self.lastPing = time.clock()
+
+        self.mainCanal = self.comm.canal(['ping', 'halt',
+                                          'victory'])
+
+        self.okCanal = self.comm.canal(['ok'])
+        self.loadConnectionInfos()
+
+
+    def loadConnectionInfos(self):
+        file = open('%(confpath)s', 'r')
+        params = file.read().split('\n')
+        file.close()
+        self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
+        new_id = int(params[1])
+        if self.id != new_id:
+            self.halter = new_id
+            self.id = new_id 
+
+    ## Needs to be changed to use the master
+    def aliveManagement(self):
+        while self.alive:
+            time.sleep(30*60)
+            if self.id == 0:
+                continue
+            self.comm.send('ping', 0)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            self.election()
+
+    def listen(self):
+        while self.alive:
+            self.comm.recv()
+
+    def main(self):
+        while self.alive:
+            message, sender = self.mainCanal.get()
+            if message == 'ping':
+                    self.comm.send('ok', sender)
+
+            elif message == 'halt':
+                self.state = 'waitingConfirm'
+                self.halter = sender
+                self.comm.send('ok', sender)
+
+            elif message == 'victory':
+                if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
+                    print '%s thinks %s is the leader\n' % (self.id, sender)
+                    self.comm.send('ok', sender)
+                self.state = 'normal'
+
+    def election(self):
+        self.inElection = True
+        self.loadConnectionInfos()
+        #Check if I'm the highest instance alive
+        for higher in range(self.id + 1, self.nbComp):
+            self.comm.send('ping', higher)
+            message, sender = self.okCanal.get()
+            if message:
+                #print '%s is alive (%s)\n' % (higher, self.id)
+                self.inElection = False
+                return False
+            continue
+
+        if not self.alive:
+            return False
+
+        #I should be the new coordinator, halt those below me
+        print 'Should be ME : %s \n' % self.id
+        self.state = 'election'
+        self.halter = self.id
+        ups = []
+        for lower in range(self.id):
+            self.comm.send('halt', lower)
+            message, sender = self.okCanal.get()
+            if message:
+                ups.append(lower)
+
+        #Broadcast Victory
+        self.state = 'reorganization'
+        for up in ups:
+            self.comm.send('victory', up)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            print 'Something is wrong... let\'s start over\n'
+            return self.election()
+        self.state = 'normal'
+        self.active = True
+        print '%s Is THE LEADER \n' % self.id
+
+        rename_broken_and_stop()
+
+        self.inElection = False
+
+        return True
+
+
+class FilteredCanal(object):
+
+    def __init__(self, accept, timeout):
+        self.accept = accept
+        self.list = []
+        self.lock = thread.allocate_lock()
+        self.timeout = timeout
+
+    def append(self, message, sender):
+        if message in self.accept:
+            self.lock.acquire()
+            self.list.append([message, sender])
+            self.lock.release()
+
+    def get(self):
+        start = time.clock()
+        while (time.clock() - start < self.timeout):
+            self.lock.acquire()
+            if self.list:
+                self.lock.release()
+                val = self.list[0]
+                self.list = self.list[1:]
+                return val
+            self.lock.release()
+        return [None, None]
+
+
+class Wrapper(object):
+
+    def __init__(self, timeout=20):
+        self.read_pipes = [os.fdopen(x) for x in read_pipes]
+        self.write_pipes = write_pipes
+        self.canals = []
+        self.ips = []
+        self.id = 0
+        self.timeout = timeout
+        self.getConnectionInfos()
+        self.socket = None
+
+    def getConnectionInfos(self):
+        file = open('%(confpath)s', 'r')
+        params = file.read().split('\n')
+        file.close()
+        self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
+        self.id = int(params[1])
+
+    def start(self):
+        self.getConnectionInfos()
+        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        self.socket.bind((self.ips[self.id], port + self.id))
+        s.listen(5)
+
+    def send(self, message, number):
+        self.getConnectionInfos()
+        try:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            s.connect((self.ips[number], port + number))
+            s.send(message + (' %s\n' % self.id))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+          pass
+        finally:
+            s.close()
+
+    def canal(self, accept):
+        created = FilteredCanal(accept, self.timeout)
+        self.canals.append(created)
+        return created
+
+    def recv(self):
+        client, _ = s.accept()
+        client_message = client.recv(1024)
+        if client_message:
+            message, sender = client_message.split()
+            for canal in self.canals:
+                canal.append(message, sender)
+
+
+wrapper = createWrapper(20)
+
+computer = ResilientInstance(wrapper)
+
+#idle waiting for connection infos
+while computer.nbComp < 2 :
+    computer.loadConnectionInfos()
+    time.sleep(30)
+
+print 'Starting\n'
+
+computer.comm.start()
+thread.start_new_thread(computer.listen, ())
+thread.start_new_thread(computer.main, ())
+thread.start_new_thread(computer.aliveManagement, ())
+
+while True:
+    continue
+
+

slapos/recipe/addres/template/conf.in.in

+%(ip_list)s
+%(self_id)s

slapos/recipe/apacheproxy/template/apache.in

@@ -3,7 +3,6 @@
 
 # Basic server configuration
 PidFile "%(pid_file)s"
-LockFile "%(lock_file)s"
 Listen %(ip)s:%(port)s
 ServerAdmin someone@email
 DefaultType text/plain
@@ -22,13 +21,15 @@ CustomLog "%(access_log)s" common
 <Directory />
     Options FollowSymLinks
     AllowOverride None
-    Order deny,allow
-    Deny from all
+    Require all denied
 </Directory>
 
 ProxyPass / %(backend_url)s
 
 # List of modules
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule authz_host_module modules/mod_authz_host.so
 LoadModule log_config_module modules/mod_log_config.so
 LoadModule setenvif_module modules/mod_setenvif.so

slapos/recipe/dropbear.py

@@ -167,6 +167,7 @@ class AddAuthorizedKey(GenericBaseRecipe):
     path_list.append(ssh)
 
     authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
-    authorized_keys.append(self.options['key'])
+    for key in self.options['key'].split(' '):
+      authorized_keys.append(key)
 
     return path_list

slapos/recipe/librecipe/generic.py

@@ -31,6 +31,8 @@ import inspect
 import re
 import urllib
 import urlparse
+from slapos import slap as slapmodule
+from json import loads as unjson
 
 import pkg_resources
 import zc.buildout
@@ -90,7 +92,7 @@ class GenericBaseRecipe(object):
 
   def createPythonScript(self, name, absolute_function, arguments=''):
     """Create a python script using zc.buildout.easy_install.scripts
-
+ok o
      * function should look like 'module.function', or only 'function'
        if it is a builtin function."""
     absolute_function = tuple(absolute_function.rsplit('.', 1))
@@ -128,6 +130,17 @@ class GenericBaseRecipe(object):
     return pkg_resources.resource_filename(name,
         'template/%s' % template_name)
 
+  def getComputerPartitionInstanceParameterDict(self):
+    slap_connection = self.buildout['slap-connection']
+    slap = slapmodule.slap()
+    slap.initializeConnection(slap_connection['server-url'],
+							  slap_connection.get('key-file'),
+							  slap_connection.get('cert-file'))
+
+    computer_partition = slap.registerComputerPartition(slap_connection['computer-id'],
+                                                        slap_connection['partition-id'])
+    return computer_partition.getInstanceParameterDict()
+
   def generatePassword(self, len_=32):
     """
     The purpose of this method is to generate a password which doesn't change

slapos/recipe/mysql/__init__.py

@@ -31,6 +31,7 @@ import subprocess
 from slapos.recipe.librecipe import GenericBaseRecipe
 from slapos.recipe.librecipe import filehash
 
+
 class Recipe(GenericBaseRecipe):
 
   def _options(self, options):

slapos/recipe/pbs.py

@@ -213,6 +213,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
         for slave in slaves:
           path_list.extend(self.add_slave(slave, known_hosts))
 
+
     else:
       command = [self.options['rdiffbackup-binary']]
       self.logger.info("Server mode")

slapos/recipe/request.py

@@ -28,6 +28,7 @@ import logging
 
 from slapos import slap as slapmodule
 
+
 class Recipe(object):
   """
   Request a partition to a slap master.
@@ -98,8 +99,9 @@ class Recipe(object):
       options['computer-id'], options['partition-id']).request
 
     isSlave = options.get('slave', '').lower() in ['y', 'yes', 'true', '1']
-
+    print '\n Slave : %s \n' % self.isSlave
     return_parameters = []
+    self.return_parameters = []
     if 'return' in options:
       return_parameters = [str(parameter).strip()
         for parameter in options['return'].split()]
@@ -119,7 +121,10 @@ class Recipe(object):
       for config_parameter in options['config'].split():
         partition_parameter_kw[config_parameter] = \
             options['config-%s' % config_parameter]
-
+    print 'from : %s \n' % options['partition-id']
+    print 'requested %s (%s), parameters : %s \n\n' % (options['name'],
+                                                       software_type,
+                                                       partition_parameter_kw)
     self.instance = instance = request(software_url, software_type,
       name, partition_parameter_kw=partition_parameter_kw,
       filter_kw=filter_kw, shared=isSlave)
@@ -129,9 +134,14 @@ class Recipe(object):
         options['connection-%s' % param] = str(
           instance.getConnectionParameter(param))
       except slapmodule.NotFoundError:
-        options['connection-%s' % param] = ''
-        if self.failed is None:
-          self.failed = param
+        try:
+          self.instance._synced = False
+          options['connection-%s' % param] = str(
+              self.instance.getConnectionParameter(param))
+        except slapmodule.NotFoundError:
+          options['connection-%s' % param] = ''
+          if self.failed is None:
+            self.failed = param
 
   def install(self):
     if self.failed is not None:

stack/lamp/apache/instance-apache-import.cfg

+[buildout]
+extends = ${template-apache-php:output}
+          ${template-pbsready-export:output}
+
+parts =
+  apache-proxy
+  logrotate
+  logrotate-entry-apache
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  sshkeys-dropbear
+  dropbear-server
+  dropbear-server-pbs-authorized-key
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:sshkeys}/requests/
+keys = $${directory:sshkeys}/keys/
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = $${dropbear-server:wrapper}
+public-key = $${dropbear-server:rsa-keyfile}.pub
+private-key = $${dropbear-server:rsa-keyfile}
+wrapper = $${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = $${slap-network-information:global-ipv6}
+port = 2222
+home = $${directory:ssh}
+wrapper = $${rootdirectory:bin}/raw_sshd
+shell = $${rdiff-backup-server:wrapper}
+rsa-keyfile = $${directory:ssh}/server_key.rsa
+dropbear-binary = ${dropbear:location}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = $${slap-parameter:authorized-key}
+
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:htdocs}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-proxy:error-log} $${apache-proxy:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+services = $${rootdirectory:etc}/run/
+run = $${rootdirectory:var}/run/
+backup = $${rootdirectory:srv}/backup/
+promises = $${rootdirectory:etc}/promise/
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www/
+logrotate-entries = $${rootdirectory:etc}/logrotate.d/
+logrotate-backup = $${basedirectory:backup}/logrotate/
+cronstamps = $${rootdirectory:etc}/cronstamps/
+cron-entries = $${rootdirectory:etc}/cron.d/
+crontabs = $${rootdirectory:etc}/crontabs/
+ssh = $${rootdirectory:etc}/ssh/
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache/

stack/lamp/buildout.cfg

 [buildout]
+
+ignore-existing = true
+
+develop =
+  /opt/slapdev
+
 parts =
   apache-php
   mariadb
@@ -9,13 +15,16 @@ parts =
   eggs
   instance-recipe-egg
   template
+
   template-apache-php
   template-mariadb
-  template-mariadb-pbsready
-  template-mariadb-pbsready-import
-  template-mariadb-pbsready-export
+
+#Contains the importer and exporter recipes for mariadb
+  template-mariadb-import
+  template-mariadb-export
 
 extends =
+  ../resilient/buildout.cfg
   ../../component/mariadb/buildout.cfg
   ../../component/apache/buildout.cfg
   ../../component/apache-php/buildout.cfg
@@ -59,64 +68,60 @@ strip-top-level-dir = true
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 8117f10e814a13c5376af4c01e6546d4
+md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11
 mode = 0644
 
 [template-apache-php]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-php.cfg
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg
 output = ${buildout:directory}/template-apache-php.cfg
 md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966
 mode = 0644
 
 [template-apache-backup]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-backup.cfg
+url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg
 output = ${buildout:directory}/template-apache-backup.cfg
 md5sum = cfb77ac8785e0d125a785f69a5339014
 mode = 0644
 
-[template-backuped]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-backuped.cfg
-output = ${buildout:directory}/template-backuped.cfg
-md5sum = f43d1c6412ea8dc83b75573dc00daf9e
+[template-resilient-lamp]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-resilient.cfg
+rendered = ${buildout:directory}/template-resilient.cfg
+
+context = key templateapache template-apache-php:output
+          key dropbear dropbear:location
+          key buildout buildout:bin-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination 
+
+md5sum = 2488daff81286a1b6be6dbc1ad3b0d7d
 mode = 0644
+ 
 
 [template-mariadb]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb.cfg
+url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg
 output = ${buildout:directory}/template-mariadb.cfg
 md5sum = 767452bc503ff6c1b7af0ebfac590c9f
 mode = 0644
 
-[template-mariadb-pbsready]
+[template-mariadb-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg
-output = ${buildout:directory}/template-mariadb-pbsready.cfg
-md5sum = d2a580dcd7efdd528be45c5ffadfe760
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg
+output = ${buildout:directory}/template-mariadb-import.cfg
+md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b
 mode = 0644
 
-[template-mariadb-pbsready-import]
+[template-mariadb-export]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg
-output = ${buildout:directory}/template-mariadb-pbsready-import.cfg
-md5sum = 4a96ff02da3898fef7077fa8baec81ac
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg
+output = ${buildout:directory}/template-mariadb-export.cfg
+md5sum = 4b7dec765265b27c8235419b82ca7b02
 mode = 0644
 
-[template-mariadb-pbsready-export]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg
-output = ${buildout:directory}/template-mariadb-pbsready-export.cfg
-md5sum = 11a9e45e8bc590bc11bfdd304b07a4a5
-mode = 0644
-
-[template-pull-backup]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pull-backup.cfg
-output = ${buildout:directory}/template-pull-backup.cfg
-md5sum = 9aab30ba5aa23a37d4b507e7c414be00
-mode = 0644
 
 # Dummy parts in case no application configuration file is needed
 [application-template]

stack/lamp/instance-backuped.cfg

-[buildout]
-
-extends =
-  ${template-apache-php:output}
-
-parts =
-  request-mariadb
-  request-mariadb-pseudo-replicating
-  request-apache-backup
-  request-pull-backup-server
-  request-pull-backup-server-mariadb
-  request-pull-backup-server-mariadb-backup
-  request-pull-backup-server-apache
-  request-pull-backup-server-apache-backup
-  url
-  apache-php
-  stunnel
-  certificate-authority
-  ca-stunnel
-  logrotate
-  logrotate-entry-apache
-  logrotate-entry-stunnel
-  cron
-  cron-entry-logrotate
-  dropbear-server
-  sshkeys-authority
-  dropbear-server-pbs-authorized-key
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-wrapper = $${basedirectory:services}/sshkeys_authority
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${basedirectory:services}/sshd
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-port = 2222
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${request-pull-backup-server:connection-ssh-key}
-
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:htdocs}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-[request-pull-backup-server]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = PBS (Pull Backup Server)
-software-url = $${slap-connection:software-release-url}
-software-type = pull-backup
-return = ssh-key notification-url feeds-url
-slave = false
-
-[request-mariadb]
-software-type = mariadb-pbsready-export
-config = authorized-key notify
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-return = url ssh-public-key ssh-url notification-id
-
-[request-mariadb-pseudo-replicating]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = MariaDB Backup
-software-url = $${slap-connection:software-release-url}
-software-type = mariadb-pbsready-import
-return = url ssh-public-key ssh-url notification-url
-config = authorized-key on-notification
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-pbs-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-push
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${:pbs-notification-id}
-
-[request-apache-backup]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup
-software-url = $${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = $${url:url}
-
-[request-pull-backup-server-apache]
-<= request-pull-backup-server
-   slap-connection
-recipe = slapos.cookbook:request
-name = PBS pulling from Apache
-software-url = $${slap-connection:software-release-url}
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
-config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = $${sshkeys-dropbear:public-key-value}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-
-[request-pull-backup-server-apache-backup]
-<= request-pull-backup-server
-   slap-connection
-recipe = slapos.cookbook:request
-name = PBS pushing to $${request-apache-backup:name}
-software-url = $${slap-connection:software-release-url}
-config = url name type server-key on-notification
-config-url = $${request-apache-backup:connection-ssh-url}
-config-name = $${request-pull-backup-server-apache:config-name}
-config-type = push
-config-server-key = $${request-apache-backup:connection-ssh-public-key}
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-apache:config-notification-id}
-slave = true
-
-[request-pull-backup-server-mariadb]
-<= request-pull-backup-server
-   slap-connection
-name = PBS pulling from $${request-mariadb:name}
-config = url name type server-key on-notification notify notification-id title
-config-url = $${request-mariadb:connection-ssh-url}
-config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
-config-type = pull
-config-server-key = $${request-mariadb:connection-ssh-public-key}
-config-on-notification = $${request-mariadb:connection-notification-id}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-pull
-config-title = Pulling from MariaDB
-slave = true
-
-[request-pull-backup-server-mariadb-backup]
-<= request-pull-backup-server
-   slap-connection
-name = PBS pushing on $${request-mariadb-pseudo-replicating:name}
-config = url name type server-key on-notification notify notification-id title
-config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url}
-config-name = $${request-pull-backup-server-mariadb:config-name}
-config-type = push
-config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key}
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-mariadb:config-notification-id}
-config-notify = $${request-mariadb-pseudo-replicating:connection-notification-url}
-config-notification-id = $${request-mariadb-pseudo-replicating:pbs-notification-id}
-config-title = Pushing to MariaDB backup
-slave = true
-
-[directory]
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys

stack/lamp/instance-mariadb-pbsready-export.cfg

-[buildout]
-extends = ${template-mariadb-pbsready:output}
-
-parts += cron-entry-mariadb-backup
-
-[urls]
-notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-mydumper:name}
-
-[mydumper]
-recipe = slapos.cookbook:mydumper
-wrapper = $${rootdirectory:bin}/raw_mydumper
-backup-directory = $${directory:mariadb-backup}
-socket = $${mariadb:socket}
-user = root
-mydumper-binary = ${mydumper:location}/bin/mydumper
-database = $${mariadb:database}
-import = false
-
-[notifier-mydumper]
-<= notifier
-recipe = slapos.cookbook:notifier.notify
-name = mydumper
-title = Dumping MariaDB Database
-executable = $${mydumper:wrapper}
-wrapper = $${rootdirectory:bin}/mydumper
-notify = $${slap-parameter:notify}
-
-[cron-entry-mariadb-backup]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = backup
-frequency = 0 * * * *
-command = $${notifier-mydumper:wrapper}

stack/lamp/instance-resilient.cfg

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+
+extends =
+#  ${template-resilient:output}
+   {{templateapache}}
+
+parts =
+  {{ parts.replicate("mariadb","3") }}
+  request-apache-backup-1
+  request-apache-backup-2
+  
+  request-pull-backup-server-apache-1
+  request-pull-backup-server-apache-backup-1
+
+  request-pull-backup-server-apache-2
+  request-pull-backup-server-apache-backup-2
+
+  publish-connection-informations
+  apache-php
+  stunnel
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  dropbear-server
+  sshkeys-authority
+  dropbear-server-pbs-authorized-key
+
+  request-pull-backup-server
+  
+{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
+
+
+[request-pull-backup-server]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = PBS (Pull Backup Server)
+software-url = ${slap-connection:software-release-url}
+software-type = pull-backup
+return = ssh-key notification-url feeds-url
+slave = false
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = ${directory:sshkeys}/requests/
+keys = ${directory:sshkeys}/keys/
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = ${sshkeys-directory:requests}
+keys-directory = ${sshkeys-directory:keys}
+wrapper = ${basedirectory:services}/sshkeys_authority
+keygen-binary = {{dropbear}}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = ${dropbear-server:wrapper}
+public-key = ${dropbear-server:rsa-keyfile}.pub
+private-key = ${dropbear-server:rsa-keyfile}
+wrapper = ${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = ${slap-network-information:global-ipv6}
+port = 2222
+home = ${directory:ssh}
+wrapper = ${rootdirectory:bin}/raw_sshd
+shell = ${rdiff-backup-server:wrapper}
+rsa-keyfile = ${directory:ssh}/server_key.rsa
+dropbear-binary = {{dropbear}}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = ${request-pull-backup-server:connection-ssh-key}
+
+[rdiff-backup-server]
+<= apache-php
+recipe = slapos.cookbook:pbs
+client = false
+path = ${apache-php:htdocs}
+wrapper = ${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = {{buildout}}/rdiff-backup
+
+[request-apache-backup-1]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 1
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-apache-backup-2]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 2
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-pull-backup-server-apache-1]
+<= request-pull-backup-server
+   slap-connection
+recipe = slapos.cookbook:request
+name = PBS pulling from Apache 1
+software-url = ${slap-connection:software-release-url}
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+
+[request-pull-backup-server-apache-2]
+<= request-pull-backup-server
+   slap-connection
+recipe = slapos.cookbook:request
+name = PBS pulling from Apache 2
+software-url = ${slap-connection:software-release-url}
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+
+
+[request-pull-backup-server-apache-backup-1]
+<= request-pull-backup-server
+   slap-connection
+recipe = slapos.cookbook:request
+name = PBS pushing to ${request-apache-backup-1:name}
+software-url = ${slap-connection:software-release-url}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-1:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-1:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
+slave = true
+
+[request-pull-backup-server-apache-backup-2]
+<= request-pull-backup-server
+   slap-connection
+recipe = slapos.cookbook:request
+name = PBS pushing to ${request-apache-backup-2:name}
+software-url = ${slap-connection:software-release-url}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-2:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-2:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
+slave = true
+
+
+[directory]
+ssh = ${rootdirectory:etc}/ssh/
+sshkeys = ${rootdirectory:srv}/sshkeys

stack/lamp/instance.cfg

 [buildout]
+
+extends = 
+  ${template-switchsoftware:output}
+
 parts =
   switch_softwaretype
 
@@ -9,9 +13,11 @@ offline = true
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = ${template-apache-php:output}
-backuped = ${template-backuped:output}
+resilient = ${template-resilient-lamp:rendered}
 mariadb = ${template-mariadb:output}
-mariadb-pbsready-import = ${template-mariadb-pbsready-import:output}
-mariadb-pbsready-export = ${template-mariadb-pbsready-export:output}
+mariadb-import = ${template-mariadb-import:output}
+mariadb-export = ${template-mariadb-export:output}
 pull-backup = ${template-pull-backup:output}
 apache-backup = ${template-apache-backup:output}
+frozen = ${template-frozen:output}
+

stack/lamp/mariadb/instance-mariadb-export.cfg

+[buildout]
+extends = ${template-mariadb:output}
+          ${template-pbsready-export:output}
+
+parts += mariadb
+
+[exporter]
+recipe = slapos.cookbook:mydumper
+wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter
+backup-directory = $${directory:backup}
+socket = $${mariadb:socket}
+user = root
+mydumper-binary = ${mydumper:location}/bin/mydumper
+database = $${mariadb:database}
+import = false
\ No newline at end of file

stack/lamp/instance-mariadb-pbsready-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg

 [buildout]
+extends = ${template-mariadb:output}
+          ${template-pbsready-import:output}
 
-extends = ${template-mariadb-pbsready:output}
+parts += mariadb
 
-parts += mariadb-import-on-notification
-
-[urls]
-notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
-
-[mydumper-import]
+[importer]
 recipe = slapos.cookbook:mydumper
 wrapper = $${rootdirectory:bin}/myloader
-backup-directory = $${directory:mariadb-backup}
+backup-directory = $${directory:backup}
 socket = $${mariadb:socket}
 user = root
 myloader-binary = ${mydumper:location}/bin/myloader
 database = $${mariadb:database}
 import = true
-
-[mariadb-import-on-notification]
-<= notifier
-recipe = slapos.cookbook:notifier.callback
-on-notification-id = $${slap-parameter:on-notification}
-callback = $${mydumper-import:wrapper}

stack/lamp/instance-mariadb.cfg → stack/lamp/mariadb/instance-mariadb.cfg

@@ -23,6 +23,7 @@ offline = true
 [urls]
 recipe = slapos.cookbook:publish
 url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
+ip = $${slap-network-information:global-ipv6}
 
 [mariadb]
 recipe = slapos.cookbook:mysql
@@ -52,6 +53,7 @@ mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
 mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
 mysqld-binary = ${mariadb:location}/bin/mysqld
 
+
 [slapmonitor]
 recipe = slapos.cookbook:slapmonitor
 pid-file = $${basedirectory:run}/mariadb.pid
@@ -195,6 +197,7 @@ recipe = slapos.cookbook:mkdirectory
 log = $${rootdirectory:var}/log/
 services = $${rootdirectory:etc}/run/
 run = $${rootdirectory:var}/run/
+script = $${rootdirectory:etc}/script/
 backup = $${rootdirectory:srv}/backup/
 promises = $${rootdirectory:etc}/promise/
 

stack/resilient/buildout.cfg

+[buildout]
+
+parts =
+#Templates needed to setup automatic backup
+  template-pbsready
+  template-pbsready-import
+  template-pbsready-export
+  template-replicated
+  template-parts
+
+#Frozen is the state used to not destroy a broken instance's content
+  template-frozen
+
+#
+  template-resilient
+  template-switchsoftware
+
+
+[template-pbsready]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready.cfg
+output = ${buildout:directory}/template-pbsready.cfg
+md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99
+mode = 0644
+
+[template-pbsready-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready-import.cfg
+output = ${buildout:directory}/template-pbsready-import.cfg
+md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4
+mode = 0644
+
+[template-pbsready-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready-export.cfg
+output = ${buildout:directory}/template-pbsready-export.cfg
+md5sum = 29d36aac2008b173cb9ce5da9e88c0fa
+mode = 0644
+
+[template-pull-backup]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pull-backup.cfg
+output = ${buildout:directory}/template-pull-backup.cfg
+md5sum = 964ea5cb91af333254e565c1ec0f312f
+mode = 0644
+
+[template-replicated]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/template-replicated.cfg
+md5sum = 0f14615814f4abe37cc580bb0e51b276
+mode = 0644
+destination = ${buildout:directory}/template-replicated.cfg
+
+[template-parts]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/template-parts.cfg
+md5sum = f5fc27235725f05fdbde76a78ebc363e
+mode = 0644
+destination = ${buildout:directory}/template-parts.cfg
+
+[template-frozen]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-frozen.cfg
+output = ${buildout:directory}/template-frozen.cfg
+
+[template-resilient]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/resilient.cfg
+output = ${buildout:directory}/resilient.cfg
+md5sum = 59e74d290d623de2c1e147e48f284fba
+mode = 0644
+
+[template-switchsoftware]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/switchsoftware.cfg
+output = ${buildout:directory}/switchsoftware.cfg
+md5sum = c94a0ed85fce2e72254ae956dce7e40d
+mode = 0644
\ No newline at end of file

stack/resilient/instance-frozen.cfg

+[buildout]
+  parts = 
\ No newline at end of file

stack/resilient/instance-pbsready-export.cfg

+[buildout]
+extends = ${template-pbsready:output}
+
+parts += cron-entry-backup
+
+[urls]
+notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
+
+#notify launches executable, and once it's done, notifies the pull-backup-servers.
+[notifier-exporter]
+<= notifier
+recipe = slapos.cookbook:notifier.notify
+name = exporter
+title = Dumping $${slap-parameter:namebase}
+executable = $${exporter:wrapper}
+wrapper = $${rootdirectory:bin}/exporter
+notify = $${slap-parameter:notify}
+
+#adds the exporter to cron
+[cron-entry-backup]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = backup
+frequency = 0 * * * *
+command = $${notifier-exporter:wrapper}

stack/resilient/instance-pbsready-import.cfg

+[buildout]
+
+extends = ${template-pbsready:output}
+
+parts += import-on-notification
+
+[urls]
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+
+#Launches callback, when a notification is received
+[import-on-notification]
+<= notifier
+recipe = slapos.cookbook:notifier.callback
+on-notification-id = $${slap-parameter:on-notification}
+callback = $${importer:wrapper}

stack/lamp/instance-mariadb-pbsready.cfg → stack/resilient/instance-pbsready.cfg

 [buildout]
 
-extends = ${template-mariadb:output}
-
 parts =
+  resiliency
   urls
-  mariadb
   stunnel
   certificate-authority
   ca-stunnel
   logrotate
-  logrotate-entry-mariadb
   logrotate-entry-stunnel
   logrotate-entry-cron
   logrotate-entry-equeue
@@ -21,6 +18,13 @@ parts =
   dropbear-server-pbs-authorized-key
   notifier
 
+# adds the resiliency script for the bully algorithm
+[resiliency]
+recipe = slapos.cookbook:addresiliency
+script = $${basedirectory:script}
+run = $${basedirectory:services}
+
+# sets up an rdiff-backup server (with a dropbear server for ssh)
 [urls]
 ssh-public-key = $${sshkeys-dropbear:public-key-value}
 ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
@@ -65,10 +69,12 @@ key = $${slap-parameter:authorized-key}
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
 client = false
-path = $${directory:mariadb-backup}
+path = $${directory:backup}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
+
+## Sets up the execution queue for the notifier
 [logrotate-entry-equeue]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
@@ -85,6 +91,9 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
+
+## notifier.notify adds the [exporter, notifier] to the execution queue
+## notifier.notify.callback sets up a callback
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -99,12 +108,15 @@ notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
 
 [basedirectory]
+script = $${rootdirectory:etc}/script/
+services = $${rootdirectory:etc}/run/
 cache = $${rootdirectory:var}/cache/
 notifier = $${rootdirectory:etc}/notifier/
 
 [directory]
-mariadb-backup = $${basedirectory:backup}/mariadb/
+backup = $${basedirectory:backup}/$${slap-parameter:namebase}
 ssh = $${rootdirectory:etc}/ssh/
 sshkeys = $${rootdirectory:srv}/sshkeys
 notifier-feeds = $${basedirectory:notifier}/feeds/
 notifier-callbacks = $${basedirectory:notifier}/callbacks/
+script = $${basedirectory:script}

stack/lamp/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg

@@ -19,6 +19,8 @@ ssh-key = $${sshkeys-dropbear:public-key-value}
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
 
+
+## sets up the equeue for the notifier
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
@@ -27,6 +29,7 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
+
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -39,6 +42,8 @@ wrapper = $${basedirectory:services}/notifier
 server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
+
+## Dropbear Client to provide ssh 
 [dropbear-client]
 recipe = slapos.cookbook:dropbear.client
 dbclient-binary = ${dropbear:location}/bin/dbclient
@@ -68,6 +73,9 @@ public-key = $${dropbear-client:identity-file}.pub
 private-key = $${dropbear-client:identity-file}
 wrapper = $${rootdirectory:bin}/do_backup
 
+
+## The pull-backup-server contains every backup (incremental).
+## to prevent a corrupt dump from destroying everything.
 [pbs]
 <= notifier
 recipe = slapos.cookbook:pbs
@@ -82,6 +90,7 @@ wrappers-directory = $${directory:pbs-wrappers}
 notifier-url = http://[$${notifier:host}]:$${notifier:port}/
 slave-instance-list = $${slap-parameter:slave_instance_list}
 
+
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
 wrapper = $${rootdirectory:bin}/cron_simplelogger

stack/resilient/resilient.cfg

+## not used at the moment
+
+[buildout]
+
+parts =
+  request-pull-backup-server
+
+[request-pull-backup-server]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = PBS (Pull Backup Server)
+software-url = $${slap-connection:software-release-url}
+software-type = pull-backup
+return = ssh-key notification-url feeds-url
+slave = false
\ No newline at end of file

stack/resilient/switchsoftware.cfg

+[buildout]
+parts =
+  switch_softwaretype
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+pull-backup = ${template-pull-backup:output}
+#frozen creates a syntax error, meaning it can keep its data.
+#It's dirty as hell, it needs to be replaced.
+frozen = ${template-frozen:output}

stack/resilient/template-parts.cfg

+## Parts Needed for the resiliency
+
+  {% macro replicate(namebase, nbbackup) %}
+
+  request-{{namebase}}
+  request-{{namebase}}-2
+  
+  {% for i in range(1,nbbackup|int) %}
+  request-{{namebase}}-pseudo-replicating-{{i}}
+  request-{{namebase}}-pseudo-replicating-{{i}}-2
+  {% endfor %}
+  
+  {% for i in range(1,nbbackup|int) %}
+  request-pbs-{{namebase}}-{{i}}
+  request-pull-backup-server-{{namebase}}-{{i}}
+  request-pull-backup-server-{{namebase}}-backup-{{i}}
+  {% endfor %}
+  
+
+
+  {% endmacro %}
\ No newline at end of file

stack/resilient/template-replicated.cfg

+{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='') %}
+
+## Tells the Backupable recipe that we want a backup
+[resilient]
+config-script = bully.py
+config-wrapper = bully
+config-namebase = {{namebase}}
+
+## Every request is double to provide the 3 IPs.
+[request-{{namebase}}]
+<= resilient
+   slap-connection
+   {{heriteLeader}}
+software-type = {{typeexport}}
+name = {{namebase}}0
+return = url ssh-public-key ssh-url notification-id ip
+
+config = number script wrapper authorized-key notify ip-list namebase
+
+config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
+
+config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
+
+config-ip-list =
+config-number = 0
+
+{% for id in range(1,nbbackup|int) %}
+
+[request-{{namebase}}-pseudo-replicating-{{id}}]
+<= slap-connection
+   resilient
+   {{heriteBackup}}
+recipe = slapos.cookbook:request
+name = {{namebase}}{{id}}
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeimport}}
+return = url ssh-public-key ssh-url notification-url ip
+
+config = number script wrapper authorized-key on-notification ip-list namebase
+
+config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+
+config-ip-list =
+config-number = {{id}}
+
+{% endfor %}
+
+[iplist]
+config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
+
+[request-{{namebase}}-2]
+<= resilient
+   slap-connection
+   iplist
+   {{heriteLeader}}
+
+recipe = slapos.cookbook:request
+name = {{namebase}}0
+
+config = number script wrapper authorized-key notify ip-list namebase
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeexport}}
+return = url ssh-public-key ssh-url notification-id ip
+
+
+config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
+config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
+
+
+config-number=0
+
+{% for id in range(1,nbbackup|int) %}
+[request-{{namebase}}-pseudo-replicating-{{id}}-2]
+<= slap-connection
+   resilient
+   iplist
+   {{heriteBackup}}
+
+recipe = slapos.cookbook:request
+name = {{namebase}}{{id}}
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeimport}}
+return = url ssh-public-key ssh-url notification-url
+
+config = number script wrapper authorized-key on-notification ip-list namebase
+
+config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+
+
+config-number = {{id}}
+{% endfor %}
+
+
+
+## The PBS and their push / pull slaves
+## Adding a PBS provides resiliency
+## Adding a backup server provides availability
+
+## Having 3 backups pulling from the same PBS provides 
+##only availability, not resiliency
+
+## WARNING : SLAVES ARE ALLOCATED AT RANDOM, THIS NEEDS TO BE FIXED.
+
+{% for id in range(1,nbbackup|int) %}
+
+[request-pbs-{{namebase}}-{{id}}]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = PBS ({{namebase}} / {{id}})
+software-url = ${slap-connection:software-release-url}
+software-type = pull-backup
+return = ssh-key notification-url feeds-url
+slave = false
+
+[request-pull-backup-server-{{namebase}}-{{id}}]
+<= request-pbs-{{namebase}}-{{id}}
+   slap-connection
+name = PBS {{id}} pulling from ${request-{{namebase}}:name}
+config = url name type server-key on-notification notify notification-id title
+config-url = ${request-{{namebase}}:connection-ssh-url}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
+config-type = pull
+config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
+config-on-notification = ${request-{{namebase}}:connection-notification-id}
+config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
+config-title = Pulling from {{namebase}}
+slave = true
+
+[request-pull-backup-server-{{namebase}}-backup-{{id}}]
+<= request-pbs-{{namebase}}-{{id}}
+   slap-connection
+name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
+config = url name type server-key on-notification notify notification-id title
+config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
+config-name = ${request-pull-backup-server-{{namebase}}-{{id}}:config-name}
+config-type = push
+config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
+config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-notification-url}
+config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
+config-title = Pushing to {{namebase}} backup {{id}}
+slave = true
+{% endfor %}
+
+{% endmacro %}
\ No newline at end of file