Merge branch 'master' into grid-computing

CHANGES.txt

 Changes
 =======
 
+0.68.1 (2012-10-03)
+-------------------
+
+  * slaprunner: fix "logfile" parameter to "log_file"
+
+0.68 (2012-10-02)
+-----------------
+
+  * request.py: Remove useless calls to master, fix "update" method. [Cedric
+    de Saint Martin]
+  * Add webrunner test recipe. [Alain Takoudjou]
+  * Add logfile for slaprunner. [Cedric de Saint Martin]
+  * Fix check_url_available promise (syntax + checks + IPv6 support). [Cedric
+    de Saint Martin]
+
+0.67 (2012-09-26)
+-----------------
+
+  * Add check_page_content promise generator. [Cedric Le Ninivin]
+  * Fix check_url_available recipe. [Cedric de Saint Martin]
+  * Set up timezone database in mariab's mysql table so that we can use
+    timezone conversion function. [Kazuhiko Shiozaki]
+  * Add many resiliency-based recipes [Timothée Lacroix]
+  * Fix and unify request and requestoptional recipes [Cedric de Saint Martin]
+  * Fix Dropbear. [Antoine Catton]
+
 0.66 (2012-09-10)
 -----------------
 

component/gcc/buildout.cfg

@@ -170,4 +170,4 @@ make-targets = install -j1
 # until gcc will be simplified by using more robust build recipe (like
 # slapos.recipe.build) each time any of parts which reuses this one gets updated
 # the hack-revision have to be increased
-hack-revision = 1
+hack-revision = 2

component/ghostscript/buildout.cfg

@@ -29,5 +29,5 @@ environment =
 
 [ghostscript-9]
 <= ghostscript-common
-url = http://downloads.ghostscript.com/public/ghostscript-9.05.tar.bz2
-md5sum = 8bcef1f33ddf8a4d12b2cf8da385c191
+url = http://downloads.ghostscript.com/public/ghostscript-9.06.tar.bz2
+md5sum = 46f9ebe40dc52755287b30704270db11

component/groonga/buildout.cfg

@@ -7,12 +7,12 @@ parts =
 
 [groonga]
 recipe = hexagonit.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-2.0.5.tar.gz
-md5sum = 6ea6634ceee97f701add89ee39f4c1da
+url = http://packages.groonga.org/source/groonga/groonga-2.0.6.tar.gz
+md5sum = 64467d1ab8d5e0df8348a636972a95a7
 configure-options =
   --disable-static
   --disable-glibtest
   --disable-benchmark
   --disable-document
-  --disable-nginx
+  --disable-groonga-httpd
   --without-mecab

component/haproxy/buildout.cfg

@@ -9,8 +9,8 @@ parts = haproxy
 
 [haproxy]
 recipe = hexagonit.recipe.cmmi
-url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.21.tar.gz
-md5sum = f36d5d13fd4a44ab454eee5d06cb38a2
+url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.22.tar.gz
+md5sum = a0b007c76f6a78524f3b3dd5e704979c
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/java/buildout.cfg

@@ -29,9 +29,9 @@ slapos_promisee =
   directory:javaws
   file:lib/rt.jar
   file:bin/java
-# http://java.com/en/download/manual.jsp?locale=en
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59621 7466b0b86bef21e3e31ae578b2b9f472
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59623 c5cfb8393d257b51e7a70e56b7784ac9
+# http://java.com/en/download/manual_v6.jsp
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68284 7eda40e790de1a907d591b62949b6e72
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68286 839548714931443ba89719a995ece846
 script =
   import glob
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
@@ -47,48 +47,20 @@ script =
   workdir = guessworkdir(extract_dir)
   self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s")
 
-[java-sdk-1.6.0]
+[java-re-1.7.0]
 recipe = slapos.recipe.build
 slapos_promisee =
   directory:bin
   directory:lib
   directory:man
   directory:plugin
-  directory:javaws
-  file:jre/lib/rt.jar
-  file:bin/java
-x86 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-i586.bin bdb5f05bd20c6aa9a4729726191bf6fd
-x86-64 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-x64.bin 94f93a3ff03f824a238ecd79ad90433e
-script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  download_file = self.download(self.options['url'], self.options.get('md5sum'))
-  extract_dir = tempfile.mkdtemp(self.name)
-  os.chdir(extract_dir)
-  (download_dir, filename) = os.path.split(download_file)
-  auto_extract_bin = os.path.join(extract_dir, filename)
-  shutil.move(download_file, auto_extract_bin)
-  os.chmod(auto_extract_bin, 0777)
-  subprocess.call(["patch", auto_extract_bin, "-i", "${jdk-6u27-no-user-interaction-patch:location}/${jdk-6u27-no-user-interaction-patch:filename}"])
-  subprocess.call([auto_extract_bin])
-  self.cleanup_dir_list.append(extract_dir)
-  workdir = guessworkdir(extract_dir)
-  self.copyTree(os.path.join(workdir, "jdk1.6.0_27"), "%(location)s")
-
-[java-sdk-1.7.0]
-recipe = slapos.recipe.build
-slapos_promisee =
-  directory:bin
-  directory:lib
-  directory:man
-  directory:jre
-  file:jre/lib/rt.jar
+  file:lib/rt.jar
   file:bin/java
-  file:bin/javac
-x86 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-i586.tar.gz f97244a104f03731e5ff69f0dd5a9927
-x86-64 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-x64.tar.gz b3c1ef5faea7b180469c129a49762b64
+# http://java.com/en/download/manual.jsp?locale=en
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68234 ea99bedd9db33e9e2970f4b70abd1e4b
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68236 5aa9bd26cdf1fa6afd2b15826b4ba139
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
   extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
   self.copyTree(workdir, "%(location)s")
-

component/mariadb/buildout.cfg

@@ -59,9 +59,11 @@ environment =
   LDFLAGS=-L${libaio:location}/lib
 
 [mroonga-mariadb]
+# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
+# http://mroonga.github.com/
 recipe = hexagonit.recipe.cmmi
-url = https://github.com/downloads/mroonga/mroonga/mroonga-2.05.tar.gz
-md5sum = 6aee0e6ff948bba7c02854326afc11df
+url = https://github.com/downloads/mroonga/mroonga/mroonga-2.06.tar.gz
+md5sum = 89ce640a57ed8f2161b8358ff4c29c66
 configure-options =
   --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
   --with-mysql-config=${mariadb:location}/bin/mysql_config

component/percona-toolkit/buildout.cfg

@@ -10,7 +10,7 @@ parts =
 recipe = hexagonit.recipe.cmmi
 depends =
   ${perl:version}
-url = http://www.percona.com/redir/downloads/percona-toolkit/2.0.4/percona-toolkit-2.0.4.tar.gz
-md5sum = df7dffcccb48d50f143849629228d4b4
+url = http://www.percona.com/redir/downloads/percona-toolkit/2.1.3/percona-toolkit-2.1.3.tar.gz
+md5sum = 6f8c4a0a0e43e467766cbdabaec9ef93
 configure-command =
   ${perl:location}/bin/perl Makefile.PL

component/poppler/buildout.cfg

@@ -13,8 +13,8 @@ extends =
 
 [poppler]
 recipe = hexagonit.recipe.cmmi
-md5sum = 45dd2c16c8c4d1a39e830e45745c4e25
-url = http://poppler.freedesktop.org/poppler-0.20.2.tar.gz
+md5sum = 28c40266f374e1960a7bcead17d39f96
+url = http://poppler.freedesktop.org/poppler-0.20.3.tar.gz
 depends =
   ${libtiff:version}
 configure-options =

component/slapos/buildout.cfg

+# This file is used to install SlapOS Node itself.
+
 [buildout]
 extends =
   ../../stack/shacache-client.cfg
@@ -112,26 +114,27 @@ scripts =
   slapgrid-supervisord = slapos.grid.svcbackend:supervisord
   slapproxy = slapos.proxy:main
   slapos = slapos.entry:main
+  slapos-watchdog = slapos.grid.watchdog:main
 
 [versions]
-zc.buildout = 1.6.0-dev-SlapOS-006
+zc.buildout = 1.6.0-dev-SlapOS-007
 
 Jinja2 = 2.6
 Werkzeug = 0.8.3
 buildout-versions = 1.7
 collective.recipe.template = 1.9
 hexagonit.recipe.cmmi = 1.6
-lxml = 3.0alpha2
-meld3 = 0.6.8
+lxml = 2.3.6
+meld3 = 0.6.9
 netaddr = 0.7.10
-slapos.core = 0.28.6
+slapos.core = 0.31.1
 slapos.libnetworkcache = 0.13.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.28.6
+# slapos.core==0.31.1
 Flask = 0.9
 
 # Required by:
@@ -139,21 +142,21 @@ Flask = 0.9
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.core==0.28.6
+# slapos.core==0.31.1
 netifaces = 0.8
 
 # Required by:
-# slapos.core==0.28.6
+# slapos.core==0.31.1
 # slapos.libnetworkcache==0.13.2
-# supervisor==3.0a12
-# zc.buildout==1.6.0-dev-SlapOS-006
+# supervisor==3.0b1
+# zc.buildout==1.6.0-dev-SlapOS-007
 # zope.interface==4.0.1
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.core==0.28.6
-supervisor = 3.0a12
+# slapos.core==0.31.1
+supervisor = 3.0b1
 
 # Required by:
-# slapos.core==0.28.6
+# slapos.core==0.31.1
 zope.interface = 4.0.1

component/sphinx/README

+This component is not used anymore - but might be revived within 4 months.
+If not used within 6 months, feel free to delete this.

component/sphinx/buildout.cfg

@@ -20,8 +20,8 @@ filename = sphinx-1.10-beta-snowball.patch
 recipe = hexagonit.recipe.cmmi
 depends =
   ${mariadb:version}
-url = http://sphinxsearch.com/files/sphinx-2.0.2-beta.tar.gz
-md5sum = fafe0f1a71d0ded32404c067eba7d0b3
+url = http://sphinxsearch.com/files/sphinx-2.0.5-release.tar.gz
+md5sum = e71fdb5b0c2911247d48fb30550b9584
 configure-options =
   --with-mysql
   --with-mysql-includes=${mariadb:location}/include/mysql

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.67-dev'
+version = '0.68.2-dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -36,10 +36,12 @@ setup(name=name,
         'xml_marshaller', # need to communication with slapgrid
         'zc.buildout', # plays with buildout
         'zc.recipe.egg', # for scripts generation
+        'pytz', # for timezone database
         ],
       zip_safe=True,
       entry_points={
         'zc.buildout': [
+          'addresiliency = slapos.recipe.addresiliency:Recipe',
           'agent = slapos.recipe.agent:Recipe',
           'apache.frontend = slapos.recipe.apache_frontend:Recipe',
           'apachephp = slapos.recipe.apachephp:Recipe',
@@ -63,6 +65,7 @@ setup(name=name,
           'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
           'dropbear.client = slapos.recipe.dropbear:Client',
           'dropbear = slapos.recipe.dropbear:Recipe',
+          'dumpmdb = slapos.recipe.dumpmdb:Recipe',
           'duplicity = slapos.recipe.duplicity:Recipe',
           'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
           'equeue = slapos.recipe.equeue:Recipe',
@@ -86,6 +89,7 @@ setup(name=name,
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
           'helloworld = slapos.recipe.helloworld:Recipe',
+          'importmdb = slapos.recipe.importmdb:Recipe',
           'java = slapos.recipe.java:Recipe',
           'kumofs = slapos.recipe.kumofs:Recipe',
           'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
@@ -116,7 +120,7 @@ setup(name=name,
           'publishurl = slapos.recipe.publishurl:Recipe',
           'pwgen = slapos.recipe.pwgen:Recipe',
           'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
-          'requestoptional = slapos.recipe.requestoptional:Recipe',
+          'requestoptional = slapos.recipe.request:RequestOptional',
           'request = slapos.recipe.request:Recipe',
           'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
           'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',
@@ -130,6 +134,7 @@ setup(name=name,
           'slapmonitor = slapos.recipe.slapmonitor:Recipe',
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
+          'slaprunner.test = slapos.recipe.slaprunner:Test',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
           'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',

slapos/recipe/requestoptional.py → slapos/recipe/addresiliency/__init__.py

@@ -24,84 +24,58 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import logging
+from slapos.recipe.librecipe import GenericSlapRecipe
 
-from slapos import slap as slapmodule
+import sys
+import os
 
-class Recipe(object):
-  """ Request a SlapOS instance. Won't fail if instance is not ready.
-  """
-
-  def __init__(self, buildout, name, options):
-    self.logger = logging.getLogger(name)
-
-    slap = slapmodule.slap()
-
-    self.software_release_url = options['software-url']
-    self.name = options['name']
-
-    slap.initializeConnection(options['server-url'],
-                              options.get('key-file'),
-                              options.get('cert-file'),
-                             )
-    computer_partition = slap.registerComputerPartition(
-      options['computer-id'], options['partition-id'])
-    self.request = computer_partition.request
-
-    self.isSlave = False
-    if 'slave' in options:
-      self.isSlave = options['slave'].lower() in ['y', 'yes', 'true', '1']
 
-    self.return_parameters = []
-    if 'return' in options:
-      self.return_parameters = [str(parameter).strip()
-                               for parameter in options['return'].split()]
-    else:
-      self.logger.debug("No parameter to return to main instance."
-                          "Be careful about that...")
-
-    software_type = 'RootSoftwareInstance'
-    if 'software-type' in options:
-      software_type = options['software-type']
-
-    filter_kw = {}
-    if 'sla' in options:
-      for sla_parameter in options['sla'].split():
-        filter_kw[sla_parameter] = options['sla-%s' % sla_parameter]
-
-    partition_parameter_kw = {}
-    if 'config' in options:
-      for config_parameter in options['config'].split():
-        partition_parameter_kw[config_parameter] = \
-            options['config-%s' % config_parameter]
-
-    self.instance = self.request(options['software-url'], software_type,
-      self.name, partition_parameter_kw=partition_parameter_kw,
-      filter_kw=filter_kw, shared=self.isSlave)
+class Recipe(GenericSlapRecipe):
+  """ This class provides the installation of the resilience
+      script on the partition.
+  """
 
-    self.failed = None
-    for param in self.return_parameters:
-      try:
-        options['connection-%s' % param] = str(
-            self.instance.getConnectionParameter(param))
-      except slapmodule.NotFoundError:
-        options['connection-%s' % param] = ''
-        if self.failed is None:
-          self.failed = param
+  def _install(self):
+    path_list = []
+    self_id = int(self.parameter_dict['number'])
+    ip = self.parameter_dict['ip-list'].split(' ')
+    print 'Creating bully script with  ips : %s\n' % ip
+    slap_connection = self.buildout['slap-connection']
 
-  def install(self):
-    if self.failed is not None:
-      # Check instance status to know if instance has been deployed
+    path_conf = os.path.join(self.options['script'], 'conf.in')
+    path_bully = os.path.join(self.options['script'], self.parameter_dict['script'])
+    path_bully_new = os.path.join(self.options['script'], 'new.py')
+    path_run = os.path.join(self.options['run'], self.parameter_dict['wrapper'])
+    print 'paths: %s\n%s\n' % (path_run, path_bully)
+    bully_conf = dict(self_id=self_id,
+                      ip_list=ip,
+                      executable=sys.executable,
+                      syspath=sys.path,
+                      server_url=slap_connection['server-url'],
+                      key_file=slap_connection.get('key-file'),
+                      cert_file=slap_connection.get('cert-file'),
+                      computer_id=slap_connection['computer-id'],
+                      partition_id=slap_connection['partition-id'],
+                      software=slap_connection['software-release-url'],
+                      namebase=self.parameter_dict['namebase'],
+                      confpath=path_conf)
     try:
-        status = self.instance.getState()
-      except slapmodule.NotFoundError:
-        status = 'not ready yet'
-      except AttributeError:
-        status = 'unknown'
-      error_message = 'Connection parameter %s not found. '\
-          'Requested instance is currently %s. If this error persists, '\
-          'check status of this instance.' % (self.failed, status)
-      self.logger.error(error_message)
-    return []
+      conf = self.createFile(path_conf,
+                             self.substituteTemplate(
+                             self.getTemplateFilename('conf.in.in'),
+                             bully_conf))
+      path_list.append(conf)
+      script = self.createExecutable(path_bully,
+                                     self.substituteTemplate(
+                                     self.getTemplateFilename('bully.py.in'),
+                                     bully_conf))
+      path_list.append(script)
 
-  update = install
+      wrapper = self.createPythonScript(
+          path_run,
+          'slapos.recipe.librecipe.execute.execute',
+          [path_bully])
+      path_list.append(wrapper)
+    except IOError:
+      pass
+    return path_list

slapos/recipe/addresiliency/template/bully.py.in

+#!%(executable)s
+
+import select
+import socket
+import threading
+import time
+import sys
+
+sys.path[:] = %(syspath)s
+
+from slapos import slap as slapmodule
+
+port = 50000
+size = 1024
+wait = True
+
+
+def loadConnectionInfos():
+  connectionInfos = {}
+  file = open('%(confpath)s', 'r')
+  params = file.read().split('\n')
+  file.close()
+  ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
+  connectionInfos['self_id'] = int(params[1])
+
+  connectionInfos['server_list'] = \
+      [(i, ip_list[i]) for i in range(len(ip_list))]
+
+  connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
+
+  return connectionInfos
+
+
+def rename_broken_and_stop():
+  try:
+    slap = slapmodule.slap()
+    slap.initializeConnection('%(server_url)s',
+                              '%(key_file)s',
+                              '%(cert_file)s')
+    computer_partition = slap.registerComputerPartition('%(computer_id)s',
+                                                        '%(partition_id)s')
+    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
+
+    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+    broken.stopped()
+    computer_partition.rename('%(namebase)s0')
+    print 'renaming done\n'
+  except slapos.slap.slap.ServerError:
+    print 'Internal server error\n'
+
+def election():
+  global wait
+  connection = loadConnectionInfos()
+  message = "%%s, %%s" %% (connection['self_id'], "Election")
+  victory = True
+
+  for (remote_id, addr) in connection['server_list']:
+    if remote_id > connection['self_id']:
+      try:
+        s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        s.connect((addr, port + remote_id))
+        s.send(message)
+        reply = s.recv(size)
+        if reply == "%%s, %%s" %% (remote_id, "Alive"):
+          victory = False
+      except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+        pass
+      finally:
+        s.close()
+
+  if victory:
+    wait = True
+    for (remote_id, addr) in connection['server_list']:
+      if remote_id < connection['self_id']:
+        try:
+          s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+          s.connect((addr, port + remote_id))
+          s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+          pass
+        finally:
+          s.close()
+    rename_broken_and_stop()
+
+
+def failure_detect():
+  global wait
+  connection = loadConnectionInfos()
+  while True:
+    time.sleep(30)
+    if wait:
+      print 'waiting 30 minutes\n'
+      time.sleep(30 * 60)
+      wait = False
+    if not connection['server_list'][0]:
+      continue
+    (remote_id, addr) = connection['server_list'][0]
+    try:
+      s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+      s.connect((addr, port + remote_id))
+      s.close()
+    except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+      s.close()
+      election()
+
+failure_detect_thread = threading.Thread(target=failure_detect)
+failure_detect_thread.start()
+
+connection = loadConnectionInfos()
+s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+s.bind((connection['self_ip'], port + connection['self_id']))
+s.listen(5)
+#election()
+while True:
+  force_election = False
+  client, _ = s.accept()
+  client_message = client.recv(1024)
+  if client_message:
+    client_id, message = client_message.split(', ')
+    client_id = eval(client_id)
+    if message == "Victory":
+      wait = True
+      print "%%s wins" %% client_id
+    elif message == "Election":
+      print "%%s starts an election" %% client_id
+      if client_id < connection['self_id']:
+        client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
+        force_election = True
+  client.close()
+  if force_election:
+    election()

slapos/recipe/addresiliency/template/bully_new.py.in

+#!%(executable)s
+
+import socket
+import time
+import sys
+import thread
+import time
+import os
+
+sys.path[:] = %(syspath)s
+
+from slapos import slap as slapmodule
+
+port = 50000
+size = 1024
+
+
+def rename_broken_and_stop():
+  try:
+    slap = slapmodule.slap()
+    slap.initializeConnection('%(server_url)s',
+                              '%(key_file)s',
+                              '%(cert_file)s')
+    computer_partition = slap.registerComputerPartition('%(computer_id)s',
+                                                        '%(partition_id)s')
+    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
+
+    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+    broken.stopped()
+    computer_partition.rename('%(namebase)s0')
+    print 'renaming done\n'
+  except slapos.slap.slap.ServerError:
+    print 'Internal server error\n'
+
+
+## Leader is always number 0
+
+class ResilientInstance(object):
+
+    def __init__(self, comm):
+        self.comm = comm
+        self.id = 0
+        self.state = 'normal'
+        self.halter = 0
+        self.nbComp = nbComp
+        self.inElection = False
+        self.alive = True
+        self.lastPing = time.clock()
+
+        self.mainCanal = self.comm.canal(['ping', 'halt',
+                                          'victory'])
+
+        self.okCanal = self.comm.canal(['ok'])
+        self.loadConnectionInfos()
+
+
+    def loadConnectionInfos(self):
+        file = open('%(confpath)s', 'r')
+        params = file.read().split('\n')
+        file.close()
+        self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
+        new_id = int(params[1])
+        if self.id != new_id:
+            self.halter = new_id
+            self.id = new_id 
+
+    ## Needs to be changed to use the master
+    def aliveManagement(self):
+        while self.alive:
+            time.sleep(30*60)
+            if self.id == 0:
+                continue
+            self.comm.send('ping', 0)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            self.election()
+
+    def listen(self):
+        while self.alive:
+            self.comm.recv()
+
+    def main(self):
+        while self.alive:
+            message, sender = self.mainCanal.get()
+            if message == 'ping':
+                    self.comm.send('ok', sender)
+
+            elif message == 'halt':
+                self.state = 'waitingConfirm'
+                self.halter = sender
+                self.comm.send('ok', sender)
+
+            elif message == 'victory':
+                if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
+                    print '%s thinks %s is the leader\n' % (self.id, sender)
+                    self.comm.send('ok', sender)
+                self.state = 'normal'
+
+    def election(self):
+        self.inElection = True
+        self.loadConnectionInfos()
+        #Check if I'm the highest instance alive
+        for higher in range(self.id + 1, self.nbComp):
+            self.comm.send('ping', higher)
+            message, sender = self.okCanal.get()
+            if message:
+                #print '%s is alive (%s)\n' % (higher, self.id)
+                self.inElection = False
+                return False
+            continue
+
+        if not self.alive:
+            return False
+
+        #I should be the new coordinator, halt those below me
+        print 'Should be ME : %s \n' % self.id
+        self.state = 'election'
+        self.halter = self.id
+        ups = []
+        for lower in range(self.id):
+            self.comm.send('halt', lower)
+            message, sender = self.okCanal.get()
+            if message:
+                ups.append(lower)
+
+        #Broadcast Victory
+        self.state = 'reorganization'
+        for up in ups:
+            self.comm.send('victory', up)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            print 'Something is wrong... let\'s start over\n'
+            return self.election()
+        self.state = 'normal'
+        self.active = True
+        print '%s Is THE LEADER \n' % self.id
+
+        rename_broken_and_stop()
+
+        self.inElection = False
+
+        return True
+
+
+class FilteredCanal(object):
+
+    def __init__(self, accept, timeout):
+        self.accept = accept
+        self.list = []
+        self.lock = thread.allocate_lock()
+        self.timeout = timeout
+
+    def append(self, message, sender):
+        if message in self.accept:
+            self.lock.acquire()
+            self.list.append([message, sender])
+            self.lock.release()
+
+    def get(self):
+        start = time.clock()
+        while (time.clock() - start < self.timeout):
+            self.lock.acquire()
+            if self.list:
+                self.lock.release()
+                val = self.list[0]
+                self.list = self.list[1:]
+                return val
+            self.lock.release()
+        return [None, None]
+
+
+class Wrapper(object):
+
+    def __init__(self, timeout=20):
+        self.read_pipes = [os.fdopen(x) for x in read_pipes]
+        self.write_pipes = write_pipes
+        self.canals = []
+        self.ips = []
+        self.id = 0
+        self.timeout = timeout
+        self.getConnectionInfos()
+        self.socket = None
+
+    def getConnectionInfos(self):
+        file = open('%(confpath)s', 'r')
+        params = file.read().split('\n')
+        file.close()
+        self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
+        self.id = int(params[1])
+
+    def start(self):
+        self.getConnectionInfos()
+        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        self.socket.bind((self.ips[self.id], port + self.id))
+        s.listen(5)
+
+    def send(self, message, number):
+        self.getConnectionInfos()
+        try:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            s.connect((self.ips[number], port + number))
+            s.send(message + (' %s\n' % self.id))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+          pass
+        finally:
+            s.close()
+
+    def canal(self, accept):
+        created = FilteredCanal(accept, self.timeout)
+        self.canals.append(created)
+        return created
+
+    def recv(self):
+        client, _ = s.accept()
+        client_message = client.recv(1024)
+        if client_message:
+            message, sender = client_message.split()
+            for canal in self.canals:
+                canal.append(message, sender)
+
+
+wrapper = createWrapper(20)
+
+computer = ResilientInstance(wrapper)
+
+#idle waiting for connection infos
+while computer.nbComp < 2 :
+    computer.loadConnectionInfos()
+    time.sleep(30)
+
+print 'Starting\n'
+
+computer.comm.start()
+thread.start_new_thread(computer.listen, ())
+thread.start_new_thread(computer.main, ())
+thread.start_new_thread(computer.aliveManagement, ())
+
+while True:
+    continue
+
+

slapos/recipe/addresiliency/template/conf.in.in

+%(ip_list)s
+%(self_id)s

slapos/recipe/apacheproxy/template/apache.in

@@ -3,7 +3,6 @@
 
 # Basic server configuration
 PidFile "%(pid_file)s"
-LockFile "%(lock_file)s"
 Listen %(ip)s:%(port)s
 ServerAdmin someone@email
 DefaultType text/plain
@@ -22,13 +21,15 @@ CustomLog "%(access_log)s" common
 <Directory />
     Options FollowSymLinks
     AllowOverride None
-    Order deny,allow
-    Deny from all
+    Require all denied
 </Directory>
 
 ProxyPass / %(backend_url)s
 
 # List of modules
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule authz_host_module modules/mod_authz_host.so
 LoadModule log_config_module modules/mod_log_config.so
 LoadModule setenvif_module modules/mod_setenvif.so

slapos/recipe/check_url_available/template/check_url.in

@@ -9,11 +9,22 @@ if [ -z $URL ]; then
   exit 3
 fi
 
-CODE=$(%(curl_path)s -k -sL $URL -w %%{http_code} -o /dev/null)
+CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} -o /dev/null)
+
+if [ $? -eq 3 ]; then
+  echo "URL malformed: $URL." >&2
+  exit 1
+fi
+
+if [ $? -eq 7 ]; then
+  echo "Failed to connect to host: $URL." >&2
+  exit 1
+fi
 
 if [ ! $CODE ]; then
   echo "$URL is not available (server not reachable)." >&2
   exit 1
+fi
 
 if [ $CODE -eq 000 ]; then
   echo "$URL is not available (server not reachable)." >&2

slapos/recipe/dropbear.py

@@ -167,6 +167,7 @@ class AddAuthorizedKey(GenericBaseRecipe):
     path_list.append(ssh)
 
     authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
-    authorized_keys.append(self.options['key'])
+    for key in self.options['key'].split(' '):
+      authorized_keys.append(key)
 
     return path_list

slapos/recipe/generic_mysql/__init__.py

@@ -154,26 +154,38 @@ class Recipe(GenericBaseRecipe):
     # TODO: move to a separate recipe (ack'ed by Cedric)
     # percona toolkit (formerly known as maatkit) installation
     for pt_script_name in (
+        'pt-align',
         'pt-archiver',
+        'pt-collect',
         'pt-config-diff',
         'pt-deadlock-logger',
+        'pt-diskstats',
         'pt-duplicate-key-checker',
         'pt-fifo-split',
         'pt-find',
+        'pt-fingerprint',
         'pt-fk-error-logger',
         'pt-heartbeat',
         'pt-index-usage',
+        'pt-ioprofile',
         'pt-kill',
         'pt-log-player',
+        'pt-mext',
+        'pt-mysql-summary',
         'pt-online-schema-change',
+        'pt-pmp',
         'pt-query-advisor',
         'pt-query-digest',
         'pt-show-grants',
+        'pt-sift',
         'pt-slave-delay',
         'pt-slave-find',
         'pt-slave-restart',
+        'pt-stalk',
+        'pt-summary',
         'pt-table-checksum',
         'pt-table-sync',
+        'pt-table-usage',
         'pt-tcp-model',
         'pt-trend',
         'pt-upgrade',

slapos/recipe/generic_mysql/mysql.py

@@ -2,6 +2,7 @@ import os
 import subprocess
 import time
 import sys
+import pytz
 
 
 def runMysql(args):
@@ -41,7 +42,6 @@ def updateMysql(args):
   sleep = 30
   is_succeed = False
   while True:
-    if not is_succeed:
     mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
     mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     result = mysql_upgrade.communicate()[0]
@@ -49,7 +49,6 @@ def updateMysql(args):
       mysql_upgrade.kill()
     if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
       print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
-        print 'Sleeping for %ss and retrying' % sleep
     else:
       if mysql_upgrade.returncode == 0:
         print "MySQL database upgraded with result:\n%s" % result
@@ -63,10 +62,30 @@ def updateMysql(args):
         mysql.kill()
       if mysql.returncode != 0:
         print 'Command %r failed with:\n%s' % (mysql_list, result)
-          print 'Sleeping for %ss and retrying' % sleep
       else:
+        # import timezone database
+        mysql_tzinfo_to_sql_binary = os.path.join(
+          os.path.dirname(conf['mysql_binary'].strip()), 'mysql_tzinfo_to_sql')
+        zoneinfo_directory = '%s/zoneinfo' % os.path.dirname(pytz.__file__)
+        mysql_tzinfo_to_sql_list = [mysql_tzinfo_to_sql_binary, zoneinfo_directory]
+        mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
+        if mysql.returncode != 0:
+          print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
+        else:
+          mysql = subprocess.Popen(mysql_list + ['mysql',], stdin=subprocess.PIPE,
+                                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          result = mysql.communicate(timezone_sql)[0]
+          if mysql.returncode is None:
+            mysql.kill()
+          if mysql.returncode != 0:
+            print 'Command %r failed with:\n%s' % (mysql_list, result)
           is_succeed = True
+    if is_succeed:
       print 'SlapOS initialisation script succesfully applied on database.'
+      break
+    print 'Sleeping for %ss and retrying' % sleep
     sys.stdout.flush()
     sys.stderr.flush()
     time.sleep(sleep)

slapos/recipe/generic_mysql/template/my.cnf.in

@@ -24,7 +24,7 @@ long_query_time = 1
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so;handlersocket.so
+plugin-load = ha_mroonga.so;handlersocket.so
 
 # By default only 100 connections are allowed, when using zeo
 # we may have much more connections

slapos/recipe/haproxy/__init__.py

@@ -73,7 +73,11 @@ class Recipe(GenericBaseRecipe):
     server_snippet = ""
     i = 0
     name = self.options['name']
-    for address in self.options['backend-list'].split():
+    backend_list = self.options['backend-list']
+    if isinstance(backend_list, str):
+      # BBB
+      backend_list = backend_list.split()
+    for address in backend_list:
       i += 1
       server_snippet += self.substituteTemplate(
           snippet_filename, dict(

slapos/recipe/lamp/template/my.cnf.in

@@ -21,7 +21,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

slapos/recipe/mysql/__init__.py

@@ -31,6 +31,7 @@ import subprocess
 from slapos.recipe.librecipe import GenericBaseRecipe
 from slapos.recipe.librecipe import filehash
 
+
 class Recipe(GenericBaseRecipe):
 
   def _options(self, options):

slapos/recipe/mysql/template/my.cnf.in

@@ -19,7 +19,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

slapos/recipe/pbs.py

@@ -213,6 +213,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
         for slave in slaves:
           path_list.extend(self.add_slave(slave, known_hosts))
 
+
     else:
       command = [self.options['rdiffbackup-binary']]
       self.logger.info("Server mode")

slapos/recipe/request.py

@@ -27,6 +27,9 @@
 import logging
 
 from slapos import slap as slapmodule
+import slapos.recipe.librecipe.generic as librecipe
+
+DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
 class Recipe(object):
   """
@@ -97,8 +100,6 @@ class Recipe(object):
     request = slap.registerComputerPartition(
       options['computer-id'], options['partition-id']).request
 
-    isSlave = options.get('slave', '').lower() in ['y', 'yes', 'true', '1']
-
     return_parameters = []
     if 'return' in options:
       return_parameters = [str(parameter).strip()
@@ -107,7 +108,7 @@ class Recipe(object):
       self.logger.debug("No parameter to return to main instance."
         "Be careful about that...")
 
-    software_type = options.get('software-type', 'RootInstanceSoftware')
+    software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE)
 
     filter_kw = {}
     if 'sla' in options:
@@ -120,6 +121,8 @@ class Recipe(object):
         partition_parameter_kw[config_parameter] = \
             options['config-%s' % config_parameter]
 
+    isSlave = options.get('slave', '').lower() in \
+        librecipe.GenericBaseRecipe.TRUE_VALUES
     self.instance = instance = request(software_url, software_type,
       name, partition_parameter_kw=partition_parameter_kw,
       filter_kw=filter_kw, shared=isSlave)
@@ -128,7 +131,7 @@ class Recipe(object):
       try:
         options['connection-%s' % param] = str(
           instance.getConnectionParameter(param))
-      except slapmodule.NotFoundError:
+      except (slapmodule.NotFoundError, slapmodule.ServerError):
         options['connection-%s' % param] = ''
         if self.failed is None:
           self.failed = param
@@ -137,9 +140,12 @@ class Recipe(object):
     if self.failed is not None:
       # Check instance status to know if instance has been deployed
       try:
+        if self.instance.getComputerId() is not None:
           status = self.instance.getState()
-      except slapmodule.NotFoundError:
-        status = 'not ready yet, please try again'
+        else:
+          status = 'not ready yet'
+      except (slapmodule.NotFoundError, slapmodule.ServerError):
+        status = 'not ready yet'
       except AttributeError:
         status = 'unknown'
       error_message = 'Connection parameter %s not found. '\
@@ -150,3 +156,28 @@ class Recipe(object):
     return []
 
   update = install
+
+class RequestOptional(Recipe):
+  """
+  Request a SlapOS instance. Won't fail if instance is not ready.
+  Same as slapos.cookbook:request, but won't raise in case of problem.
+  """
+  def install(self):
+    if self.failed is not None:
+      # Check instance status to know if instance has been deployed
+      try:
+        if self.instance.getComputerId() is not None:
+          status = self.instance.getState()
+        else:
+          status = 'not ready yet'
+      except (slapmodule.NotFoundError, slapmodule.ServerError):
+        status = 'not ready yet'
+      except AttributeError:
+        status = 'unknown'
+      error_message = 'Connection parameter %s not found. '\
+          'Requested instance is currently %s. If this error persists, '\
+          'check status of this instance.' % (self.failed, status)
+      self.logger.warning(error_message)
+    return []
+
+  update = install

slapos/recipe/slaprunner/__init__.py

@@ -39,6 +39,7 @@ class Recipe(GenericBaseRecipe):
     self.instance_directory = options['instance-directory'].strip()
     self.partition_amount = options['partition-amount'].strip()
     self.cloud9_url = options['cloud9-url'].strip()
+    self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
     # Set slaprunner access URL
     options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
 
@@ -79,10 +80,12 @@ class Recipe(GenericBaseRecipe):
     path_list.append(config_file)
 
     environment = dict(
-        PATH=os.path.dirname(self.options['git-binary']) + ':' + os.environ['PATH'],
+        PATH=os.path.dirname(
+            self.options['git-binary']) + ':' + os.environ['PATH'],
         GIT_SSH=self.options['ssh_client']
     )
-    launch_args = [self.options['slaprunner'].strip(), config_file]
+    launch_args = [self.options['slaprunner'].strip(), config_file,
+                   '--log_file', self.log_file]
     if self.optionIsTrue('debug', default=False):
       launch_args.append('--debug')
 
@@ -93,3 +96,69 @@ class Recipe(GenericBaseRecipe):
     path_list.append(wrapper)
 
     return path_list
+
+class Test(GenericBaseRecipe):
+  def _options(self, options):
+    self.ipv4 = options['ipv4'].strip()
+    self.ipv6 = options['ipv6'].strip()
+    self.proxy_port = options['proxy_port'].strip()
+    self.runner_port = options['runner_port'].strip()
+    self.workdir = options['working-directory'].strip()
+    self.software_directory = options['software-directory'].strip()
+    self.instance_directory = options['instance-directory'].strip()
+    self.partition_amount = options['partition-amount'].strip()
+    self.cloud9_url = options['cloud9-url'].strip()
+    # Set slaprunner access URL
+    options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
+
+  def install(self):
+    path_list = []
+
+    configuration = dict(
+        software_root=self.software_directory,
+        instance_root=self.instance_directory,
+        master_url='http://%s:%s/' % (self.ipv4, self.proxy_port),
+        computer_id='slaprunner',
+        partition_amount=self.partition_amount,
+        slapgrid_sr=self.options['slapgrid_sr'],
+        slapgrid_cp=self.options['slapgrid_cp'],
+        slapproxy=self.options['slapproxy'],
+        supervisor=self.options['supervisor'],
+        supervisord_config=os.path.join(self.instance_directory, 'etc',
+          'supervisord.conf'),
+        runner_workdir=self.workdir,
+        etc_dir=self.options['etc_dir'],
+        runner_host=self.ipv6,
+        runner_port=self.runner_port,
+        ipv4_address=self.ipv4,
+        ipv6_address=self.ipv6,
+        proxy_host=self.ipv4,
+        proxy_port=self.proxy_port,
+        proxy_database=os.path.join(self.workdir, 'proxy.db'),
+        git=self.options['git-binary'],
+        ssh_client=self.options['ssh_client'],
+        public_key=self.options['public_key'],
+        private_key=self.options['private_key'],
+        cloud9_url=self.cloud9_url
+    )
+
+    config_file = self.createFile(self.options['slapos.cfg'],
+        self.substituteTemplate(self.getTemplateFilename('slapos.cfg.in'),
+        configuration))
+    path_list.append(config_file)
+
+    environment = dict(
+        PATH=os.path.dirname(
+            self.options['git-binary']) + ':' + os.environ['PATH'],
+        GIT_SSH=self.options['ssh_client'],
+        CONFIG_FILE_PATH=config_file
+    )
+    launch_args = [self.options['slaprunnertest'].strip()]
+
+    wrapper = self.createPythonScript(self.options['wrapper'],
+        'slapos.recipe.librecipe.execute.executee',
+        (launch_args, environment)
+    )
+    path_list.append(wrapper)
+
+    return path_list

slapos/recipe/slaprunner/execute.py

-import os
-
-
-def execute(args):
-  """Portable execution with process replacement"""
-  if args.get("path", None):
-    os.environ['PATH'] = args["path"]
-  os.execv(args["launch_args"][0], args["launch_args"])

slapos/recipe/softwaretype.py

@@ -87,6 +87,8 @@ class Recipe:
       computer_partition_id)
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()
     software_type = self.parameter_dict['slap_software_type']
+    self.logger.info('Deploying instance with software type %s' % \
+        software_type)
 
     if software_type not in self.options:
       if 'default' in self.options:

slapos/recipe/tidstorage/__init__.py

@@ -29,13 +29,21 @@ from slapos.recipe.librecipe import GenericBaseRecipe
 
 class Recipe(GenericBaseRecipe):
   def install(self):
-    path_list = []
-    a = path_list.append
-    configuration_file = self.createFile(self.options['configuration-path'], self.substituteTemplate(self.getTemplateFilename('tidstorage.py.in'), self.options))
-    a(configuration_file)
-    tidstorage_wrapper = self.createPythonScript(self.options['tidstorage-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstoraged-binary'], '--nofork', '--config', configuration_file])
-    a(tidstorage_wrapper)
-    repozo_wrapper = self.createPythonScript(self.options['repozo-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstorage-repozo-binary'], '--config', configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
-    a(repozo_wrapper)
+    configuration_file = self.createFile(
+      self.options['configuration-path'],
+      self.substituteTemplate(
+        self.getTemplateFilename('tidstorage.py.in'), self.options))
 
-    return path_list
+    tidstorage_wrapper = self.createPythonScript(
+      self.options['tidstorage-wrapper'],
+      'slapos.recipe.librecipe.execute.execute',
+      [self.options['tidstoraged-binary'],  '--nofork', '--config',
+        configuration_file])
+
+    repozo_wrapper = self.createPythonScript(
+      self.options['repozo-wrapper'],
+      'slapos.recipe.librecipe.execute.execute',
+      [self.options['tidstorage-repozo-binary'], '--config',
+        configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
+
+    return [configuration_file, tidstorage_wrapper, repozo_wrapper]

slapos/recipe/tidstorage/template/tidstorage.py.in

 known_tid_storage_identifier_dict = %(known-tid-storage-identifier-dict)s
-base_url = '%(base-url)s'
 address = '%(ip)s'
 port = %(port)s
 

slapos/recipe/xwiki/template/my.cnf.in

@@ -21,7 +21,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

software/slaprunner/instance-runner.cfg

@@ -2,6 +2,7 @@
 parts =
   cloud9
   slaprunner
+  test-runner
   sshkeys-dropbear
   dropbear-server-add-authorized-key
   sshkeys-authority
@@ -38,13 +39,18 @@ services = $${rootdirectory:etc}/run/
 run = $${rootdirectory:var}/run/
 backup = $${rootdirectory:srv}/backup/
 promises = $${rootdirectory:etc}/promise/
+test = $${rootdirectory:etc}/test/
 
 [runnerdirectory]
 recipe = slapos.cookbook:mkdirectory
 home = $${rootdirectory:srv}/runner/
+test = $${rootdirectory:srv}/test/
 project = $${:home}/project
 software-root = $${:home}/software
 instance-root = $${:home}/instance
+project-test = $${:test}/project
+software-test = $${:test}/software
+instance-test = $${:test}/instance
 
 #Create password recovery code for slaprunner
 [recovery-code]
@@ -85,9 +91,10 @@ ipv4 = $${slap-network-information:local-ipv4}
 ipv6 = $${slap-network-information:global-ipv6}
 proxy_port = 50000
 runner_port = 50000
-partition-amount = 7
+partition-amount = $${slap-parameter:instance-amount}
 cloud9-url = $${cloud9:access-url}
 wrapper = $${basedirectory:services}/slaprunner
+debug = False
 
 # Deploy dropbear (minimalist ssh server)
 [sshkeys-directory]
@@ -95,6 +102,18 @@ recipe = slapos.cookbook:mkdirectory
 requests = $${directory:sshkeys}/requests/
 keys = $${directory:sshkeys}/keys/
 
+[test-runner]
+<= slaprunner
+recipe = slapos.cookbook:slaprunner.test
+slaprunnertest = ${buildout:directory}/bin/slaprunnertest
+slapos.cfg = $${rootdirectory:etc}/slapos-test.cfg
+working-directory = $${runnerdirectory:test}
+project-directory = $${runnerdirectory:project-test}
+software-directory = $${runnerdirectory:software-test}
+instance-directory = $${runnerdirectory:instance-test}
+wrapper = $${rootdirectory:bin}/runTestSuite
+etc_dir = $${basedirectory:test}
+
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
 request-directory = $${sshkeys-directory:requests}
@@ -182,3 +201,5 @@ port = $${dropbear-server:port}
 [slap-parameter]
 # Default value if no ssh key is specified
 authorized-key =
+# Default value of instances number in slaprunner
+instance-amount = 10

software/slaprunner/software.cfg

@@ -25,7 +25,7 @@ md5sum = 5307e4200f044ae57b504ad68444491c
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg
-md5sum = b57ebff565595fa92cd7d5adc6a22c7e
+md5sum = bd1c73c149b4adf2fbab80bc29c782c8
 mode = 0644
 
 [eggs]
@@ -54,44 +54,48 @@ signature-certificate-list =
   -----END CERTIFICATE-----
 
 [versions]
+# Forced stable version of lxml - 3.0 alpha keeps appearing/diseappearing on
+# Pypi
+lxml = 2.3.6
+
 Flask-Auth = 0.8
 Jinja2 = 2.6
 Werkzeug = 0.8.3
-apache-libcloud = 0.11.1
+apache-libcloud = 0.11.3
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-meld3 = 0.6.8
+meld3 = 0.6.9
 plone.recipe.command = 1.1
 pycrypto = 2.6
-slapos.cookbook = 0.64.2
+slapos.cookbook = 0.68.1
 slapos.libnetworkcache = 0.13.2
-slapos.recipe.build = 0.11
+slapos.recipe.build = 0.11.5
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.30
+slapos.toolbox = 0.31.1
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
+# slapos.core==0.31.2
+# slapos.toolbox==0.31.1
 Flask = 0.9
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31.1
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.68
 PyXML = 0.8.4
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31.1
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31.1
 feedparser = 5.1.2
 
 # Required by:
@@ -99,58 +103,56 @@ feedparser = 5.1.2
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.68
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
-# xml-marshaller==0.9.7
-lxml = 3.0alpha2
-
-# Required by:
-# slapos.cookbook==0.64.2
-netaddr = 0.7.9
+# slapos.cookbook==0.68
+netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 netifaces = 0.8
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31.1
 paramiko = 1.7.7.2
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31.1
 psutil = 0.6.1
 
 # Required by:
-# slapos.cookbook==0.64.2
-# slapos.core==0.28.5
+# slapos.cookbook==0.68
+pytz = 2012f
+
+# Required by:
+# slapos.cookbook==0.68
+# slapos.core==0.31.2
 # slapos.libnetworkcache==0.13.2
-# slapos.toolbox==0.30
-# supervisor==3.0a12
+# slapos.toolbox==0.31.1
+# supervisor==3.0b1
 # zc.buildout==1.6.0-dev-SlapOS-007
 # zc.recipe.egg==1.3.2
 # zope.interface==4.0.1
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.64.2
-slapos.core = 0.28.5
+# slapos.cookbook==0.68
+slapos.core = 0.31.2
 
 # Required by:
-# slapos.core==0.28.5
-supervisor = 3.0a12
+# slapos.core==0.31.2
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.68
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.68
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 zope.interface = 4.0.1

stack/erp5/buildout.cfg

@@ -17,7 +17,7 @@ find-links =
 
 extends =
 # Exact version of Zope
-  http://svn.zope.org/repos/main/Zope/tags/2.12.23/versions.cfg
+  http://svn.zope.org/repos/main/Zope/tags/2.12.25/versions.cfg
   ../../stack/slapos.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/dcron/buildout.cfg
@@ -48,7 +48,6 @@ extends =
   ../../component/python-2.7/buildout.cfg
   ../../component/python-ldap-python/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
-  ../../component/sphinx/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/subversion/buildout.cfg
   ../../component/tesseract/buildout.cfg
@@ -85,7 +84,6 @@ parts =
   ghostscript
   mariadb
   mroonga-mariadb
-  sphinx
   imagemagick
   libdmtx
   dmtx-utils
@@ -186,7 +184,7 @@ context =
 [template-mariadb]
 < = template-jinja2-base
 filename = instance-mariadb.cfg
-md5sum = 0e8e9bfc8c8a5f1bd71f5c4f346ef68b
+md5sum = 867bd8e3c4f56db2aab2fcad59f2efd8
 extra-context =
     key coreutils_location coreutils:location
     key dcron_location dcron:location
@@ -200,13 +198,6 @@ extra-context =
     key sed_location sed:location
     key xtrabackup_location xtrabackup:location
 
-[template-sphinx]
-< = template-jinja2-base
-filename = instance-sphinx.cfg
-md5sum = e3975afd5220f251d6b6e7024a2ca08b
-extra-context =
-    key sphinx_location sphinx:location
-
 [template-zope]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-zope.cfg.in
@@ -227,7 +218,7 @@ extra-context =
 [template-tidstorage]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-tidstorage.cfg.in
-md5sum = 520e39f2612a045b42c0ad914bfdc5a9
+md5sum = 1f3e7cfc719d3c2eee4860678f6a67df
 mode = 640
 
 [template-cloudooo]
@@ -239,7 +230,7 @@ mode = 640
 [template-zope-conf]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/zope.conf.in
-md5sum = 599e004c18fd3461aa846ae75bd112f2
+md5sum = bc2154161a1d5baddc4ed4dfaaf94fbe
 mode = 640
 
 [template]
@@ -247,7 +238,7 @@ mode = 640
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 26d26c357053af48602b5110120bd085
+md5sum = 700b8729a4d9a8423ab2a99de486cc31
 extra-context =
     key apache_location apache:location
     key aspell_location aspell:location
@@ -292,7 +283,6 @@ extra-context =
     key template_kumofs template-kumofs:rendered
     key template_mariadb template-mariadb:rendered
     key template_memcached template-memcached:rendered
-    key template_sphinx template-sphinx:rendered
     key template_tidstorage template-tidstorage:target
     key template_varnish template-varnish:target
     key template_zope template-zope:target
@@ -320,7 +310,7 @@ mode = 640
 [template-erp5-cluster]
 < = template-jinja2-base
 filename = instance-erp5-cluster.cfg
-md5sum = 3051f3fea39d7409c417dcac26027b56
+md5sum = b4a9688ca7983fac5dd573e66c6e4263
 extra-context =
     key local_bt5_repository local-bt5-repository:list
 
@@ -473,9 +463,10 @@ eggs =
   huBarcode
   qrcode
 
-# Zope 2.12 with patched acquisition
+# Zope
   ZODB3
   Zope2
+# Zope acquisition patch
   Acquisition
 
 # Other Zope 2 packages
@@ -574,7 +565,7 @@ pysvn = 1.7.4nxd006
 
 # CMF 2.3 requries Zope 2.13.
 Products.CMFCalendar = 2.2.2
-Products.CMFCore = 2.2.5
+Products.CMFCore = 2.2.6
 Products.CMFDefault = 2.2.2
 Products.CMFTopic = 2.2.1
 Products.CMFUid = 2.2.1
@@ -583,7 +574,7 @@ Products.CMFUid = 2.2.1
 slapos.cookbook =
 
 # Pinned versions
-Flask = 0.8
+Flask = 0.9
 GitPython = 0.3.2.RC1
 MySQL-python = 1.2.3
 PIL = 1.1.7
@@ -591,68 +582,63 @@ Paste = 1.7.5.1
 PasteDeploy = 1.5.0
 PasteScript = 1.7.5
 Products.CMFActionIcons = 2.1.3
-Products.CMFCalendar = 2.2.2
-Products.CMFCore = 2.2.5
-Products.CMFDefault = 2.2.2
-Products.CMFTopic = 2.2.1
-Products.CMFUid = 2.2.1
 Products.DCWorkflowGraph = 0.4.1
 Products.ExternalEditor = 1.1.0
-Products.GenericSetup = 1.7.1
+Products.GenericSetup = 1.7.2
 Products.MimetypesRegistry = 2.0.3
-Products.PluggableAuthService = 1.7.7
-Products.PluginRegistry = 1.3b1
-Products.TIDStorage = 5.4.7.dev-r45842
+Products.PluggableAuthService = 1.8.0
+Products.PluginRegistry = 1.3
+Products.TIDStorage = 5.4.8
 Products.Zelenium = 1.0.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
 Werkzeug = 0.8.3
-apache-libcloud = 0.9.1
-argparse = 1.1
+apache-libcloud = 0.11.1
+argparse = 1.2.1
 async = 0.6.1
 atomize = 0.1.1
 buildout-versions = 1.7
 cElementTree = 1.0.5-20051216
-chardet = 1.0.1
-coverage = 3.5.1
+chardet = 1.1
+coverage = 3.5.2
 csp-eventlet = 0.7.0
 elementtree = 1.2.7-20070827-preview
 erp5.recipe.cmmiforcei686 = 0.1.3
-erp5.util = 0.4.6
+erp5.util = 0.4.7
 erp5diff = 0.8.1.5
-eventlet = 0.9.16
-feedparser = 5.1.1
+eventlet = 0.9.17
+feedparser = 5.1.2
 five.localsitemanager = 2.0.5
 fpconst = 0.7.2
 gitdb = 0.5.4
-greenlet = 0.3.4
-hexagonit.recipe.cmmi = 1.5.0
-http-parser = 0.7.5
+greenlet = 0.4.0
+hexagonit.recipe.cmmi = 1.6
+http-parser = 0.7.8
 huBarcode = 0.63
 inotifyx = 0.2.0
-ipdb = 0.6.1
-ipython = 0.12
+ipdb = 0.7
+ipython = 0.13
 meld3 = 0.6.8
-netaddr = 0.7.6
+netaddr = 0.7.9
 netifaces = 0.8
 ordereddict = 1.1
-paramiko = 1.7.7.1
+paramiko = 1.7.7.2
 plone.recipe.command = 1.1
 ply = 3.4
 polib = 1.0.0
-psutil = 0.4.1
-python-ldap = 2.4.9
-python-magic = 0.4.2
+psutil = 0.6.1
+python-ldap = 2.4.10
+python-magic = 0.4.3
 python-memcached = 1.47
-restkit = 4.1.2
+restkit = 4.2.0
 rtjp-eventlet = 0.3.2
 setuptools = 0.6c12dev-r88846
-slapos.core = 0.24
-slapos.recipe.build = 0.7
-slapos.recipe.template = 2.4
-slapos.toolbox = 0.23
+slapos.core = 0.28
+slapos.recipe.build = 0.10
+slapos.recipe.template = 2.4.2
+slapos.toolbox = 0.29
 smmap = 0.8.2
-socketpool = 0.3.0
+socketpool = 0.5.2
 supervisor = 3.0a12
 threadframe = 0.2
 timerserver = 2.0.2
@@ -661,3 +647,6 @@ uuid = 1.30
 validictory = 0.8.3
 xml-marshaller = 0.9.7
 xupdate-processor = 0.4
+mr.developer = 1.21
+qrcode = 2.4.1
+requests = 0.13.2

stack/erp5/instance-erp5-cluster.cfg.in

@@ -24,13 +24,6 @@ cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
 
-[request-sphinx]
-<=request-common
-name = Sphinx Search Engine
-software-type = sphinx
-sla-computer_guid = ${slap-parameter:sphinx-computer-guid}
-return = url-sphinx url-sphinx-sql
-
 [request-mariadb]
 <=request-common
 name = MariaDB DataBase
@@ -67,8 +60,6 @@ config =
   memcached-url
   cloudooo-url
   kumofs-url
-  sphinx-url-sphinx
-  sphinx-url-sphinx-sql
   smtp-url
   bt5
   bt5-repository-url
@@ -77,8 +68,6 @@ config-mysql-url = ${request-mariadb:connection-url}
 config-memcached-url = ${request-memcached:connection-url}
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-kumofs-url = ${request-kumofs:connection-url}
-config-sphinx-url-sphinx = ${request-sphinx:connection-url-sphinx}
-config-sphinx-url-sphinx-sql = ${request-sphinx:connection-url-sphinx-sql}
 config-bt5 = ${slap-parameter:bt5}
 config-bt5-repository-url = ${slap-parameter:bt5-repository-url}
 config-smtp-url = ${slap-parameter:smtp-url}
@@ -97,7 +86,6 @@ sla-computer_guid = ${slap-parameter:varnish-computer-guid}
 
 [slap-parameter]
 # Default value if no computer_guid is specified for each type
-sphinx-computer-guid = ${slap-connection:computer-id}
 mariadb-computer-guid = ${slap-connection:computer-id}
 cloudooo-computer-guid = ${slap-connection:computer-id}
 memcached-computer-guid = ${slap-connection:computer-id}

stack/erp5/instance-mariadb.cfg.in

@@ -30,7 +30,7 @@ recipe = slapos.cookbook:generic.mysql
 
 # Options
 user = user
-parallel-test-database-amount = 100
+parallel-test-database-amount = ${slap-parameter:test-database-amount}
 port = 45678
 ip = ${slap-network-information:local-ipv4}
 database = erp5
@@ -178,3 +178,6 @@ recipe = slapos.cookbook:check_port_listening
 path = ${basedirectory:promise}/mariadb
 hostname = ${mariadb-instance:ip}
 port = ${mariadb-instance:port}
+
+[slap-parameter]
+test-database-amount = 100

stack/erp5/instance-sphinx.cfg.in

-[buildout]
-parts =
-  publish-sphinx-url
-  sphinxd-instance
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
-[publish-sphinx-url]
-recipe = slapos.cookbook:publish
-url-sphinx = sphinx://${sphinxd-instance:ip}:${sphinxd-instance:sphinx-port}
-url-sphinx-sql = mysql://${sphinxd-instance:ip}:${sphinxd-instance:sql-port}
-
-[sphinxd-instance]
-recipe = slapos.cookbook:sphinx
-data-directory = ${directory:sphinx-data}
-configuration-file = ${rootdirectory:etc}/sphinx.conf
-searchd-log = ${basedirectory:log}/sphinx-searchd.log
-query-log = ${basedirectory:log}/sphinx-query.log
-pid = ${basedirectory:run}/sphinx-searchd.pid
-ip = ${slap-network-information:local-ipv4}
-sphinx-port = 9312
-sql-port = 9306
-wrapper = ${basedirectory:services}/sphinxd
-sphinx-searchd-binary = {{ sphinx_location }}/bin/searchd
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = ${buildout:directory}/etc
-var = ${buildout:directory}/var
-srv = ${buildout:directory}/srv
-bin = ${buildout:directory}/bin
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = ${rootdirectory:var}/log
-services = ${rootdirectory:etc}/run
-run = ${rootdirectory:var}/run
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-sphinx-data = ${rootdirectory:srv}/sphinx

stack/erp5/instance-tidstorage.cfg.in

@@ -485,7 +485,6 @@ recipe = slapos.cookbook:publish
 [tidstorage]
 recipe = slapos.cookbook:tidstorage
 known-tid-storage-identifier-dict = {{ known_tid_storage_identifier_dict }}
-base-url = http://${zope-admin:ip}:${zope-admin:port}/%s/serialize
 configuration-path = ${directory:etc}/tidstorage.py
 ip = {{ ipv4 }}
 port = 6001

stack/erp5/instance.cfg.in

@@ -197,7 +197,6 @@ memcached = {{ template_memcached }}
 cloudooo = ${dynamic-template-cloudooo:rendered}
 zope = ${dynamic-template-zope:rendered}
 mariadb = {{ template_mariadb }}
-sphinx = {{ template_sphinx }}
 tidstorage = ${dynamic-template-tidstorage:rendered}
 varnish = ${dynamic-template-varnish:rendered}
 

stack/erp5/zope.conf.in

@@ -43,10 +43,11 @@ products {{ instance_products }}
 </product-config>
 
 {% endif -%}
-{% if 'timerserver-interval' in parameter_dict -%}
+{% set timerserver_interval = parameter_dict.get('timerserver-interval', '0') | int -%}
+{% if timerserver_interval -%}
 %import timerserver
 <timer-server>
-  interval {{ parameter_dict['timerserver-interval'] }}
+  interval {{ timerserver_interval }}
 </timer-server>
 
 {% endif -%}

stack/lamp/apache/instance-apache-import.cfg

+[buildout]
+extends = ${template-apache-php:output}
+          ${template-pbsready-export:output}
+
+parts =
+  apache-proxy
+  logrotate
+  logrotate-entry-apache
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  sshkeys-dropbear
+  dropbear-server
+  dropbear-server-pbs-authorized-key
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:sshkeys}/requests/
+keys = $${directory:sshkeys}/keys/
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = $${dropbear-server:wrapper}
+public-key = $${dropbear-server:rsa-keyfile}.pub
+private-key = $${dropbear-server:rsa-keyfile}
+wrapper = $${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = $${slap-network-information:global-ipv6}
+port = 2222
+home = $${directory:ssh}
+wrapper = $${rootdirectory:bin}/raw_sshd
+shell = $${rdiff-backup-server:wrapper}
+rsa-keyfile = $${directory:ssh}/server_key.rsa
+dropbear-binary = ${dropbear:location}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = $${slap-parameter:authorized-key}
+
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:htdocs}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-proxy:error-log} $${apache-proxy:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+services = $${rootdirectory:etc}/run/
+run = $${rootdirectory:var}/run/
+backup = $${rootdirectory:srv}/backup/
+promises = $${rootdirectory:etc}/promise/
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www/
+logrotate-entries = $${rootdirectory:etc}/logrotate.d/
+logrotate-backup = $${basedirectory:backup}/logrotate/
+cronstamps = $${rootdirectory:etc}/cronstamps/
+cron-entries = $${rootdirectory:etc}/cron.d/
+crontabs = $${rootdirectory:etc}/crontabs/
+ssh = $${rootdirectory:etc}/ssh/
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache/

stack/lamp/instance-apache-php.cfg → stack/lamp/apache/instance-apache-php.cfg

@@ -107,7 +107,7 @@ software-url = $${slap-connection:software-release-url}
 software-type = mariadb
 return = url
 sla = computer_guid
-sla-computer_guid = ${slap-connection:computer-id}
+sla-computer_guid = $${slap-connection:computer-id}
 
 [mariadb-urlparse]
 recipe = slapos.cookbook:urlparse

stack/lamp/buildout.cfg

 [buildout]
+
+ignore-existing = true
+
 parts =
   apache-php
   mariadb
@@ -9,13 +12,16 @@ parts =
   eggs
   instance-recipe-egg
   template
+
   template-apache-php
   template-mariadb
-  template-mariadb-pbsready
-  template-mariadb-pbsready-import
-  template-mariadb-pbsready-export
+
+#Contains the importer and exporter recipes for mariadb
+  template-mariadb-import
+  template-mariadb-export
 
 extends =
+  ../resilient/buildout.cfg
   ../../component/mariadb/buildout.cfg
   ../../component/apache/buildout.cfg
   ../../component/apache-php/buildout.cfg
@@ -59,64 +65,60 @@ strip-top-level-dir = true
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 8117f10e814a13c5376af4c01e6546d4
+md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11
 mode = 0644
 
 [template-apache-php]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-php.cfg
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg
 output = ${buildout:directory}/template-apache-php.cfg
-md5sum = 59cee571ea0fcdf2e6c9b3195e258738
+md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966
 mode = 0644
 
 [template-apache-backup]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-backup.cfg
+url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg
 output = ${buildout:directory}/template-apache-backup.cfg
 md5sum = cfb77ac8785e0d125a785f69a5339014
 mode = 0644
 
-[template-backuped]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-backuped.cfg
-output = ${buildout:directory}/template-backuped.cfg
-md5sum = f43d1c6412ea8dc83b75573dc00daf9e
+[template-resilient-lamp]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-resilient.cfg
+rendered = ${buildout:directory}/template-resilient.cfg
+
+context = key templateapache template-apache-php:output
+          key dropbear dropbear:location
+          key buildout buildout:bin-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = 03aafcba5c626a4a1bd180d71007be1e
 mode = 0644
  
+
 [template-mariadb]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb.cfg
+url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg
 output = ${buildout:directory}/template-mariadb.cfg
-md5sum = 767452bc503ff6c1b7af0ebfac590c9f
+md5sum = fa9dc10efbcf61119f4cbab37c741322
 mode = 0644
 
-[template-mariadb-pbsready]
+[template-mariadb-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg
-output = ${buildout:directory}/template-mariadb-pbsready.cfg
-md5sum = d2a580dcd7efdd528be45c5ffadfe760
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg
+output = ${buildout:directory}/template-mariadb-import.cfg
+md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b
 mode = 0644
 
-[template-mariadb-pbsready-import]
+[template-mariadb-export]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg
-output = ${buildout:directory}/template-mariadb-pbsready-import.cfg
-md5sum = 4a96ff02da3898fef7077fa8baec81ac
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg
+output = ${buildout:directory}/template-mariadb-export.cfg
+md5sum = 4b7dec765265b27c8235419b82ca7b02
 mode = 0644
 
-[template-mariadb-pbsready-export]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg
-output = ${buildout:directory}/template-mariadb-pbsready-export.cfg
-md5sum = 11a9e45e8bc590bc11bfdd304b07a4a5
-mode = 0644
-
-[template-pull-backup]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pull-backup.cfg
-output = ${buildout:directory}/template-pull-backup.cfg
-md5sum = 9aab30ba5aa23a37d4b507e7c414be00
-mode = 0644
 
 # Dummy parts in case no application configuration file is needed
 [application-template]
@@ -265,43 +267,45 @@ signature-certificate-list =
   -----END CERTIFICATE-----
 
 [versions]
+# 1.2.4b doesn't download
+MySQL-python = 1.2.3
+
 Jinja2 = 2.6
 Werkzeug = 0.8.3
-apache-libcloud = 0.11.1
+apache-libcloud = 0.11.3
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-meld3 = 0.6.8
+meld3 = 0.6.9
 pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.cookbook = 0.66
-slapos.libnetworkcache = 0.13.2
-slapos.recipe.build = 0.11.2
+slapos.cookbook = 0.68
+slapos.recipe.build = 0.11.5
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.30
+slapos.toolbox = 0.31
 smmap = 0.8.2
 
 # Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
+# slapos.core==0.31.2
+# slapos.toolbox==0.31
 Flask = 0.9
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.68
 PyXML = 0.8.4
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 feedparser = 5.1.2
 
 # Required by:
@@ -309,58 +313,61 @@ feedparser = 5.1.2
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.68
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.core==0.28.5
+# slapos.cookbook==0.68
+# slapos.core==0.31.2
 # xml-marshaller==0.9.7
 lxml = 3.0alpha2
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.68
 netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 netifaces = 0.8
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 paramiko = 1.7.7.2
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 psutil = 0.6.1
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
-# zc.buildout==1.6.0-dev-SlapOS-006
+# slapos.cookbook==0.68
+pytz = 2012f
+
+# Required by:
+# slapos.cookbook==0.68
+# slapos.core==0.31.2
+# slapos.toolbox==0.31
+# zc.buildout==1.6.0-dev-SlapOS-010
 # zc.recipe.egg==1.3.2
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.toolbox==0.30
-slapos.core = 0.28.5
+# slapos.cookbook==0.68
+# slapos.toolbox==0.31
+slapos.core = 0.31.2
 
 # Required by:
-# slapos.core==0.28.5
-supervisor = 3.0a12
+# slapos.core==0.31.2
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.toolbox==0.30
+# slapos.cookbook==0.68
+# slapos.toolbox==0.31
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.68
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 zope.interface = 4.0.1
-

stack/lamp/instance-backuped.cfg

-[buildout]
-
-extends =
-  ${template-apache-php:output}
-
-parts =
-  request-mariadb
-  request-mariadb-pseudo-replicating
-  request-apache-backup
-  request-pull-backup-server
-  request-pull-backup-server-mariadb
-  request-pull-backup-server-mariadb-backup
-  request-pull-backup-server-apache
-  request-pull-backup-server-apache-backup
-  url
-  apache-php
-  stunnel
-  certificate-authority
-  ca-stunnel
-  logrotate
-  logrotate-entry-apache
-  logrotate-entry-stunnel
-  cron
-  cron-entry-logrotate
-  dropbear-server
-  sshkeys-authority
-  dropbear-server-pbs-authorized-key
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-wrapper = $${basedirectory:services}/sshkeys_authority
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${basedirectory:services}/sshd
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-port = 2222
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${request-pull-backup-server:connection-ssh-key}
-
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:htdocs}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-[request-pull-backup-server]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = PBS (Pull Backup Server)
-software-url = $${slap-connection:software-release-url}
-software-type = pull-backup
-return = ssh-key notification-url feeds-url
-slave = false
-
-[request-mariadb]
-software-type = mariadb-pbsready-export
-config = authorized-key notify
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-return = url ssh-public-key ssh-url notification-id
-
-[request-mariadb-pseudo-replicating]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = MariaDB Backup
-software-url = $${slap-connection:software-release-url}
-software-type = mariadb-pbsready-import
-return = url ssh-public-key ssh-url notification-url
-config = authorized-key on-notification
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-pbs-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-push
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${:pbs-notification-id}
-
-[request-apache-backup]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup
-software-url = $${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = $${url:url}
-
-[request-pull-backup-server-apache]
-<= request-pull-backup-server
-   slap-connection
-recipe = slapos.cookbook:request
-name = PBS pulling from Apache
-software-url = $${slap-connection:software-release-url}
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
-config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = $${sshkeys-dropbear:public-key-value}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-
-[request-pull-backup-server-apache-backup]
-<= request-pull-backup-server
-   slap-connection
-recipe = slapos.cookbook:request
-name = PBS pushing to $${request-apache-backup:name}
-software-url = $${slap-connection:software-release-url}
-config = url name type server-key on-notification
-config-url = $${request-apache-backup:connection-ssh-url}
-config-name = $${request-pull-backup-server-apache:config-name}
-config-type = push
-config-server-key = $${request-apache-backup:connection-ssh-public-key}
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-apache:config-notification-id}
-slave = true
-
-[request-pull-backup-server-mariadb]
-<= request-pull-backup-server
-   slap-connection
-name = PBS pulling from $${request-mariadb:name}
-config = url name type server-key on-notification notify notification-id title
-config-url = $${request-mariadb:connection-ssh-url}
-config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
-config-type = pull
-config-server-key = $${request-mariadb:connection-ssh-public-key}
-config-on-notification = $${request-mariadb:connection-notification-id}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-pull
-config-title = Pulling from MariaDB
-slave = true
-
-[request-pull-backup-server-mariadb-backup]
-<= request-pull-backup-server
-   slap-connection
-name = PBS pushing on $${request-mariadb-pseudo-replicating:name}
-config = url name type server-key on-notification notify notification-id title
-config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url}
-config-name = $${request-pull-backup-server-mariadb:config-name}
-config-type = push
-config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key}
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-mariadb:config-notification-id}
-config-notify = $${request-mariadb-pseudo-replicating:connection-notification-url}
-config-notification-id = $${request-mariadb-pseudo-replicating:pbs-notification-id}
-config-title = Pushing to MariaDB backup
-slave = true
-
-[directory]
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys

stack/lamp/instance-mariadb-pbsready-export.cfg

-[buildout]
-extends = ${template-mariadb-pbsready:output}
-
-parts += cron-entry-mariadb-backup
-
-[urls]
-notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-mydumper:name}
-
-[mydumper]
-recipe = slapos.cookbook:mydumper
-wrapper = $${rootdirectory:bin}/raw_mydumper
-backup-directory = $${directory:mariadb-backup}
-socket = $${mariadb:socket}
-user = root
-mydumper-binary = ${mydumper:location}/bin/mydumper
-database = $${mariadb:database}
-import = false
-
-[notifier-mydumper]
-<= notifier
-recipe = slapos.cookbook:notifier.notify
-name = mydumper
-title = Dumping MariaDB Database
-executable = $${mydumper:wrapper}
-wrapper = $${rootdirectory:bin}/mydumper
-notify = $${slap-parameter:notify}
-
-[cron-entry-mariadb-backup]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = backup
-frequency = 0 * * * *
-command = $${notifier-mydumper:wrapper}

stack/lamp/instance-resilient.cfg

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+
+extends =
+   {{templateapache}}
+
+parts =
+  {{ parts.replicate("mariadb","3") }}
+  request-apache-backup-1
+  request-apache-backup-2
+
+  request-pull-backup-server-apache-1
+  request-pull-backup-server-apache-backup-1
+
+  request-pull-backup-server-apache-2
+  request-pull-backup-server-apache-backup-2
+
+  publish-connection-informations
+  apache-php
+  stunnel
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  dropbear-server
+  sshkeys-authority
+  dropbear-server-pbs-authorized-key
+
+  request-pull-backup-server
+
+{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
+
+
+[request-pull-backup-server]
+<= request-pbs-common
+name = PBS (Pull Backup Server)
+return = ssh-key notification-url feeds-url
+slave = false
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = ${directory:sshkeys}/requests/
+keys = ${directory:sshkeys}/keys/
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = ${sshkeys-directory:requests}
+keys-directory = ${sshkeys-directory:keys}
+wrapper = ${basedirectory:services}/sshkeys_authority
+keygen-binary = {{dropbear}}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = ${dropbear-server:wrapper}
+public-key = ${dropbear-server:rsa-keyfile}.pub
+private-key = ${dropbear-server:rsa-keyfile}
+wrapper = ${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = ${slap-network-information:global-ipv6}
+port = 2222
+home = ${directory:ssh}
+wrapper = ${rootdirectory:bin}/raw_sshd
+shell = ${rdiff-backup-server:wrapper}
+rsa-keyfile = ${directory:ssh}/server_key.rsa
+dropbear-binary = {{dropbear}}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = ${request-pull-backup-server:connection-ssh-key}
+
+[rdiff-backup-server]
+<= apache-php
+recipe = slapos.cookbook:pbs
+client = false
+path = ${apache-php:htdocs}
+wrapper = ${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = {{buildout}}/rdiff-backup
+
+[request-apache-backup-1]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 1
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-apache-backup-2]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 2
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-pull-backup-server-apache-1]
+<= request-pbs-common
+name = PBS pulling from Apache 1
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+
+[request-pull-backup-server-apache-2]
+<= request-pbs-common
+name = PBS pulling from Apache 2
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+
+
+[request-pull-backup-server-apache-backup-1]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-1:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-1:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-1:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
+slave = true
+
+[request-pull-backup-server-apache-backup-2]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-2:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-2:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-2:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
+slave = true
+
+
+[directory]
+ssh = ${rootdirectory:etc}/ssh/
+sshkeys = ${rootdirectory:srv}/sshkeys

stack/lamp/instance.cfg

 [buildout]
+
+extends = 
+  ${template-switchsoftware:output}
+
 parts =
   switch_softwaretype
 
@@ -9,9 +13,11 @@ offline = true
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = ${template-apache-php:output}
-backuped = ${template-backuped:output}
+resilient = ${template-resilient-lamp:rendered}
 mariadb = ${template-mariadb:output}
-mariadb-pbsready-import = ${template-mariadb-pbsready-import:output}
-mariadb-pbsready-export = ${template-mariadb-pbsready-export:output}
+mariadb-import = ${template-mariadb-import:output}
+mariadb-export = ${template-mariadb-export:output}
 pull-backup = ${template-pull-backup:output}
 apache-backup = ${template-apache-backup:output}
+frozen = ${template-frozen:output}
+

stack/lamp/mariadb/instance-mariadb-export.cfg

+[buildout]
+extends = ${template-mariadb:output}
+          ${template-pbsready-export:output}
+
+parts += mariadb
+
+[exporter]
+recipe = slapos.cookbook:mydumper
+wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter
+backup-directory = $${directory:backup}
+socket = $${mariadb:socket}
+user = root
+mydumper-binary = ${mydumper:location}/bin/mydumper
+database = $${mariadb:database}
+import = false
\ No newline at end of file

stack/lamp/instance-mariadb-pbsready-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg

 [buildout]
+extends = ${template-mariadb:output}
+          ${template-pbsready-import:output}
 
-extends = ${template-mariadb-pbsready:output}
+parts += mariadb
 
-parts += mariadb-import-on-notification
-
-[urls]
-notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
-
-[mydumper-import]
+[importer]
 recipe = slapos.cookbook:mydumper
 wrapper = $${rootdirectory:bin}/myloader
-backup-directory = $${directory:mariadb-backup}
+backup-directory = $${directory:backup}
 socket = $${mariadb:socket}
 user = root
 myloader-binary = ${mydumper:location}/bin/myloader
 database = $${mariadb:database}
 import = true
-
-[mariadb-import-on-notification]
-<= notifier
-recipe = slapos.cookbook:notifier.callback
-on-notification-id = $${slap-parameter:on-notification}
-callback = $${mydumper-import:wrapper}

stack/lamp/instance-mariadb.cfg → stack/lamp/mariadb/instance-mariadb.cfg

@@ -23,6 +23,7 @@ offline = true
 [urls]
 recipe = slapos.cookbook:publish
 url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
+ip = $${slap-network-information:global-ipv6}
 
 [mariadb]
 recipe = slapos.cookbook:mysql
@@ -52,6 +53,7 @@ mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
 mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
 mysqld-binary = ${mariadb:location}/bin/mysqld
 
+
 [slapmonitor]
 recipe = slapos.cookbook:slapmonitor
 pid-file = $${basedirectory:run}/mariadb.pid
@@ -195,6 +197,7 @@ recipe = slapos.cookbook:mkdirectory
 log = $${rootdirectory:var}/log/
 services = $${rootdirectory:etc}/run/
 run = $${rootdirectory:var}/run/
+script = $${rootdirectory:etc}/script/
 backup = $${rootdirectory:srv}/backup/
 promises = $${rootdirectory:etc}/promise/
 

stack/resilient/buildout.cfg

+[buildout]
+
+parts =
+#Templates needed to setup automatic backup
+  template-pbsready
+  template-pbsready-import
+  template-pbsready-export
+  template-replicated
+  template-parts
+
+#Frozen is the state used to not destroy a broken instance's content
+  template-frozen
+
+  template-resilient
+  template-switchsoftware
+
+
+[template-pbsready]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready.cfg
+output = ${buildout:directory}/template-pbsready.cfg
+md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99
+mode = 0644
+
+[template-pbsready-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready-import.cfg
+output = ${buildout:directory}/template-pbsready-import.cfg
+md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4
+mode = 0644
+
+[template-pbsready-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready-export.cfg
+output = ${buildout:directory}/template-pbsready-export.cfg
+md5sum = 29d36aac2008b173cb9ce5da9e88c0fa
+mode = 0644
+
+[template-pull-backup]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pull-backup.cfg
+output = ${buildout:directory}/template-pull-backup.cfg
+md5sum = f88cc9192a63c88f83a9e5191075534e
+mode = 0644
+
+[template-replicated]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/template-replicated.cfg
+md5sum = 61780842ccda1600a102456b0da69ac9
+mode = 0644
+destination = ${buildout:directory}/template-replicated.cfg
+
+[template-parts]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/template-parts.cfg
+md5sum = f5fc27235725f05fdbde76a78ebc363e
+mode = 0644
+destination = ${buildout:directory}/template-parts.cfg
+
+[template-frozen]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-frozen.cfg
+output = ${buildout:directory}/template-frozen.cfg
+
+[template-resilient]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/resilient.cfg
+output = ${buildout:directory}/resilient.cfg
+md5sum = 59e74d290d623de2c1e147e48f284fba
+mode = 0644
+
+[template-switchsoftware]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/switchsoftware.cfg
+output = ${buildout:directory}/switchsoftware.cfg
+md5sum = c94a0ed85fce2e72254ae956dce7e40d
+mode = 0644
\ No newline at end of file

stack/resilient/instance-frozen.cfg

+[buildout]
+  parts = 
\ No newline at end of file

stack/resilient/instance-pbsready-export.cfg

+[buildout]
+extends = ${template-pbsready:output}
+
+parts += cron-entry-backup
+
+[urls]
+notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
+
+#notify launches executable, and once it's done, notifies the pull-backup-servers.
+[notifier-exporter]
+<= notifier
+recipe = slapos.cookbook:notifier.notify
+name = exporter
+title = Dumping $${slap-parameter:namebase}
+executable = $${exporter:wrapper}
+wrapper = $${rootdirectory:bin}/exporter
+notify = $${slap-parameter:notify}
+
+#adds the exporter to cron
+[cron-entry-backup]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = backup
+frequency = 0 * * * *
+command = $${notifier-exporter:wrapper}

stack/resilient/instance-pbsready-import.cfg

+[buildout]
+
+extends = ${template-pbsready:output}
+
+parts += import-on-notification
+
+[urls]
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+
+#Launches callback, when a notification is received
+[import-on-notification]
+<= notifier
+recipe = slapos.cookbook:notifier.callback
+on-notification-id = $${slap-parameter:on-notification}
+callback = $${importer:wrapper}

stack/lamp/instance-mariadb-pbsready.cfg → stack/resilient/instance-pbsready.cfg

 [buildout]
 
-extends = ${template-mariadb:output}
-
 parts =
+  resiliency
   urls
-  mariadb
   stunnel
   certificate-authority
   ca-stunnel
   logrotate
-  logrotate-entry-mariadb
   logrotate-entry-stunnel
   logrotate-entry-cron
   logrotate-entry-equeue
@@ -21,6 +18,13 @@ parts =
   dropbear-server-pbs-authorized-key
   notifier
 
+# adds the resiliency script for the bully algorithm
+[resiliency]
+recipe = slapos.cookbook:addresiliency
+script = $${basedirectory:script}
+run = $${basedirectory:services}
+
+# sets up an rdiff-backup server (with a dropbear server for ssh)
 [urls]
 ssh-public-key = $${sshkeys-dropbear:public-key-value}
 ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
@@ -65,10 +69,12 @@ key = $${slap-parameter:authorized-key}
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
 client = false
-path = $${directory:mariadb-backup}
+path = $${directory:backup}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
+
+## Sets up the execution queue for the notifier
 [logrotate-entry-equeue]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
@@ -85,6 +91,9 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
+
+## notifier.notify adds the [exporter, notifier] to the execution queue
+## notifier.notify.callback sets up a callback
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -99,12 +108,15 @@ notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
 
 [basedirectory]
+script = $${rootdirectory:etc}/script/
+services = $${rootdirectory:etc}/run/
 cache = $${rootdirectory:var}/cache/
 notifier = $${rootdirectory:etc}/notifier/
 
 [directory]
-mariadb-backup = $${basedirectory:backup}/mariadb/
+backup = $${basedirectory:backup}/$${slap-parameter:namebase}
 ssh = $${rootdirectory:etc}/ssh/
 sshkeys = $${rootdirectory:srv}/sshkeys
 notifier-feeds = $${basedirectory:notifier}/feeds/
 notifier-callbacks = $${basedirectory:notifier}/callbacks/
+script = $${basedirectory:script}

stack/lamp/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg

@@ -19,6 +19,8 @@ ssh-key = $${sshkeys-dropbear:public-key-value}
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
 
+
+## sets up the equeue for the notifier
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
@@ -27,6 +29,7 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
+
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -39,6 +42,8 @@ wrapper = $${basedirectory:services}/notifier
 server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
+
+## Dropbear Client to provide ssh 
 [dropbear-client]
 recipe = slapos.cookbook:dropbear.client
 dbclient-binary = ${dropbear:location}/bin/dbclient
@@ -68,6 +73,9 @@ public-key = $${dropbear-client:identity-file}.pub
 private-key = $${dropbear-client:identity-file}
 wrapper = $${rootdirectory:bin}/do_backup
 
+
+## The pull-backup-server contains every backup (incremental).
+## to prevent a corrupt dump from destroying everything.
 [pbs]
 <= notifier
 recipe = slapos.cookbook:pbs
@@ -82,6 +90,7 @@ wrappers-directory = $${directory:pbs-wrappers}
 notifier-url = http://[$${notifier:host}]:$${notifier:port}/
 slave-instance-list = $${slap-parameter:slave_instance_list}
 
+
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
 wrapper = $${rootdirectory:bin}/cron_simplelogger
@@ -170,3 +179,8 @@ notifier = $${rootdirectory:etc}/notifier/
 
 [rootdirectory]
 home = $${buildout:directory}/home/
+
+
+# Default values
+[slap-parameter]
+slave_instance_list = []

stack/resilient/resilient.cfg

+## not used at the moment
+
+[buildout]
+
+parts =
+  request-pull-backup-server
+
+[request-pull-backup-server]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = PBS (Pull Backup Server)
+software-url = $${slap-connection:software-release-url}
+software-type = pull-backup
+return = ssh-key notification-url feeds-url
+slave = false
\ No newline at end of file

stack/resilient/switchsoftware.cfg

+[buildout]
+parts =
+  switch_softwaretype
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+pull-backup = ${template-pull-backup:output}
+#frozen creates a syntax error, meaning it can keep its data.
+#It's dirty as hell, it needs to be replaced.
+frozen = ${template-frozen:output}

stack/resilient/template-parts.cfg

+## Parts Needed for the resiliency
+
+  {% macro replicate(namebase, nbbackup) %}
+
+  request-{{namebase}}
+  request-{{namebase}}-2
+  
+  {% for i in range(1,nbbackup|int) %}
+  request-{{namebase}}-pseudo-replicating-{{i}}
+  request-{{namebase}}-pseudo-replicating-{{i}}-2
+  {% endfor %}
+  
+  {% for i in range(1,nbbackup|int) %}
+  request-pbs-{{namebase}}-{{i}}
+  request-pull-backup-server-{{namebase}}-{{i}}
+  request-pull-backup-server-{{namebase}}-backup-{{i}}
+  {% endfor %}
+  
+
+
+  {% endmacro %}
\ No newline at end of file

stack/resilient/template-replicated.cfg

+{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='') %}
+
+## Tells the Backupable recipe that we want a backup
+[resilient]
+config-script = bully.py
+config-wrapper = bully
+config-namebase = {{namebase}}
+
+## Every request is double to provide the 3 IPs.
+[request-{{namebase}}]
+<= resilient
+   slap-connection
+   {{heriteLeader}}
+software-type = {{typeexport}}
+name = {{namebase}}0
+return = url ssh-public-key ssh-url notification-id ip
+
+config = number script wrapper authorized-key notify ip-list namebase
+
+config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
+
+config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
+
+config-ip-list =
+config-number = 0
+
+{% for id in range(1,nbbackup|int) %}
+
+[request-{{namebase}}-pseudo-replicating-{{id}}]
+<= slap-connection
+   resilient
+   {{heriteBackup}}
+recipe = slapos.cookbook:request
+name = {{namebase}}{{id}}
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeimport}}
+return = url ssh-public-key ssh-url notification-url ip
+
+config = number script wrapper authorized-key on-notification ip-list namebase
+
+config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+
+config-ip-list =
+config-number = {{id}}
+
+{% endfor %}
+
+[iplist]
+config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
+
+[request-{{namebase}}-2]
+<= resilient
+   slap-connection
+   iplist
+   {{heriteLeader}}
+
+recipe = slapos.cookbook:request
+name = {{namebase}}0
+
+config = number script wrapper authorized-key notify ip-list namebase
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeexport}}
+return = url ssh-public-key ssh-url notification-id ip
+
+
+config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
+config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
+
+
+config-number=0
+
+{% for id in range(1,nbbackup|int) %}
+[request-{{namebase}}-pseudo-replicating-{{id}}-2]
+<= slap-connection
+   resilient
+   iplist
+   {{heriteBackup}}
+
+recipe = slapos.cookbook:request
+name = {{namebase}}{{id}}
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeimport}}
+return = url ssh-public-key ssh-url notification-url
+
+config = number script wrapper authorized-key on-notification ip-list namebase
+
+config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+
+
+config-number = {{id}}
+{% endfor %}
+
+
+
+## The PBS and their push / pull slaves
+## Adding a PBS provides resiliency
+## Adding a backup server provides availability
+
+## Having 3 backups pulling from the same PBS provides
+##only availability, not resiliency
+
+## WARNING : SLAVES ARE ALLOCATED AT RANDOM, THIS NEEDS TO BE FIXED.
+
+[request-pbs-common]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+software-type = pull-backup
+
+{% for id in range(1,nbbackup|int) %}
+
+[request-pbs-{{namebase}}-{{id}}]
+<= request-pbs-common
+name = PBS ({{namebase}} / {{id}})
+return = ssh-key notification-url feeds-url
+slave = false
+
+[request-pull-backup-server-{{namebase}}-{{id}}]
+<= request-pbs-common
+name = PBS {{id}} pulling from ${request-{{namebase}}:name}
+config = url name type server-key on-notification notify notification-id title
+config-url = ${request-{{namebase}}:connection-ssh-url}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
+config-type = pull
+config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
+config-on-notification = ${request-{{namebase}}:connection-notification-id}
+config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
+config-title = Pulling from {{namebase}}
+slave = true
+
+[request-pull-backup-server-{{namebase}}-backup-{{id}}]
+<= request-pbs-common
+name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
+config = url name type server-key on-notification notify notification-id title
+config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
+config-name = ${request-pull-backup-server-{{namebase}}-{{id}}:config-name}
+config-type = push
+config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
+config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-notification-url}
+config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
+config-title = Pushing to {{namebase}} backup {{id}}
+slave = true
+{% endfor %}
+
+{% endmacro %}
\ No newline at end of file

stack/slapos.cfg

@@ -24,6 +24,7 @@ find-links +=
 
 # Use only quite well working sites.
 allow-hosts +=
+  *.googlecode.com
   *.nexedi.org
   *.python.org
   *.sourceforge.net
@@ -34,7 +35,6 @@ allow-hosts +=
   github.com
   launchpad.net
   peak.telecommunity.com
-  psutil.googlecode.com
   sourceforge.net
   www.dabeaz.com
   www.owlfish.com
@@ -59,7 +59,7 @@ eggs =
 
 [versions]
 # Use SlapOS patched zc.buildout
-zc.buildout = 1.6.0-dev-SlapOS-007
+zc.buildout = 1.6.0-dev-SlapOS-010
 
 [networkcache]
 download-cache-url = http://www.shacache.org/shacache