Merge branch 'master' into erp5-component

component/apache-php/buildout.cfg

@@ -15,7 +15,8 @@ extends =
   ../mariadb/buildout.cfg
   ../openldap/buildout.cfg
   ../pkgconfig/buildout.cfg
-  ../zip/buildout.cfg
+  ../zlib/buildout.cfg
+
 
 [apache-php]
 # Note: Shall react on each build of apache and reinstall itself
@@ -29,18 +30,16 @@ configure-options =
   --with-libxml-dir=${libxml2:location}
   --with-mysql=${mariadb:location}
   --with-zlib-dir=${zlib:location}
-  --with-bz2-dir=${bzip2:location}
+  --with-bz2=${bzip2:location}
   --with-mcrypt=${libmcrypt:location}
   --with-gd
   --with-jpeg-dir=${libjpeg:location}
   --with-png-dir=${libpng:location}
   --enable-gd-native-ttf
-  --with-ttf
   --with-freetype-dir=${freetype:location}
   --with-pdo-mysql=mysqlnd
   --with-mysqli=mysqlnd
   --with-curl=${curl:location}
-  --with-zip-dir=${zip:location}
   --with-imap=${cclient:location}
   --with-iconv-dir=${libiconv:location}
   --with-gettext=${gettext:location}
@@ -52,10 +51,8 @@ configure-options =
   --enable-session
   --enable-exif
   --enable-zip
-  --enable-bz2
   --enable-ftp
 
-
 # Changing TMPDIR is required for PEAR installation.
 # It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
 # XXX we could mkdir tmp there
@@ -69,23 +66,25 @@ environment =
 
 
 
-
 [apache-php-postgres]
-<=apache-php
+# Note: Shall react on each build of apache and reinstall itself
+recipe = hexagonit.recipe.cmmi
+depends =
+   ${postgresql:url}
+url = http://fr2.php.net/distributions/php-5.4.12.tar.bz2
+md5sum = 5c7b614242ae12e9cacca21c8ab84818
 configure-options =
   --with-apxs2=${apache:location}/bin/apxs
   --with-libxml-dir=${libxml2:location}
   --with-zlib-dir=${zlib:location}
-  --with-bz2-dir=${bzip2:location}
+  --with-bz2=${bzip2:location}
   --with-mcrypt=${libmcrypt:location}
   --with-gd
   --with-jpeg-dir=${libjpeg:location}
   --with-png-dir=${libpng:location}
   --enable-gd-native-ttf
-  --with-ttf
   --with-freetype-dir=${freetype:location}
   --with-curl=${curl:location}
-  --with-zip-dir=${zip:location}
   --with-imap=${cclient:location}
   --with-iconv-dir=${libiconv:location}
   --with-gettext=${gettext:location}
@@ -97,10 +96,21 @@ configure-options =
   --enable-session
   --enable-exif
   --enable-zip
-  --enable-bz2
   --enable-ftp
   --with-pgsql=${postgresql:location}
 
+# Changing TMPDIR is required for PEAR installation.
+# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
+# XXX we could mkdir tmp there
+
+environment =
+  PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
+  LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid
+  TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
+  HOME=${apache:location}
+
+
 
 [libmcrypt]
 recipe = hexagonit.recipe.cmmi

component/stunnel/buildout.cfg

@@ -17,8 +17,8 @@ filename = stunnel-4-hooks.py
 
 [stunnel-4]
 recipe = hexagonit.recipe.cmmi
-url = ftp://ftp.stunnel.org/stunnel/archive/4.x/stunnel-4.54.tar.gz
-md5sum = c2b1db99e3ed547214568959a8ed18ac
+url = ftp://ftp.stunnel.org/stunnel/archive/4.x/stunnel-4.55.tar.gz
+md5sum = ebe6670368d30c6a01ca33b9c50cb239
 pre-configure-hook = ${stunnel-4-hook-download:location}/${stunnel-4-hook-download:filename}:pre_configure_hook
 configure-options =
   --enable-ipv6

slapos/recipe/apache_frontend/__init__.py

@@ -85,8 +85,20 @@ class Recipe(BaseSlapRecipe):
     domain_dict = {}
 
     for slave_instance in slave_instance_list:
+      # Sanitize inputs
       backend_url = slave_instance.get("url", None)
       reference = slave_instance.get("slave_reference")
+
+      if slave_instance.haskey("enable_cache"):
+        enable_cache = slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE')
+      else:
+        enable_cache = False
+
+      if slave_instance.haskey("type"):
+        slave_type = slave_instance.get("type", "").lower()
+      else:
+        slave_type = None
+
       # Set scheme (http? https?)
       # Future work may allow to choose between http and https (or both?)
       scheme = 'http://'
@@ -120,12 +132,14 @@ class Recipe(BaseSlapRecipe):
       slave_dict[reference] = "%s%s/" % (scheme, domain)
 
       # Check if we want varnish+stunnel cache.
-      if slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE'):
-        # XXX-Cedric : need to refactor to clean code? (to many variables)
-        rewrite_rule = self.configureVarnishSlave(
-            base_varnish_port, backend_url, reference, service_dict, domain)
-        base_varnish_port += 2
-      else:
+      #if enable_cache:
+      #  # XXX-Cedric : need to refactor to clean code? (to many variables)
+      #  rewrite_rule = self.configureVarnishSlave(
+      #      base_varnish_port, backend_url, reference, service_dict, domain)
+      #  base_varnish_port += 2
+      #else:
+      #  rewrite_rule = "%s %s" % (domain, backend_url)
+      # Temporary forbid activation of cache until it is properly tested
       rewrite_rule = "%s %s" % (domain, backend_url)
 
       # Finally, if successful, we add the rewrite rule to our list of rules
@@ -134,7 +148,7 @@ class Recipe(BaseSlapRecipe):
         # rule structure.
         # So we will have one RewriteMap for normal websites, and one
         # RewriteMap for Zope Virtual Host Monster websites.
-        if slave_instance.get("type", "").lower() in ['zope']:
+        if slave_type in ['zope']:
           rewrite_rule_zope_list.append(rewrite_rule)
           # For Zope, we have another dict containing the path e.g '/erp5/...
           rewrite_rule_path = "%s %s" % (domain, slave_instance.get('path', ''))
@@ -529,7 +543,7 @@ class Recipe(BaseSlapRecipe):
     self._createDirectory(mod_ssl_cache_location)
 
     # Create "custom" apache configuration files if it does not exist.
-    # Note : Those files won't be erased or changed when slapgrid is ran.
+    # Note : Those files won't be erased or changed by slapgrid.
     # It can be freely customized by node admin.
     custom_apache_configuration_directory = os.path.join(
         self.data_root_directory, 'apache-conf.d')
@@ -537,12 +551,14 @@ class Recipe(BaseSlapRecipe):
     # First one is included in the end of the apache configuration file
     custom_apache_configuration_file_location = os.path.join(
         custom_apache_configuration_directory, 'apache_frontend.custom.conf')
-    open(custom_apache_configuration_file_location, 'a')
+    if not os.path.exists(custom_apache_configuration_file_location):
+      open(custom_apache_configuration_file_location, 'w')
     # Second one is included in the virtualhost of apache configuration file
     custom_apache_virtual_configuration_file_location = os.path.join(
         custom_apache_configuration_directory,
         'apache_frontend.virtualhost.custom.conf')
-    open(custom_apache_virtual_configuration_file_location, 'a')
+    if not os.path.exists(custom_apache_virtual_configuration_file_location):
+      open(custom_apache_virtual_configuration_file_location, 'w')
 
     # Create backup of custom apache configuration
     backup_path = self.createBackupDirectory('custom_apache_conf_backup')

slapos/recipe/postgres/__init__.py

@@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe):
                 '# TYPE  DATABASE        USER            ADDRESS                 METHOD',
                 '',
                 '# "local" is for Unix domain socket connections only (check unix_socket_permissions!)',
-                'local   all             all                                     ident',
+                'local   all             all                                     trust',
                 'host    all             all             127.0.0.1/32            md5',
                 'host    all             all             ::1/128                 md5',
             ]

slapos/recipe/postgres/backup.py

@@ -63,6 +63,7 @@ class ExportRecipe(GenericBaseRecipe):
                 umask 077
                 %(bin)s/pg_dump \\
                         --host=%(pgdata-directory)s \\
+                        --username postgres \\
                         --format=custom \\
                         --file=%(backup-directory)s/database.dump \\
                         %(dbname)s
@@ -102,6 +103,7 @@ class ImportRecipe(GenericBaseRecipe):
                 #!/bin/sh
                 %(bin)s/pg_restore \\
                         --host=%(pgdata-directory)s \\
+                        --username postgres \\
                         --dbname=%(dbname)s \\
                         --clean \\
                         --no-owner \\

software/apache-frontend/README.apache_frontend.txt

@@ -43,6 +43,25 @@ all slave instances.
 Finally, the slave instance will be accessible from:
 https://someidentifier.moulefrite.org.
 
+
+How to have custom configuration in frontend server
+===================================================
+
+In your instance directory, you, as sysadmin, can directly edit two
+configuration files that won't be overwritten by SlapOS to customize your
+instance:
+
+ * $PARTITION_PATH/srv/srv/apache-conf.d/apache_frontend.custom.conf
+ * $PARTITION_PATH/srv/srv/apache-conf.d/apache_frontend.virtualhost.custom.conf
+
+The first one is included in the end of the main apache configuration file.
+The second one is included in the virtualhost of the main apache configuration file.
+
+SlapOS will jsut create those two files for you, then completely forget them.
+
+Note: make sure that the UNIX user of the instance has read access to those
+files if you edit them.
+
 Instance Parameters
 ===================
 

software/maarch/instance-custom.cfg.in

@@ -3,17 +3,22 @@
 [maarch-configuration]
 recipe = slapos.recipe.maarch:default
 htdocs = $${apache-php:htdocs}
-db_host = $${postgres-urlparse:host}
-db_port = $${postgres-urlparse:port}
-db_dbname = $${postgres-urlparse:path}
-db_username = $${postgres-urlparse:username}
-db_password = $${postgres-urlparse:password}
+db-host = $${postgres-urlparse:host}
+db-port = $${postgres-urlparse:port}
+db-dbname = $${postgres-urlparse:path}
+db-username = $${postgres-urlparse:username}
+db-password = $${postgres-urlparse:password}
 language = en
-php_ini = $${directory:php-ini-dir}/php.ini
-root_docservers = $${buildout:directory}/srv/docservers
+php-ini = $${directory:php-ini-dir}/php.ini
+root-docservers = $${buildout:directory}/srv/docservers
 dependency = $${apache-php:recipe}
+maarch-sql-data-file = $${slap-parameter:maarch-sql-data-file}
 
 [publish-connection-informations]
 # XXX login should not be hardcoded
 login = superadmin
-password = $${maarch-configuration:db_password}
+password = $${maarch-configuration:db-password}
+
+[slap-parameter]
+maarch-sql-data-file =
+

software/maarch/software.cfg

@@ -5,11 +5,6 @@ extends =
   ../../stack/lapp/buildout.cfg
 
 
-develop =
-  ${:parts-directory}/slapos.cookbook-repository
-  ${:parts-directory}/slapos.recipe.maarch-repository
-
-
 # += since we need rdiff-backup and friends
 parts +=
   apache-php-postgres
@@ -17,27 +12,19 @@ parts +=
   eggs
   instance
   instance-apache-php
-  slapos.recipe.maarch-repository
-  check-recipe
-
-
-[versions]
-slapos.recipe.maarch = 
-
+  slapos-recipe-maarch-egg
 
-[slapos.recipe.maarch-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.recipe.maarch.git
-branch = master
-git-executable = ${git:location}/bin/git
 
 
-[check-recipe]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command =
-  grep parts ${buildout:develop-eggs-directory}/slapos.recipe.maarch.egg-link
+#----------------
+#--
+#-- Explicitly provide the configuration egg,
+#-- otherwise instance.cfg won't be able to use it.
+#--
+[slapos-recipe-maarch-egg]
+recipe = zc.recipe.egg
+eggs =
+    slapos.recipe.maarch
 
 
 #----------------
@@ -63,7 +50,7 @@ part-list = maarch-configuration
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-custom.cfg.in
 output = ${buildout:directory}/instance-custom.cfg
-md5sum = d1b02a8c571886a225f2f1c8da06286c
+md5sum = f86b311d443156e327a1b5f5acfb22b1
 mode = 0644
 
 
@@ -83,3 +70,6 @@ install_cmd =
 
 #----------------
 
+[versions]
+slapos.recipe.maarch = 0.4
+

stack/erp5/buildout.cfg

@@ -141,7 +141,7 @@ parts =
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = erp5-component
+branch = erp5
 git-executable = ${git:location}/bin/git
 
 [check-recipe]
@@ -346,7 +346,7 @@ repository_id_list = erp5
 [erp5]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/erp5.git
-branch = erp5-component
+branch = master
 git-executable = ${git:location}/bin/git
 
 [fix-products-paths]

stack/lamp/apache/instance-apache-export.cfg.jinja2

@@ -29,9 +29,13 @@ parts +=
 
 {{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
 
-# Nothing to do for the exporter. Just dummy part that does nothing.
-# For httpd instance, PBS will directly pull data from srv/www.
-# XXX-Cedric: write a real backup system.
+# Nothing to do for the exporter. This wrapper is intended
+# to produce the "dump" files that have to be backed up.
+# So, in case of binary DB data, we would back up the ASCII dump
+# files, or a similar data format that can be restored on any machine
+# (i.e. postgres 'custom')
+# In the case of an httpd instance, the src/www directory can be directly
+# pulled from the PBS, we don't need to prepare anything.
 [exporter]
 wrapper = /bin/true
 

stack/lapp/apache/instance-apache-backup.cfg.in

-[buildout]
-
-parts =
-  urls
-  apache-proxy
-  logrotate
-  logrotate-entry-apache
-  cron
-  cron-entry-logrotate
-  sshkeys-authority
-  sshkeys-dropbear
-  dropbear-server
-  dropbear-server-pbs-authorized-key
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-
-#----------------
-#--
-#-- Creation of all needed directories.
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-var = $${buildout:directory}/var
-srv = $${buildout:directory}/srv
-bin = $${buildout:directory}/bin
-tmp = $${buildout:directory}/tmp
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log
-services = $${rootdirectory:etc}/service
-run = $${rootdirectory:var}/run
-backup = $${rootdirectory:srv}/backup
-promises = $${rootdirectory:etc}/promise
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-htdocs = $${rootdirectory:srv}/www
-logrotate-entries = $${rootdirectory:etc}/logrotate.d
-logrotate-backup = $${basedirectory:backup}/logrotate
-cronstamps = $${rootdirectory:etc}/cronstamps
-cron-entries = $${rootdirectory:etc}/cron.d
-crontabs = $${rootdirectory:etc}/crontabs
-ssh = $${rootdirectory:etc}/ssh
-sshkeys = $${rootdirectory:srv}/sshkeys
-httpd-log = $${basedirectory:log}/apache
-
-
-#----------------
-#--
-#-- Deploy cron.
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-
-#----------------
-#--
-#-- Deploy logrotate.
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
-
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
-
-
-#----------------
-#--
-#-- sshkeys
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests
-keys = $${directory:sshkeys}/keys
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-wrapper = $${basedirectory:services}/sshkeys_authority
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${basedirectory:services}/sshd
-
-
-#----------------
-#--
-#-- Dropbear.
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-port = 2222
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${slap-parameter:authorized-key}
-
-
-#----------------
-#--
-#-- rdiff
-
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:htdocs}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-
-#----------------
-#--
-#-- Apache Proxy.
-
-[apache-proxy]
-recipe = slapos.cookbook:apacheproxy
-url = $${slap-parameter:proxy-url}
-pid-file = $${basedirectory:run}/apache.pid
-lock-file = $${basedirectory:run}/apache.lock
-ip = $${slap-network-information:global-ipv6}
-port = 8080
-error-log = $${directory:httpd-log}/error.log
-access-log = $${directory:httpd-log}/access.log
-httpd-conf = $${rootdirectory:etc}/apache.conf
-wrapper = $${basedirectory:services}/apache
-
-promise = $${basedirectory:promises}/apache
-
-httpd-binary = ${apache:location}/bin/httpd
-
-[logrotate-entry-apache]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = apache
-log = $${apache-proxy:error-log} $${apache-proxy:access-log}
-frequency = daily
-rotate-num = 30
-sharedscripts = true
-notifempty = true
-create = true
-
-
-#----------------
-#--
-#-- Publish instance parameters.
-
-[urls]
-recipe = slapos.cookbook:publish
-url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
-ssh-public-key = $${sshkeys-dropbear:public-key-value}
-ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
-

stack/lapp/apache/instance-apache-export.cfg.jinja2

+# This file is responsible of three things:
+# 1/ Act as "Apache exporter"
+# 2/ Act as "Postgres backup infrastructure requester"
+# 3/ Act as "Apache" instance
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+extends = {{templateapache}}
+          {{templatepbsreadyexport}}
+
+parts +=
+    {{ parts.replicate("postgres", "3") }}
+
+# Repeating parts from instance-apache-php.
+# XXX-Cedric: how to simplify this?
+    certificate-authority
+    ca-stunnel
+    logrotate
+    logrotate-entry-apache
+    logrotate-entry-stunnel
+    cron
+    cron-entry-logrotate
+    promise
+    frontend-promise
+    content-promise
+    publish-connection-informations
+
+{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }}
+
+# Nothing to do for the exporter. This wrapper is intended
+# to produce the "dump" files that have to be backed up.
+# So, in case of binary DB data, we would back up the ASCII dump
+# files, or a similar data format that can be restored on any machine
+# (i.e. postgres 'custom')
+# In the case of an httpd instance, the src/www directory can be directly
+# pulled from the PBS, we don't need to prepare anything.
+[exporter]
+wrapper = /bin/true
+
+# State that we want to backup srv/www directory, not srv/backup.
+# XXX-Cedric: works well, but doesn't work with big data.
+[rdiff-backup-server]
+path = ${directory:www}
+
+# Add "exporter" parameters to list of published connection parameters
+[publish-connection-informations]
+# XXX-Cedric: Long term goal: could be a recipe that requests an instance and
+# bubbles ALL
+# parameters of the requested instance. Requirement: aggregated publish.
+<= resilient-publish-connection-parameter
+
+# XXX-Cedric: resilient overwrites what's returned from request-postgres
+# XXX-Cedric: change the request method to return everything from
+#             getConnectionParameterDict()
+[request-postgres]
+return = ssh-public-key ssh-url notification-id ip url

stack/lapp/apache/instance-apache-import.cfg.in

+[buildout]
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+extends = ${pbsready-import:output}
+
+[directory]
+srv = $${buildout:directory}/srv
+www = $${:srv}/www/
+
+# Nothing to do for the import. Just dummy part that does nothing.
+# For httpd instance, PBS will directly push data to srv/www.
+# XXX-Cedric: write a real backup system.
+[importer]
+wrapper = /bin/true
+
+# State that we want to push to srv/www directory, not srv/backup.
+[rdiff-backup-server]
+path = $${directory:www}

stack/lapp/apache/instance-apache-php.cfg.in

@@ -57,6 +57,7 @@ logrotate-backup = $${basedirectory:backup}/logrotate
 report = $${rootdirectory:etc}/report
 stunnel-conf = $${rootdirectory:etc}/stunnel
 xml-report = $${rootdirectory:var}/xml_report
+www = $${rootdirectory:srv}/www/
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -195,7 +196,7 @@ url = $${request-postgres:connection-url}
 recipe = slapos.cookbook:apachephp
 source = ${application:location}
 
-htdocs = $${rootdirectory:srv}/www/
+htdocs = $${directory:www}
 pid-file = $${basedirectory:run}/apache.pid
 lock-file = $${basedirectory:run}/apache.lock
 ip = $${slap-network-information:global-ipv6}

stack/lapp/buildout.cfg

 [buildout]
+
+ignore-existing = true
+
 parts =
   slapos-cookbook
   apache-php-postgres
@@ -15,11 +18,14 @@ parts =
   instance-postgres-import
   instance-postgres-export
 
+#Contains the importer and exporter recipes for apache
+  instance-apache-import
+  instance-apache-export
+
 
 extends =
-  ../resilient/buildout.cfg
-  ../../component/apache/buildout.cfg
   ../../component/apache-php/buildout.cfg
+  ../../component/apache/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dcron/buildout.cfg
   ../../component/gzip/buildout.cfg
@@ -29,6 +35,7 @@ extends =
   ../../component/rdiff-backup/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/dropbear/buildout.cfg
+  ../resilient/buildout.cfg
   ../slapos.cfg
 
 
@@ -48,21 +55,35 @@ strip-top-level-dir = true
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
-md5sum = a482fa0e72839b4bd75b169ac1460d64
+md5sum = 25d07b5101d5f566398686642ada4cee
 mode = 0644
 
 [instance-apache-php]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
 output = ${buildout:directory}/instance-apache-php.cfg
-md5sum = 72b70452d1c077cfcd0f268181506b8e
+md5sum = 823257dda6f3068a38c6b69c771cf307
 mode = 0644
 
-[instance-apache-backup]
+[instance-apache-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
-output = ${buildout:directory}/instance-apache-backup.cfg
-md5sum = db879141c0b6a77ef8b3b7e699f5583a
+url = ${:_profile_base_location_}/apache/instance-apache-import.cfg.in
+output = ${buildout:directory}/instance-apache-import.cfg
+md5sum = f1dc2a71d362b5d2d36481ffefdd2293
+mode = 0644
+
+[instance-apache-export]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2
+rendered = ${buildout:directory}/instance-apache-export.cfg
+
+context = key templateapache instance-apache-php:output
+          key templatepbsreadyexport pbsready-export:output
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = bdc7e126567ece6bf93a9bb493e29fac
 mode = 0644
 
 [instance-resilient]
@@ -70,17 +91,16 @@ recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
 rendered = ${buildout:directory}/instance-resilient.cfg
 
-context = key templateapache instance-apache-php:output
-          key dropbear dropbear:location
-          key buildout buildout:bin-directory
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
 
 import-list = file parts template-parts:destination
               file replicated template-replicated:destination
 
-md5sum = 46c7d8f691bd37d84e0bd03b83e51d14
+md5sum = ef38aa9810ce20960382261f235abfcd
 mode = 0644
 
-
 [instance-postgres]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/postgres/instance-postgres.cfg.in
@@ -99,7 +119,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/postgres/instance-postgres-export.cfg.in
 output = ${buildout:directory}/instance-postgres-export.cfg
-md5sum = 68080e5d861eb3474442211dd65c668b
+md5sum = d95205a5fc2825e9709ed6db295111e2
 mode = 0644
 
 
@@ -155,7 +175,6 @@ eggs =
   ${psycopg2:egg}
   slapos.toolbox
 
-
 [networkcache]
 # Romain Courteaud + Sebastien Robin + Alain Takoudjou
 # + Cedric de Saint Martin signature certificate

stack/lapp/instance-resilient.cfg.jinja2

@@ -4,172 +4,47 @@
 {% import 'replicated' as replicated %}
 
 [buildout]
-
-extends =
-   {{templateapache}}
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
 
 # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
 parts +=
-  {{ parts.replicate("postgres","3") }}
-  request-apache-backup-1
-  request-apache-backup-2
-
-  request-pull-backup-server-apache-1
-  request-pull-backup-server-apache-backup-1
-
-  request-pull-backup-server-apache-2
-  request-pull-backup-server-apache-backup-2
-
+  {{ parts.replicate("apache", "3") }}
   publish-connection-informations
-  apache-php
-  stunnel
-  certificate-authority
-  ca-stunnel
-  logrotate
-  logrotate-entry-apache
-  logrotate-entry-stunnel
-  cron
-  cron-entry-logrotate
-  dropbear-server
-  sshkeys-authority
-  dropbear-server-pbs-authorized-key
-
-  request-pull-backup-server
-
-{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }}
-
-
-[request-pull-backup-server]
-<= request-pbs-common
-name = PBS (Pull Backup Server)
-return = ssh-key notification-url feeds-url
-slave = false
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = ${directory:sshkeys}/requests
-keys = ${directory:sshkeys}/keys
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = ${sshkeys-directory:requests}
-keys-directory = ${sshkeys-directory:keys}
-wrapper = ${basedirectory:services}/sshkeys_authority
-keygen-binary = {{dropbear}}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = ${dropbear-server:wrapper}
-public-key = ${dropbear-server:rsa-keyfile}.pub
-private-key = ${dropbear-server:rsa-keyfile}
-wrapper = ${basedirectory:services}/sshd
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = ${slap-network-information:global-ipv6}
-port = 2222
-home = ${directory:ssh}
-wrapper = ${rootdirectory:bin}/raw_sshd
-shell = ${rdiff-backup-server:wrapper}
-rsa-keyfile = ${directory:ssh}/server_key.rsa
-dropbear-binary = {{dropbear}}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = ${request-pull-backup-server:connection-ssh-key}
-
-[rdiff-backup-server]
-<= apache-php
-recipe = slapos.cookbook:pbs
-client = false
-path = ${apache-php:htdocs}
-wrapper = ${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = {{buildout}}/rdiff-backup
-
-[request-apache-backup-1]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup 1
-software-url = ${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = ${publish-connection-informations:url}
-
-[request-apache-backup-2]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup 2
-software-url = ${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = ${publish-connection-informations:url}
-
-[request-pull-backup-server-apache-1]
-<= request-pbs-common
-name = PBS pulling from Apache 1
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
-config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = ${sshkeys-dropbear:public-key-value}
-config-notify = ${request-pull-backup-server:connection-notification-url}
-config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-[request-pull-backup-server-apache-2]
-<= request-pbs-common
-name = PBS pulling from Apache 2
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
-config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = ${sshkeys-dropbear:public-key-value}
-config-notify = ${request-pull-backup-server:connection-notification-url}
-config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-
-[request-pull-backup-server-apache-backup-1]
-<= request-pbs-common
-name = PBS pushing to ${request-apache-backup-1:name}
-config = url name type server-key on-notification
-config-url = ${request-apache-backup-1:connection-ssh-url}
-config-name = ${request-pull-backup-server-apache-1:config-name}
-config-type = push
-config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
-config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-[request-pull-backup-server-apache-backup-2]
-<= request-pbs-common
-name = PBS pushing to ${request-apache-backup-2:name}
-config = url name type server-key on-notification
-config-url = ${request-apache-backup-2:connection-ssh-url}
-config-name = ${request-pull-backup-server-apache-2:config-name}
-config-type = push
-config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
-config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
 
-[directory]
-ssh = ${rootdirectory:etc}/ssh
-sshkeys = ${rootdirectory:srv}/sshkeys
+{{ replicated.replicate("apache", "3", "apache-export", "apache-import") }}
+
+# Bubble up the parameters
+[request-apache]
+return = url ssh-public-key ssh-url notification-id ip url backend_url
+# XXX: hardcoded values
+config = domain number authorized-key notify ip-list namebase postgres1-computer-guid pbs-postgres1-computer-guid postgres2-computer-guid pbs-postgres2-computer-guid postgres3-computer-guid pbs-postgres3-computer-guid maarch-sql-data-file
+config-postgres1-computer-guid = ${slap-parameter:postgres1-computer-guid}
+config-pbs-postgres1-computer-guid = ${slap-parameter:pbs-postgres1-computer-guid}
+config-postgres2-computer-guid = ${slap-parameter:postgres2-computer-guid}
+config-pbs-postgres2-computer-guid = ${slap-parameter:pbs-postgres2-computer-guid}
+config-postgres3-computer-guid = ${slap-parameter:postgres3-computer-guid}
+config-pbs-postgres3-computer-guid = ${slap-parameter:pbs-postgres3-computer-guid}
+config-domain = ${slap-parameter:domain}
+config-maarch-sql-data-file = ${slap-parameter:maarch-sql-data-file}
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = ${request-apache:connection-backend_url}
+url = ${request-apache:connection-url}
+
+[slap-parameter]
+# Default parameters for distributed deployment
+# I.e state "backup1 of postgres should go there, ..."
+# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
+postgres1-computer-guid =
+pbs-postgres1-computer-guid =
+postgres2-computer-guid =
+pbs-postgres2-computer-guid =
+postgres3-computer-guid =
+pbs-postgres3-computer-guid =
+# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
+domain =
+# the following parameters are specific to maarch
+maarch-sql-data-file = 

stack/lapp/instance.cfg.in

@@ -14,7 +14,8 @@ resilient = ${instance-resilient:rendered}
 postgres = ${instance-postgres:output}
 postgres-import = ${instance-postgres-import:output}
 postgres-export = ${instance-postgres-export:output}
-apache-backup = ${instance-apache-backup:output}
+apache-import = ${instance-apache-import:output}
+apache-export = ${instance-apache-export:rendered}
 
 #frozen creates a syntax error, meaning it can keep its data.
 #It's dirty as hell, it needs to be replaced.

stack/lapp/postgres/instance-postgres-export.cfg.in

@@ -19,3 +19,8 @@ bin = $${postgres-instance:bin}
 pgdata-directory = $${postgres-instance:pgdata-directory}
 backup-directory = $${postgres-instance:backup-directory}
 dbname = $${postgres-instance:dbname}
+
+
+# Extends publish section with resilient parameters
+[urls]
+<= resilient-publish-connection-parameter