Merge branch 'erp5-cluster-z2.13-scalability-N' into erp5-cluster-z2.13-scalability

component/findutils/buildout.cfg

+[buildout]
+parts =
+  findutils
+
+[findutils]
+recipe = slapos.recipe.cmmi
+url = http://ftp.gnu.org/gnu/findutils/findutils-4.4.2.tar.gz
+md5sum = 351cc4adb07d54877fa15f75fb77d39f

setup.py

@@ -108,6 +108,7 @@ setup(name=name,
           'erp5.update = slapos.recipe.erp5_update:Recipe',
           'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
           'erp5testnode = slapos.recipe.erp5testnode:Recipe',
+          'scalability = slapos.recipe.scalability:Recipe',
           'firefox = slapos.recipe.firefox:Recipe',
           'fontconfig = slapos.recipe.fontconfig:Recipe',
           'generate.mac = slapos.recipe.generatemac:Recipe',

slapos/recipe/apache_zope_backend/template/vhost.in

 Listen %(ip)s:%(port)s
 <VirtualHost *:%(port)s>
   %(ssl_enable)s
-  RewriteRule (.*) %(backend)s%(backend-path)s$1 [L,P]
+  RewriteRule (.*) %(backend)s%(backend-path)s//VirtualHostBase/https/%(ip)s:%(port)s/VirtualHostRoot/$1 [L,P]
 </VirtualHost>

slapos/recipe/erp5_bootstrap/__init__.py

@@ -46,14 +46,62 @@ class Recipe(GenericBaseRecipe):
       password=parsed.password
     )
 
+    # Init zope configuration
+    zope_username = None
+    zope_password = None    
+    zope_hostname = None
+    zope_port = None
+    zope_protocol = None
+
+    # Get informations from zope url
+    if self.options.get('zope-url'):
       zope_parsed = urlparse.urlparse(self.options['zope-url'])
+    # Zope hostname
+    if self.options.get('zope-hostname'):
+      zope_hostname = self.options['zope-hostname']
+    elif self.options.get('zope-url'):
+      zope_hostname = zope_parsed.hostname
+    else:
+      zope_hostname = 'localhost'
+    # Zope port
+    if self.options.get('zope-port'):
+      zope_port = self.options['zope-port']
+    elif self.options.get('zope-url'):
+      zope_port = zope_parsed.port
+    else:
+      zope_port = 8080
+    # Zope username and password
+    if self.options.get('zope-username') and self.options.get('zope-password'):
+      zope_username = self.options['zope-username']
+      zope_password = self.options['zope-password']
+    elif self.options.get('zope-url'):
+      zope_username = zope_parsed.username
+      zope_password = zope_parsed.password
+    else:
+      zope_username = 'zope'
+      zope_password = 'insecure'
+    # Zope protocol
+    if self.options.get('zope-protocol'):
+      zope_protocol = self.options['zope-protocol']
+    elif self.options.get('zope-url'):
+      zope_protocol = zope_parsed.scheme      
+    else:
+      zope_protocol = 'http'
+    # Zope site-id
+    if self.options.get('zope-site-id'):
+      zope_site_id = self.options['zope-site-id']
+    elif self.options.get('zope-url'):
+      zope_site_id = zope_parsed.path.split('/')[1],
+    else:
+      zope_site_id = 'erp5'
 
     config = dict(
       python_path=sys.executable,
-      user=zope_parsed.username,
-      password=zope_parsed.password,
-      site_id=zope_parsed.path.split('/')[1],
-      host="%s:%s" % (zope_parsed.hostname, zope_parsed.port),
+      user=zope_username,
+      password=zope_password,
+      site_id=zope_site_id,
+      host="%s:%s" % (zope_hostname, zope_port),
+      protocol=zope_protocol,
       sql_connection_string=mysql_connection_string,
     )
 

slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in

@@ -10,10 +10,17 @@ host = "%(host)s"
 site_id = "%(site_id)s"
 erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
 mysql_url = "%(sql_connection_string)s"
+protocol = "%(protocol)s"
 
 header_dict = {'Authorization': 'Basic %%s' %% \
   base64.encodestring('%%s:%%s' %% (user, password)).strip()}
-zope_connection = httplib.HTTPConnection(host)
+
+if protocol == 'https':
+  zope_connection = httplib.HTTPSConnection(host)
+elif protocol == 'http':
+  zope_connection = httplib.HTTPConnection(host)  
+else:
+  raise ValueError("Protocol not implemented")
   
 # Check if an ERP5 site is already created, as ERP5 does support having
 # 2 instances in the same zope, and this script should not destroy user data
@@ -23,7 +30,12 @@ result = zope_connection.getresponse()
 if result.status == 204: # and (result.read() == "False"):
 
   # Use a new connection
+  if protocol == 'https':
+    zope_connection = httplib.HTTPSConnection(host)
+  elif protocol == 'http':
     zope_connection = httplib.HTTPConnection(host)  
+  else:
+    raise ValueError("Protocol pot implemented") 
 
   # Create the expected ERP5 instance
   zope_connection.request(

slapos/recipe/scalability/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import os
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+class Recipe(GenericBaseRecipe):
+  def install(self):
+    path_list = []
+    configuration_file = self.createFile(
+      self.options['configuration-file'],
+      '',
+    )
+    path_list.append(configuration_file)
+    path_list.append(
+      self.createPythonScript(
+        self.options['wrapper'],
+        'slapos.recipe.librecipe.execute.executee',
+        [ # Executable
+          [ self.options['exec-path'],
+            '--erp5-url', self.options['erp5-url'],
+            '--test-result-path', self.options['test-result-path'],
+            '--test-suite-master-url', self.options['test-suite-master-url'],
+            '--node-title', self.options['node-title'],
+            '--revision', self.options['revision'],
+            '--log-path', self.options['log-path'],
+          ],
+          # Environment
+          {
+            'GIT_SSL_NO_VERIFY': '1',
+          }
+        ],
+      )
+    )
+    return path_list

slapos/recipe/tidstorage/__init__.py

@@ -43,7 +43,11 @@ class Recipe(GenericBaseRecipe):
     repozo_wrapper = self.createPythonScript(
       self.options['repozo-wrapper'],
       'slapos.recipe.librecipe.execute.execute',
-      [self.options['tidstorage-repozo-binary'], '--config',
-        configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
+      [self.options['tidstorage-repozo-binary'],
+        '--config', configuration_file,
+        '--repozo', self.options['repozo-binary'],
+        '--gzip',
+        '--quick',
+      ])
 
     return [configuration_file, tidstorage_wrapper, repozo_wrapper]

software/erp5/README.txt

@@ -14,70 +14,113 @@ Parameters are expected to be passed as of *.serialised recipes expect them::
 
 where `...` is a json expression (typically a dict).
 
+TCPv4 ports allocation
+----------------------
+Service listening ports are allocated in the following pattern.
+Base port of each software can be overridden, those are the default values.
+- kumofs (persistent)
+  2000: manager
+  2001: server port (?)
+  2002: server listen port (?)
+  2003: gateway port (?)
+- kumofs (volatile)
+  2010: manager
+  2011: server port (?)
+  2012: server listen port (?)
+  2013: gateway port (?)
+- cloudooo
+  2020: cloudooo
+  2021: openoffice
+- mariadb
+  2099: mariadb
+- zeo & tidstorage
+  2100: tidstorage
+  2101: first zeo
+  2102: second zeo
+  (etc)
+- haproxy
+  2150: first haproxy
+  2151: first apache
+  2152: second haproxy
+  2153: second apache
+  (etc)
+- cluster-zope
+  2200: first zope
+  2201: second zope
+  (etc)
+Note: these are not applicable when (yet unsupported) ipv6 mode is enabled, as
+stunnel (used to tunnel ipv4-only services over ipv6) needs its own listening
+ports.
+This pattern was chosen to make it possible to deploy this software release in
+a setup where all partitions would share the same IPv4 without having to
+provide many parameters.
+Some ports are unused in the overall range to allow software types to grow.
+
 common
 ------
 Parameters which are available to all software types.
 
-'test-database-amount' (int, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Number of test databases to generate in addition of requested databases.
-A test database, if it were provided as a database-list entry, would look like:
-  {'name': 'erp5_test_0', 'user': 'testuser_0', 'password': 'testpassword0'}
-with '0' being all numbers from 0 to test-database-amount - 1.
-Defaults to 0.
-
-'mariadb-binlogs' (dict, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Enable writing data-modifying requests to binlogs. Useful to restore data
-further after latest available full backup.
-Defaults to {'expire-days': 0} (binlogs are disabled).
+'mariadb-dict' (dict, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+MariaDB (used for catalog, activity tables and id generator) instance
+parameters.
+Defaults to {}.
 Possible keys and associated value types:
-'expire-days' (int, optional)
-  The number of days binlogs will be kept.
-  If 0, binlogs are disabled.
-  If -1, binlogs are never expired.
-  Defaults to 7.
-
+'tcpv4-port' (int, optional)
+  TCPv4 port to listen on.
+  Defaults to 2099.
 'database-list' (list, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Define the list of databases mariadb must provide, and the user having entire
-access to each database. Each entry in the list is a dict, with these possible
-keys and associated value types:
-'name' (str, mandatory)
+  Define the list of databases mariadb must provide, and the user having entire
+  access to each database. Each entry in the list is a dict, with these
+  possible keys and associated value types:
+  'name' (str, mandatory)
     Database name
-'user' (str, mandatory)
+  'user' (str, mandatory)
     User login
-'password' (str, mandatory)
+  'password' (str, mandatory)
     User password
-Defaults to: [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]
-
+  Defaults to: [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]
+'test-database-amount' (int, optional)
+  Number of test databases to generate in addition of requested databases.
+  A test database, if it were provided as a database-list entry, would look
+  like:
+    {'name': 'erp5_test_0', 'user': 'testuser_0', 'password': 'testpassword0'}
+  with '0' being all numbers from 0 to test-database-amount - 1.
+  Defaults to 30.
+  Note: the default is way too much for "normal" usage. You are encouraged to
+  provide this key to some lower value: 0 if you don't intend to run any unit
+  test (ex: production instance) 3..5 if you intend to run tests without
+  paralellism.
+'full-retention-days' (int, optional)
+  The number of days full backups will be kept.
+  If -1, backups are disabled.
+  If 0, backup never expire.
+  Defaults to 7.
+'incremental-retention-days' (int, optional)
+  The number of days binlogs will be kept.
+  If -1, binlogs are disabled.
+  If 0, binlogs are never expired.
+  Defaults to 'full-retention-days' value.
 'innodb-buffer-pool-size' (str, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-See mariadb documentation for innodb_buffer_pool_size configuration parameter.
-Value is used verbatim in configuration file.
-Empty string means unconfigured (ie, bail out to mariadb's default).
-Defaults to "".
-
+  See mariadb documentation for innodb_buffer_pool_size configuration
+  parameter. Value is used verbatim in configuration file.
+  Empty string means unconfigured (ie, bail out to mariadb's default).
+  Defaults to "".
 'innodb-log-file-size' (str, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-See mariadb documentation for innodb_log_file_size configuration parameter.
-Value is used verbatim in configuration file.
-Empty string means unconfigured (ie, bail out to mariadb's default).
-Defaults to "".
-
+  See mariadb documentation for innodb_log_file_size configuration parameter.
+  Value is used verbatim in configuration file.
+  Empty string means unconfigured (ie, bail out to mariadb's default).
+  Defaults to "".
 'innodb-log-buffer-size' (str, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-See mariadb documentation for innodb_log_buffer_size configuration parameter.
-Value is used verbatim in configuration file.
-Empty string means unconfigured (ie, bail out to mariadb's default).
-Defaults to "".
-
+  See mariadb documentation for innodb_log_buffer_size configuration parameter.
+  Value is used verbatim in configuration file.
+  Empty string means unconfigured (ie, bail out to mariadb's default).
+  Defaults to "".
 'mariadb-relaxed-writes' (int, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Controls relaxed writes, which improves performances at the cost of data
-safety. DO NOT ENABLE THIS ON PRODUCTION. It's fine for unit tests and may be
-acceptable for development instances. Set to 1 to enable.
-Default: 0
+  Controls relaxed writes, which improves performances at the cost of data
+  safety. DO NOT ENABLE THIS ON PRODUCTION. It's fine for unit tests and may be
+  acceptable for development instances. Set to 1 to enable.
+  Default: 0
 
 single (default)
 ----------------
@@ -124,6 +167,12 @@ Storage mechanism to use. To know the list of supported values, see all keys in
 instance.cfg.in's section [switch-softwaretype] which start with "zodb-".
 Defaults to 'zeo'.
 
+'zodb-tcpv4-port' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Base TCPv4 port for ZODB provider, if applicable (depends on chosen
+software-type).
+Defaults to 2100.
+
 'zodb-dict' (dict, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Describes ZODBs to create and use in the instance. At least one entry is
@@ -218,7 +267,7 @@ value (dict)
     Start allocating ports at this value. Useful if one needs to make several
     partitions share the same port range (ie, several partitions bound to a
     single address).
-    Defaults to 2000.
+    Defaults to 2200.
 
 'haproxy-maxconn' (int, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -226,6 +275,11 @@ The number of connections haproxy accepts for a given backend.
 See haproxy's "server maxconn" setting.
 Defaults to 1 (correct for single-worker-threaded zopes).
 
+'haproxy-tcpv4-port' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Base TCPv4 port for load-balancer (haproxy + backend apache).
+Defaults to 2150.
+
 'haproxy-server-check-path' (str, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 The path haproxy accesses on a each backend to test their responsiveness.
@@ -268,26 +322,42 @@ Possible keys and associated value types:
 'company' (str, optional)
   Company name. Defaults to 'Dummy Company'.
 
-'use-ipv6' (boolean, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Use IPv6 to communicate between partitions.
-Defaults to False.
-
 'mariadb-computer-guid' (str, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Computer GUID identifying the partition mariadb is to be requested on.
 Defaults to "cluster" software type's partition's effective computer GUID.
 
+'cloudooo-tcpv4-port' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Base TCPv4 port for cloudooo.
+Defaults to 2020.
+
 'cloudooo-computer-guid' (str, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Computer GUID identifying the partition cloudooo is to be requested on.
 Defaults to "cluster" software type's partition's effective computer GUID.
 
+'memcached-size' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Megabytes of ram to allocate for volatile memcached use.
+Defaults to 64.
+Negative/zero values cause undefined behaviour which may change in the future.
+
+'memcached-tcpv4-port'
+~~~~~~~~~~~~~~~~~~~~~~
+Base TCPv4 port for volatile memcached.
+Defaults to 2010.
+
 'memcached-computer-guid' (str, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Computer GUID identifying the partition memcached is to be requested on.
 Defaults to "cluster" software type's partition's effective computer GUID.
 
+'kumofs-tcpv4-port'
+~~~~~~~~~~~~~~~~~~~
+Base TCPv4 port for persistent memcached.
+Defaults to 2000.
+
 'kumofs-computer-guid' (str, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Computer GUID identifying the partition kumofs is to be requested on.
@@ -304,10 +374,10 @@ Computer GUID identifying the partition balander (haproxy, apache, some HTTP
 cache) is to be requested on.
 Defaults to "cluster" software type's partition's effective computer GUID.
 
-'cloudooo-json' (str, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-XXX: what is this ?
-XXX: should not require serialising json by hand as parameter value
+'font-url-list' (list of strings, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+List of extra fonts URLs to be used by cloudooo.
+Defaults to [].
 
 'bt5' (str, optional)
 ~~~~~~~~~~~~~~~~~~~~~

software/erp5/instance-erp5-input-schema.json

+{
+  "definitions": {
+    "tcpv4port": {
+      "minimum": 1,
+      "maximum": 65535,
+      "type": "integer"
+    }
+  },
+
+  "description": "Parameters to instanciate ERP5",
+  "type": "object",
+  "properties": {
+    "site-id": {
+      "description": "ERP5Site object's id",
+      "default": "erp5",
+      "type": "string"
+    },
+    "timezone": {
+      "description": "Main site's timezone",
+      "default": "UTC",
+      "type": "string"
+    },
+    "zope-partition-dict": {
+      "description": "Zope layout definition",
+      "patternProperties": {
+        ".*": {
+          "properties": {
+            "family": {
+              "description": "The family this partition is part of. For example: 'public', 'admin', 'backoffice', 'web-service'... Each family gets its own frontend (=client-facing ip & port). It has no special meaning for the system.",
+              "default": "default",
+              "type": "string",
+            },
+            "instance-count": {
+              "description": "Number of Zopes to setup on this partition",
+              "default": 1,
+              "type": "integer"
+            },
+            "thread-amount": {
+              "description": "Number of worker threads for each created Zope process",
+              "default": 1,
+              "type": "integer"
+            },
+            "timerserver-interval": {
+              "description": "Timerserver tick perdiod, in seconds, or 0 to disable",
+              "default": 5,
+              "type": "integer" // float actually ?
+            },
+            "longrequest-logger-interval": {
+              "description": "Period, in seconds, with which LongRequestLogger polls worker thread stack traces, or -1 to disable",
+              "default": -1,
+              "type": "integer"
+            },
+            "longrequest-logger-timeout": {
+              "description": "Transaction duration after which LongRequestLogger will start logging its stack trace, in seconds",
+              "default": 1,
+              "type": "integer"
+            },
+            "port-base": {
+              "allOf": [{
+                "$ref": "#/definitions/tcpv4port"
+              }, {
+                "description": "Start allocating ports at this value. Useful if one needs to make several partitions share the same port range (ie, several partitions bound to a single address)",
+                "default": 2200
+              }]
+            },
+            "computer-guid": {
+              "description": "GUID of computer instance must be requested on",
+              "type": "string"
+            },
+          },
+          "type": "object"
+        },
+      },
+      "type": "object"
+    },
+// Deprecated, to move to a user-triggered tool dedicated to initialising an ERP5 site
+//  "bt5-repository-url": {},
+//  "bt5": {},
+//  "smtp-url": {},
+// Not ready yet.
+//   "use-ipv6": {
+//     "description": "Use IPv6 between partitions",
+//     "default": false,
+//     "type": "boolean"
+//   },
+    "frontend": {
+      "properties": {
+        "software-url": {
+          "description": "Software URL of an existing frontend",
+          "type": "string",
+        },
+        // XXX: instance-guid should be replaced by something which allows
+        // replacing frontend partition without having to change related
+        // instances' parameters.
+        "instance-guid": {
+          "description": "Frontend's instance GUID",
+          "type": "string"
+        },
+        "software-type": {
+          "description": "Frontend software type in given software relase",
+          "default": "RootSoftwareInsance",
+          "type": "string"
+        },
+        "domain": {
+          "description": "Domain name belonging to this instance",
+          "default": "",
+          "type": "string",
+        },
+      },
+      "default": {},
+      "type": "object"
+    },
+
+    "kumofs-computer-guid": {
+      "description": "GUID of computer instance must be requested on",
+      "type": "string"
+    },
+    "kumofs": {
+      "description": "Persistent memcached service",
+      "additionalProperties": {
+        "$ref": "./instance-kumofs-input-schema.json",
+      },
+      "type": "object"
+    },
+    "memcached-computer-guid": {
+      "description": "GUID of computer instance must be requested on",
+      "type": "string"
+    },
+    "memcached": {
+      "description": "Volatile memcached service",
+      "additionalProperties": {
+        "$ref": "./instance-kumofs-input-schema.json",
+      },
+      "type": "object"
+    },
+    "cloudooo-computer-guid": {
+      "description": "GUID of computer instance must be requested on",
+      "type": "string"
+    },
+    "cloudooo": {
+      "description": "Format conversion service",
+      "additionalProperties": {
+        "$ref": "./instance-cloudooo-input-schema.json",
+      },
+      "type": "object"
+    },
+    "mariadb-computer-guid": {
+      "description": "GUID of computer instance must be requested on",
+      "type": "string"
+    },
+    "mariadb": {
+      "description": "Relational database service",
+      "additionalProperties": {
+        "$ref": "./instance-mariadb-input-schema.json",
+      },
+      "type": "object"
+    },
+    "zodb-software-type": {
+      "description": "Zope Object DataBase implementation",
+      "default": "zeo",
+      "enum": ["zeo"],
+      "type": "string"
+    },
+    "zodb": {
+      "description": "Zope Object DataBase service",
+      "oneOf": [
+        {"$ref": "./instance-zeo-input-schema.json"},
+      ]
+    },
+  },
+  "$schema": "http://json-schema.org/draft-04/schema#"
+}

stack/erp5/buildout.cfg

@@ -68,6 +68,7 @@ extends =
   ../../component/cloudooo/buildout.cfg
   ../../component/jsl/buildout.cfg
   ../../component/6tunnel/buildout.cfg
+  ../../component/findutils/buildout.cfg
 
 parts =
   rdiff-backup
@@ -76,6 +77,7 @@ parts =
   apache
   apache-antiloris
   file
+  findutils
   graphviz
   haproxy
   jsl
@@ -142,7 +144,6 @@ parts =
   erp5.util
   slapos.cookbook
 
-  slapos.recipe.template
 # Create instance template
   template
 
@@ -152,7 +153,7 @@ parts =
 
 [erp5.dev-repository]
 recipe = slapos.recipe.build:gitclone
-repository = http://192.168.242.75:1235/erp5.git
+repository = http://192.168.242.72:1235/erp5.git
 branch = scalability-master2
 git-executable = ${git:location}/bin/git
 
@@ -177,8 +178,9 @@ mode = 644
 # Local development
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.git
-branch = erp5-cluster-z2.13-scalability
+#repository = http://git.erp5.org/repos/slapos.git
+repository = http://192.168.242.72:1236/slapos.git
+branch = erp5-cluster-z2.13-scalability-N
 git-executable = ${git:location}/bin/git
 
 [check-recipe]
@@ -187,16 +189,13 @@ stop-on-error = true
 update-command = ${:command}
 command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 
+
 [slapos.cookbook]
 recipe = zc.recipe.egg
 eggs = slapos.cookbook
 scripts =
 ugly-depend-on = ${slapos.cookbook-repository:repository} ${slapos.cookbook-repository:branch}
 
-[slapos.recipe.template]
-recipe = zc.recipe.egg
-eggs = slapos.recipe.template
-scripts =
 
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
@@ -214,7 +213,7 @@ context =
 [template-mariadb]
 < = download-base
 filename = instance-mariadb.cfg.in
-md5sum = 2108e8ac8bd214a8a29bb9fc6a480241
+md5sum = 60f5e662d7301e3d56c51f934ea4a303
 
 [template-zope]
 < = download-base
@@ -224,12 +223,12 @@ md5sum = 2b68521ebde309d6453336f1a2e46381
 [template-kumofs]
 < = download-base
 filename = instance-kumofs.cfg.in
-md5sum = 75aae1e3d6836dab1af2f7c00690869e
+md5sum = 0dd51fb85ced7e77d4a8544115f90f8e
 
 [template-cloudooo]
 < = download-base
 filename = instance-cloudoo.cfg.in
-md5sum = 3bb277c9181210753fa3a09901cffc61
+md5sum = 6660382993d14e1d538e737e881219ba
 
 [template-zope-conf]
 < = download-base
@@ -251,7 +250,7 @@ md5sum = 564006953b7d7a12d40a14b6648b32f0
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = e47a41e88f816a1d66664e76a21e3b75
+md5sum = 16fc8b6907dcff36e2c47f11a0f2cd46
 extra-context =
     key apache_location apache:location
     key aspell_location aspell:location
@@ -263,6 +262,7 @@ extra-context =
     key dmtx_utils_location dmtx-utils:location
     key erp5_location erp5:location
     key file_location file:location
+    key findutils_location findutils:location
     key fontconfig_location fontconfig:location
     key fonts_location fonts:location
     key freetype_location freetype:location
@@ -319,22 +319,22 @@ extra-context =
 [template-erp5-single]
 < = download-base
 filename = instance-erp5-single.cfg.in
-md5sum = 69961d8b60ce529baaf958dc76537ac5
+md5sum = 77ec4f095a22b5bd7dfea9dff63bade5
 
 [template-erp5-cluster]
 < = download-base
 filename = instance-erp5-cluster.cfg.in
-md5sum = d4d95f1749e1d9d3701565f934fcd4dc
+md5sum = c745d794b28cae64feba527f894d7340
 
 [template-scalability]
 < = download-base
 filename = instance-scalability.cfg.in
-md5sum = fbbfece0272b819cda88895b6cb7258a
+md5sum = cd676f9ad7c9cbb63a71404b699feb38
 
 [template-zeo]
 < = download-base
 filename = instance-zeo.cfg.in
-md5sum = 9d49e6787600a416545a8ff228abb720
+md5sum = 6d0bdee21ac4837f07852b98b6fcea36
 
 [template-cluster-zope]
 < = download-base
@@ -344,7 +344,7 @@ md5sum = 019f306860857e8536f26aba32101b30
 [template-balancer]
 < = download-base
 filename = instance-balancer.cfg.in
-md5sum = 896cfa69cf2529a3b64b7d783f73a447
+md5sum = 9c9741c4db7e46f3cd13c0b2f392fd5d
 
 [template-logrotate-base]
 < = template-jinja2-base
@@ -501,6 +501,7 @@ eggs =
   ${python-ldap-python:egg}
   ${pysvn-python:egg}
   ${pycrypto-python:egg}
+  lock_file
   PyXML
   SOAPpy
   cElementTree
@@ -596,7 +597,7 @@ scripts =
   web_checker_utility = erp5.util.webchecker:web_checker_utility
 
 extra-paths =
-#  ${erp5:location}
+  ${erp5:location}
   ${erp5.dev-repository:location}
 
 [zodbanalyze]

stack/erp5/instance-balancer.cfg.in

@@ -23,7 +23,7 @@ ipv4 = {{ ipv4 }}
 {% endif -%}
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
-{% set next_port = 9000 -%}
+{% set next_port = slapparameter_dict['tcpv4-port'] -%}
 {% for family_name, parameter_id_list in slapparameter_dict['zope-family-dict'].items() -%}
 {%   set zope_family_address_list = [] -%}
 {%   for parameter_id in parameter_id_list -%}

stack/erp5/instance-cloudoo.cfg.in

 {% if software_type == slap_software_type -%}
-{% set json = json_module.loads(parameter_dict.get('cloudooo-json', '{}')) -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 [buildout]
@@ -26,8 +25,9 @@ recipe = slapos.cookbook:generic.cloudooo
 
 # Network options
 ip = ${slap-network-information:local-ipv4}
-port = 23000
-openoffice-port = 23060
+{% set tcpv4_port = slapparameter_dict['tcpv4-port'] -%}
+port = {{ tcpv4_port }}
+openoffice-port = {{ tcpv4_port + 1 }}
 
 # Paths
 configuration-file = ${directory:etc}/cloudooo.cfg
@@ -51,7 +51,7 @@ recipe = slapos.cookbook:fontconfig
 conf-path = ${directory:etc}/font.conf
 font-system-folder = {{ parameter_dict['fonts'] }}
 font-folder = ${directory:font}
-url-list = {{ json.get('font_url_list', []) | join(' ') }}
+url-list = {{ slapparameter_dict.get('font-url-list', []) | join(' ') }}
 service-folder = ${directory:service}
 onetimedownload_path = {{ bin_directory }}/onetimedownload
 

stack/erp5/instance-erp5-cluster.cfg.in

 {% if slap_software_type == software_type -%}
-{% set has_frontend = slapparameter_dict.get('frontend-software-url', '') != '' -%}
-{% set port_base = 2000 -%}
+{% set publish_dict = {} -%}
+{% set frontend_dict = slapparameter_dict.get('frontend', {}) %}
+{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
 {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
 [request-common]
 recipe = slapos.cookbook:request.serialised
 software-url = ${slap-connection:software-release-url}
 sla = computer_guid
-return = url
 server-url = ${slap-connection:server-url}
 key-file = ${slap-connection:key-file}
 cert-file = ${slap-connection:cert-file}
@@ -18,45 +18,34 @@ config =
 extra-config =
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 
-[request-mariadb]
-<=request-common
-name = MariaDB DataBase
-software-type = mariadb
-sla-computer_guid = {{ slapparameter_dict.get('mariadb-computer-guid', computer_id) }}
-{% set config_list = [] -%}
-{% for option in ('test-database-amount', 'mariadb-binlogs', 'database-list', 'innodb-buffer-size', 'innodb-log-file-size', 'innodb-log-buffer-size', 'mariadb-relaxed-writes') -%}
-{%   if option in slapparameter_dict -%}
-{%     do config_list.append(option) -%}
-config-{{ option }} = {{ dumps(slapparameter_dict[option]) }}
+{% macro request(name, software_type, config_key, config={}, ret={'url': True}) -%}
+{% do config.update(slapparameter_dict.get(config_key, {})) -%}
+{% set section = 'request-' ~ name -%}
+[{{ section }}]
+< = request-common
+name = {{ name }}
+software-type = {{ software_type }}
+return = {{ ret.keys() | join(' ') }}
+{% for ret, publish in ret.items() -%}
+{%   if publish -%}
+{%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%}
 {%   endif -%}
 {% endfor -%}
-extra-config = {{ config_list | join(' ') }}
-
-[request-cloudooo]
-<=request-common
-name = Cloudooo
-extra-config = cloudooo-json
-config-cloudooo-json = {{ slapparameter_dict.get('cloudooo-json', '') }}
-software-type = cloudooo
-sla-computer_guid = {{ slapparameter_dict.get('cloudooo-computer-guid', computer_id) }}
-
-[request-memcached]
-<=request-common
-name = KumoFS-ram
-software-type = kumofs
-sla-computer_guid = {{ slapparameter_dict.get('memcached-computer-guid', computer_id) }}
-extra-config =
-  ram-storage-size
-config-ram-storage-size = 64m
+sla-computer_guid = {{ slapparameter_dict.get(config_key + '-computer-guid', computer_id) }}
+extra-config = {{ ' '.join(config) }}
+{% for option, value in config.items() -%}
+config-{{ option }} = {{ dumps(value) }}
+{% endfor -%}
+{% endmacro -%}
 
-[request-kumofs]
-<=request-common
-name = KumoFS
-software-type = kumofs
-sla-computer_guid = {{ slapparameter_dict.get('kumofs-computer-guid', computer_id) }}
+{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}) }}
+{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }}
+{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
+{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}) }}
+{{ request('zodb', 'zodb-' ~ slapparameter_dict.get('zodb-software-type', 'zeo'), 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
 
 [request-zope-base]
-<=request-common
+< = request-common
 return =
   zope-address-list
 extra-config =
@@ -80,9 +69,9 @@ extra-config =
   longrequest-logger-timeout
   port-base
 config-mysql-url = ${request-mariadb:connection-url}
-config-memcached-url = ${request-memcached:connection-url}
+config-memcached-url = ${request-memcached-volatile:connection-url}
 config-cloudooo-url = ${request-cloudooo:connection-url}
-config-kumofs-url = ${request-kumofs:connection-url}
+config-kumofs-url = ${request-memcached-persistent:connection-url}
 config-bt5 = {{ slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc') }}
 config-bt5-repository-url = {{ slapparameter_dict.get('bt5-repository-url', local_bt5_repository) }}
 config-smtp-url = {{ slapparameter_dict.get('smtp-url', 'smtp://localhost:25/') }}
@@ -90,26 +79,10 @@ config-zodb-dict = ${request-zodb:connection-zodb-dict}
 config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
 config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
 config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-config-timezone = {{ slapparameter_dict.get('timezone', 'Europe/Paris') }}
+config-timezone = {{ slapparameter_dict.get('timezone', 'UTC') }}
 config-site-id = {{ site_id }}
 software-type = cluster-zope
 
-[request-zodb]
-< = request-common
-name = zodb
-return =
-  zodb-storage-type
-  zodb-dict
-  tidstorage-ip
-  tidstorage-port
-extra-config =
-  zodb-dict
-  tidstorage-dict
-config-zodb-dict = {{ dumps(slapparameter_dict.get('zodb-dict', {'root': {}})) }}
-config-tidstorage-dict = {{ dumps(slapparameter_dict.get('tidstorage-dict')) }}
-software-type = zodb-{{ slapparameter_dict.get('zodb-software-type', 'zeo') }}
-sla-computer_guid = {{ slapparameter_dict.get('zodb-computer-guid', computer_id) }}
-
 {% set zope_family_dict = {} -%}
 {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}
 {%   set base_name = 'zope-' ~ custom_name -%}
@@ -123,14 +96,13 @@ config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 1)) }}
 config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
-config-port-base = {{ dumps(zope_parameter_dict.get('port-base', port_base)) }}
+config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 sla-computer_guid = {{ zope_parameter_dict.get('computer-guid', computer_id) }}
 {% endfor -%}
 
 {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#}
 {% set zope_address_list_id_dict = {} -%}
 {% set zope_family_parameter_dict = {} -%}
-{% set publish_dict = {} -%}
 {% for family_name, zope_section_id_list in zope_family_dict.items() -%}
 {%   for zope_section_id in zope_section_id_list -%}
 {%     set parameter_name = 'zope-family-entry-' ~ zope_section_id -%}
@@ -147,7 +119,7 @@ config-url = {{ backend_url }}
 {%   else -%}
 {%     set publishable = 'request-balancer:connection-' ~ family_name -%}
 {%   endif -%}
-{%   do publish_dict.__setitem__(family_name, publishable) -%}
+{%   do publish_dict.__setitem__('family-' ~ family_name, '${' ~ publishable ~ '}' ) -%}
 {% endfor -%}
 
 [request-balancer]
@@ -156,6 +128,7 @@ name = balancer
 software-type = balancer
 sla-computer_guid = {{ slapparameter_dict.get('balancer-computer-guid', computer_id) }}
 extra-config =
+  tcpv4-port
   haproxy-maxconn
   haproxy-server-check-path
   apache-access-control-string
@@ -167,6 +140,7 @@ extra-config =
 return =
   {{ zope_family_dict.keys() | join(' ') }}
 config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
+config-tcpv4-port = {{ dumps(slapparameter_dict.get('', 2150)) }}
 {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
 config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}
 {% endfor -%}
@@ -181,16 +155,16 @@ config-ca = {{ dumps(slapparameter_dict.get('ca', {})) }}
 [request-frontend-base]
 {% if has_frontend -%}
 < = request-common
-software-url = {{ slapparameter_dict['frontend-software-url'] }}
-software-type = {{ slapparameter_dict.get('frontend-software-type', 'RootSoftwareInstance') }}
+software-url = {{ frontend_dict['software-url'] }}
+software-type = {{ frontend_dict.get('software-type', 'RootSoftwareInstance') }}
 sla = instance_guid
-sla-instance_guid = {{ slapparameter_dict['frontend-instance-guid'] }}
+sla-instance_guid = {{ frontend_dict['instance-guid'] }}
 slave = true
 {% set config_dict = {
   'type': 'zope',
 } -%}
-{%   if slapparameter_dict.get('frontend-domain') -%}
-{%     do config_dict.__setitem__('custom_domain', slapparameter_dict['frontend-domain']) -%}
+{%   if frontend_dict.get('domain') -%}
+{%     do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
 {%   endif -%}
 extra-config = url {{ config_dict.keys() | join(' ') }}
 {%   for name, value in config_dict.items() -%}
@@ -201,8 +175,8 @@ return = site_url
 
 [publish]
 recipe = slapos.cookbook:publish.serialised
-{% for family_name, publishable in publish_dict.items() -%}
-{{   family_name }} = {{ '${' ~ publishable ~ '}' }}
+{% for name, value in publish_dict.items() -%}
+{{   name }} = {{ value }}
 {% endfor -%}
 
 [buildout]

stack/erp5/instance-erp5-single.cfg.in

@@ -103,15 +103,11 @@ partition-id = ${slap-connection:partition-id}
 name = MariaDB DataBase
 config = test-database-amount mariadb-binlogs database-list innodb-buffer-size innodb-log-file-size innodb-log-buffer-size mariadb-relaxed-writes
 software-type = mariadb
-{% set config_list = ('mariadb-binlogs', 'database-list', 'innodb-buffer-size', 'innodb-log-file-size', 'innodb-log-buffer-size', 'mariadb-relaxed-writes') -%}
-config = test-database-amount {{ config_list | join(' ') }}
-# BBB: 100 is too much, but kept for backward compatibility.
-config-test-database-amount = {{ dumps(slapparameter_dict.get('test-database-amount', 100)) }}
-{% for option in config_list -%}
-{%   if option in slapparameter_dict -%}
-config-{{ option }} = {{ dumps(slapparameter_dict[option]) }}
-{%   endif -%}
+{% set mariadb_dict = slapparameter_dict.get('mariadb-dict', {}) -%}
+{% for option, value in mariadb_dict.items() -%}
+config-{{ option }} = {{ dumps(value) }}
 {% endfor -%}
+config = {{ ' '.join(mariadb_dict) }}
 
 [request-cloudooo]
 <=request-common

stack/erp5/instance-kumofs.cfg.in

@@ -36,15 +36,15 @@ address-family = inet6
 ip = ${slap-network-information:local-ipv4}
 address-family = inet4
 {% endif -%}
-manager-port = 13101
-server-port = 13201
-server-listen-port = 13202
-gateway-port = 13301
+{% set tcpv4_port = slapparameter_dict['tcpv4-port'] -%}
+manager-port = {{ tcpv4_port }}
+server-port = {{ tcpv4_port + 1 }}
+server-listen-port = {{ tcpv4_port + 2 }}
+gateway-port = {{ tcpv4_port + 3 }}
 # Paths: Data
 {% set ram_storage_size = slapparameter_dict.get('ram-storage-size') -%}
 {% if ram_storage_size -%}
-data-path = *#capsiz=${:storage-size}
-storage-size = {{ ram_storage_size }}
+data-path = *#capsiz={{ ram_storage_size }}m
 {% else -%}
 data-directory = ${directory:kumofs-data}
 {% endif -%}
@@ -69,8 +69,7 @@ shell-path = {{ parameter_dict['dash-location'] }}/bin/dash
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
 name = kumofs
-log = ${kumofs-instance:kumo-gateway-log} ${kumofs-instance:kumo-manager-log}
-  ${kumofs-instance:kumo-server-log}
+log = ${kumofs-instance:kumo-gateway-log} ${kumofs-instance:kumo-manager-log} ${kumofs-instance:kumo-server-log}
 
 # rest of parts are candidates for some generic stuff
 

stack/erp5/instance-mariadb.cfg.in

 {% if software_type == slap_software_type -%}
+{% set part_list = [] -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 {% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
 {% for database_count in range(slapparameter_dict.get('test-database-amount', 0)) -%}
 {%   do database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}
 {% endfor -%}
-{% set binlog_parameter_dict = slapparameter_dict.get('mariadb-binlogs', {'expire-days': 0}) -%}
-{% set binlog_expire_days = slapparameter_dict.get('expire-days', 7) %}
-[buildout]
-parts =
-  publish-mariadb-url
-  logrotate
-  logrotate-entry-mariadb
-  cron
-  cron-entry-logrotate
-  cron-entry-mariadb-backup
-  binary-link
-  update-mysql
-  mysqld
-  promise
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
+{% set catalog_backup = slapparameter_dict.get('catalog-backup', {}) -%}
+{% set full_backup_retention_days = catalog_backup.get('full-retention-days', 7) -%}
+{% set incremental_backup_retention_days = catalog_backup.get('incremental-retention-days', full_backup_retention_days) -%}
 
 [publish-mariadb-url]
 recipe = slapos.cookbook:publish.serialised
@@ -34,7 +21,8 @@ recipe = slapos.cookbook:publish.serialised
 {% set database = database_list[0] %}
 url = mysql://{{ database['user'] }}:{{ database['password'] }}@{{ address }}:${my-cnf-parameters:port}/{{ database['name'] }}
 
-[cron-entry-mariadb-backup]
+{% if full_backup_retention_days > -1 -%}
+[{{ section('cron-entry-mariadb-backup') }}]
 <= cron
 recipe = slapos.cookbook:cron.d
 name = mariadb-backup
@@ -45,8 +33,20 @@ frequency = 0 22 * * *
 #   can be fully restored.
 # master-data: use value "2" as we are not in a replication case
 #}
-command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases {% if binlog_expire_days %}--flush-logs --master-data=2 {% endif %}| gzip > "${directory:mariadb-backup-full}/$(date "+%Y%m%d%H%M%S").sql.gz"
-# TODO: expire full backups
+command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
+{# KEEP GLOB PATTERN IN SYNC with generated filenames above
+#           YYYYmmddHHMMSS -#}
+file-glob = ??????????????.sql.gz
+
+{% if full_backup_retention_days > 0 -%}
+[{{ section("cron-entry-mariadb-backup-expire") }}]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = mariadb-backup-expire
+frequency = 0 22 * * *
+command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +{{ full_backup_retention_days }} -delete
+{%- endif %}
+{%- endif %}
 
 [my-cnf-parameters]
 {% if use_ipv6 -%}
@@ -54,7 +54,7 @@ ip = {{ (ipv6_set | list)[0] }}
 {% else -%}
 ip = {{ (ipv4_set | list)[0] }}
 {% endif -%}
-port = 45678
+port = {{ slapparameter_dict['tcpv4-port'] }}
 socket = ${directory:run}/mariadb.sock
 data-directory = ${directory:mariadb-data}
 pid-file = ${directory:run}/mariadb.pid
@@ -64,10 +64,10 @@ innodb-buffer-size = {{ slapparameter_dict.get('innodb-buffer-size', '') }}
 innodb-log-file-size = {{ slapparameter_dict.get('innodb-log-file-size', '') }}
 innodb-log-buffer-size = {{ slapparameter_dict.get('innodb-log-buffer-size', '') }}
 relaxed_writes = {{ dumps(slapparameter_dict.get('mariadb-relaxed-writes', 0)) }}
-{% if binlog_expire_days -%}
+{% if incremental_backup_retention_days > -1 -%}
 binlog-path = ${directory:mariadb-backup-incremental}/binlog
 # XXX: binlog rotation happens along with other log's rotation
-binlog-expire-days = {{ binlog_expire_days }}
+binlog-expire-days = {{ incremental_backup_retention_days }}
 {% endif -%}
 
 [my-cnf]
@@ -217,4 +217,23 @@ recipe = slapos.cookbook:check_port_listening
 path = ${directory:promise}/mariadb
 hostname = ${my-cnf-parameters:ip}
 port = ${my-cnf-parameters:port}
+
+[buildout]
+parts =
+  publish-mariadb-url
+  logrotate
+  logrotate-entry-mariadb
+  cron
+  cron-entry-logrotate
+  cron-entry-mariadb-backup
+  cron-entry-mariadb-backup-expire
+  binary-link
+  update-mysql
+  mysqld
+  promise
+  {{ part_list | join('\n  ') }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
 {% endif %}

stack/erp5/instance-scalability.cfg.in

 {% if slap_software_type == software_type -%}
+{% set site_id = slapparameter_dict.get('cluster').get('site-id', 'erp5') -%}
+{% set promise_path = slapparameter_dict.get('promise-path', 'erp5') -%}
 [buildout]
+extends = 
 parts =
   erp5-cluster
   binary-wrap-launcher
+  scalability-launcher-instance
+  binary-wrap-performance-tester
+  erp5-bootstrap
+  erp5-promise
+  promise-erp5-site
+  publish
+
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
@@ -20,38 +30,95 @@ name = ERP5 Cluster
 software-url = ${slap-connection:software-release-url}
 software-type = cluster
 sla-computer_guid = {{ slapparameter_dict.get('launcher-computer-guid', computer_id) }}
-config =
-  use-ipv6
-  ${:extra-config}
-extra-config =
-config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
-
+{# Add a zope scalability family to the erp5 cluster configuration, -#}
+{# this zope family is used by erp5-bootstrap and may serve to scalability monitoring tools. -#}
+{% set additional_scalability_zope = {'scalability':{'family':'scalability', 'computer-guid':computer_id}} -%}
+{% do slapparameter_dict.get('cluster').get('zope-partition-dict').update( additional_scalability_zope ) -%}
+{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
+config = {{ cluster_parameter_dict.keys() | join(' ') }}
+{% for key, value in cluster_parameter_dict.items() -%}
+config-{{ key }} = {{ dumps(value) }}
+{% endfor -%}
+return = mariadb-url family-scalability family-user memcached-persistent-url memcached-volatile-url cloudooo-url
 
-# Create wrapper and execute command with parameters
+# Create wrapper
 [binary-wrap-launcher]
 recipe = slapos.cookbook:wrapper
 log-path = ${basedirectory:log}/runScalabilityTestSuite.log
 binary-path = {{ bin_directory }}/runScalabilityTestSuite
 wrapper-path = ${rootdirectory:bin}/runScalabilityTestSuite
 output = ${binary-wrap-launcher:binary-path}
-# TODO : get the erp5-cluster main url (apache listening user port ?)
-erp5-url = http://foo.bar
-# Run scalability test suite wrapper command with parameters
-# -\ dirty way to conserve spaces
-command-line = "${binary-wrap-launcher:binary-path}" --erp5-url ${binary-wrap-launcher:erp5-url} --test-result-path {{ slapparameter_dict.get('test-result-path') }} --revision {{ slapparameter_dict.get('revision') }} --node-title {{ slapparameter_dict.get('node-title') }} --test-suite-master-url {{ slapparameter_dict.get('test-suite-master-url') }} --log-path ${binary-wrap-launcher:log-path}
+site-id = "/{{ site_id }}"
+# Suppose that there is a user zope family in the configuration
+erp5-url = ${erp5-cluster:connection-family-user}${:site-id}
+parameters-extra = $*
+command-line = ${binary-wrap-launcher:binary-path}
+return = url
+
+# Create wrapper
+[binary-wrap-performance-tester]
+recipe = slapos.cookbook:wrapper
+binary-path = {{ bin_directory }}/performance_tester
+wrapper-path = ${rootdirectory:bin}/performance_tester
+output = ${binary-wrap-performance-tester:binary-path}
+parameters-extra = $*
+command-line = ${binary-wrap-performance-tester:binary-path}
 return = url
 
+[scalability-launcher-instance]
+recipe = slapos.cookbook:scalability
+configuration-file = ${rootdirectory:etc}/scalability_runner.conf
+wrapper = ${basedirectory:services}/scalability_runner
+exec-path = ${binary-wrap-launcher:wrapper-path}
+erp5-url = ${binary-wrap-launcher:erp5-url}
+test-result-path = {{ slapparameter_dict.get('test-result-path') }}
+revision = {{ slapparameter_dict.get('revision') }}
+node-title = {{ slapparameter_dict.get('node-title') }}
+test-suite-master-url = {{ slapparameter_dict.get('test-suite-master-url') }}
+log-path = ${binary-wrap-launcher:log-path}
 
 # Create partition's directories
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
 log = ${rootdirectory:var}/log
+services = ${rootdirectory:etc}/run
+promises = ${rootdirectory:etc}/promise/
 
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc
 var = ${buildout:directory}/var
-srv = ${buildout:directory}/srv
 bin = ${buildout:directory}/bin
 
+[erp5-bootstrap]
+recipe = slapos.cookbook:erp5.bootstrap
+runner-path = ${basedirectory:services}/erp5-bootstrap
+mysql-url =  ${erp5-cluster:connection-mariadb-url}
+zope-username = zope
+zope-password = insecure
+zope-site-id = {{ site_id }}
+zope-url = ${erp5-cluster:connection-family-scalability}
+
+[erp5-promise]
+recipe = slapos.cookbook:erp5.promise
+promise-path = {{ promise_path }}
+kumofs-url = ${erp5-cluster:connection-memcached-persistent-url}
+memcached-url = ${erp5-cluster:connection-memcached-volatile-url}
+cloudooo-url = ${erp5-cluster:connection-cloudooo-url}
+smtp-url =
+bt5 =
+bt5-repository-url =
+
+[promise-erp5-site]
+recipe = slapos.cookbook:check_url_available
+path = ${basedirectory:promises}/erp5site
+site-id = "/{{ site_id }}"
+url = ${erp5-cluster:connection-family-scalability}${:site-id}
+dash_path = {{ dash_location }}/bin/dash
+curl_path = {{ curl_location }}/bin/curl
+
+[publish]
+recipe = slapos.cookbook:publish.serialised
+test = test
+
 {% endif %}

stack/erp5/instance-zeo.cfg.in

 {% if software_type == slap_software_type -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
-{% set next_port = 2000 -%}
+{% set next_port = slapparameter_dict['tcpv4-port'] -%}
 {% set part_list = [] -%}
 {% set storage_dict = {} -%}
 {% set zodb_dict = {} -%}
@@ -19,6 +19,11 @@ partition. No more (undefined result), no less (IndexError).
 {% set tidstorage_port = 6001 -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
 
+[logrotate-base]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:backup}
+
 [zeo-base]
 recipe = slapos.cookbook:zeo
 log-path = ${directory:log}/${:base-name}.log
@@ -50,7 +55,7 @@ ipv6 = {{ ipv6 }}
      )) -%}
 {% endfor -%}
 {% for storage_family, export_list in storage_dict.items() -%}
-{%   set known_tid_storage_identifier_host = (ipv4, local_current_port) -%}
+{%   set known_tid_storage_identifier_host = ((ipv4, next_port), ) -%}
 {%   set client_dict = {} -%}
 {%   for export_id, mount_point, cache_size, storage_dict in export_list -%}
 {%     do storage_dict.__setitem__('path', storage_dict.get('path', '%(zodb)s/' ~ export_id ~ '.fs') % {'zodb': default_zodb_path}) -%}
@@ -64,7 +69,7 @@ ipv6 = {{ ipv6 }}
        ]) -%}
 {%     if tidstorage_dict != None -%}
 {%       do known_tid_storage_identifier_dict.__setitem__(
-           (known_tid_storage_identifier_host, export_id), (
+           json_module.dumps((known_tid_storage_identifier_host, export_id)), (
              storage_dict['path'],
              tidstorage_dict.get('zodb-dict', {}).get(export_id, '%(backup)s/' ~ export_id) % {'backup': zodb_backup_path},
              mount_point,
@@ -83,6 +88,12 @@ port = {{ next_port }}
 {%   endfor -%}
 storage = {{ dumps(storage_list) }}
 
+[{{ section("logrotate-" ~ zeo_section_name) }}]
+< = logrotate-base
+name = {{ "${" ~ zeo_section_name ~ ":base-name}" }}
+log = {{ "${" ~ zeo_section_name ~ ":log-path}" }}
+post = {{ bin_directory }}/killpidfromfile {{ "${" ~ zeo_section_name ~ ":pid-path}" }} SIGUSR2
+
 [{{ section(zeo_section_name ~ "-promise") }}]
 recipe = slapos.cookbook:check_port_listening
 hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }}
@@ -158,9 +169,7 @@ frequency = 0 0 * * *
 command = ${tidstorage:repozo-wrapper}
 
 [{{ section("logrotate-tidstorage") }}]
-recipe = slapos.cookbook:logrotate.d
-logrotate-entries = ${logrotate:logrotate-entries}
-backup = ${logrotate:backup}
+< = logrotate-base
 name = tidstorage
 log = ${tidstorage:logfile-name}
 post = {{ bin_directory }}/killpidfromfile ${tidstorage:pidfile-name} SIGHUP

stack/erp5/instance.cfg.in

@@ -54,7 +54,6 @@ template = {{ template_cloudooo }}
 filename = instance-cloudoo.cfg
 extra-context =
     section parameter_dict dynamic-template-cloudooo-parameters
-    import json_module json
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type cloudooo
 
@@ -207,6 +206,7 @@ extensions = jinja2.ext.do
 extra-context =
     key buildout_directory buildout:directory
     section parameter_dict dynamic-template-zeo-parameters
+    import json_module json
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type zodb-zeo
 
@@ -248,7 +248,12 @@ extensions = jinja2.ext.do
 bin_directory = {{ bin_directory }}
 extra-context =
     key bin_directory dynamic-template-scalability:bin_directory
+    key curl_location template-erp5-single-parameters:curl
+    key dash_location template-erp5-single-parameters:dash
     key local_bt5_repository dynamic-template-cluster-erp5-parameters:local-bt5-repository
+    key template_zope dynamic-template-zope:rendered
+    key template_cluster_zope dynamic-template-cluster-zope:rendered
+    key template_erp5_cluster dynamic-template-cluster-erp5:rendered
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type scalability
 
@@ -271,6 +276,7 @@ extra-context =
 [dynamic-template-mariadb-parameters]
 coreutils-location = {{ coreutils_location }}
 dcron-location = {{ dcron_location }}
+findutils-location = {{ findutils_location }}
 gettext-location = {{ gettext_location }}
 grep-location = {{ grep_location }}
 gzip-location = {{ gzip_location }}