Split tidstorage software type in to several software types.

XXX: which newpart should contain coreutils ? production-zope ?
XXX: disable varnish section so it gets updated later

software/erp5/instance-balancer.cfg.in

+{% if software_type == slap_software_type -%}
+{#
+XXX: This template only supports exactly one IPv4 and one IPv6 per
+partition. No more (undefined result), no less (IndexError).
+-#}
+# TODO: insert varnish between apache & haproxy.
+# And think of a way to specify which urls goe through varnish, which go
+# directly to haproxy. (maybe just passing literal configuration file chunk)
+{% set ipv4 = (ipv4_set | list)[0] -%}
+{% set ipv6 = (ipv6_set | list)[0] -%}
+[buildout]
+extends = {{ parameter_dict['instance-logrotate-cfg'] }}
+parts +=
+  publish
+  logrotate-apache
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[publish]
+recipe = slapos.cookbook:publish
+url = ${apache:scheme}://[${apache:ip}]:${apache:port}
+
+[haproxy]
+recipe = slapos.cookbook:haproxy
+name = haproxy
+conf-path = ${directory:etc}/haproxy.cfg
+ip = {{ ipv4 }}
+port = 8001
+maxconn = {{ slapparameter_dict['haproxy-maxconn'] }}
+# XXX: should be optional (check recipe code)
+server-check-path = {{ slapparameter_dict['haproxy-server-check-path'] }}
+wrapper-path = ${directory:services}/haproxy
+binary-path = {{ parameter_dict['haproxy'] }}/sbin/haproxy
+backend-list = {{ slapparameter_dict['zope-address-list'] }}
+
+[apache]
+recipe = slapos.cookbook:apache.zope.backend
+backend = http://${haproxy:ip}:${haproxy:port}
+ip = {{ ipv6 }}
+port = 8000
+wrapper = ${directory:bin}/apache
+scheme = https
+key-file = ${directory:apache-conf}/apache.key
+cert-file = ${directory:apache-conf}/apache.crt
+configuration-file = ${directory:apache-conf}/apache.conf
+access-control-string = {{ slapparameter_dict['apache-access-control-string'] }}
+pid-file = ${directory:run}/apache.pid
+lock-file = ${directory:run}/apache.lock
+ssl-session-cache = $${directory:log}/apache-ssl-session-cache
+error-log = ${directory:log}/apache-error.log
+access-log = ${directory:log}/apache-access.log
+apache-binary = {{ parameter_dict['apache'] }}/bin/httpd
+ssl-authentication = {{ slapparameter_dict['apache-ssl-authentication'] }}
+backend-path = {{ slapparameter_dict['apache-backend-path'] }}
+# Note: Without certificate-authority main certificate have to be hardcoded
+ssl-authentication-certificate = ${certificate-authority:ca-dir}/cacert.pem
+ssl-authentication-crl = ${certificate-authority:ca-crl}
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = {{ parameter_dict['openssl'] }}/bin/openssl
+ca-dir = ${directory:ca-dir}
+requests-directory = ${directory:requests}
+wrapper = ${directory:services}/ca
+ca-private = ${directory:private}
+ca-certs = ${directory:certs}
+ca-newcerts = ${directory:newcerts}
+ca-crl = ${directory:crl}
+{% set ca = slapparameter_dict['ca'] -%}
+country-code = {{ ca['country-code'] }}
+email = {{ ca['email'] }}
+state = {{ ca['state'] }}
+city = {{ ca['city'] }}
+company = {{ ca['company'] }}
+
+[logrotate-apache]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:logrotate-backup}
+name = apache
+log = ${apache:error-log} ${apache:access-log}
+post = {{ bin_directory }}/killpidfromfile ${apache:pid-file} SIGUSR1
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+apache-conf = ${:etc}/apache
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+services = ${:etc}/run
+var = ${buildout:directory}/var
+run = ${:var}/run
+log = ${:var}/log
+ca-dir = ${buildout:directory}/srv/ssl
+requests = ${:ca-dir}/requests
+private = ${:ca-dir}/private
+certs = ${:ca-dir}/certs
+newcerts = ${:ca-dir}/newcerts
+crl = ${:ca-dir}/crl
+{% endif %}

software/erp5/instance-erp5-production.cfg.in

 {% if slap_software_type == software_type -%}
+{% set part_list = [] -%}
 #############################
 #
 # Request erp5 production environnment
 #
 #############################
-[buildout]
-
-parts =
-  request-tidstorage
-  basedirectory
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
 [request-common]
 recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}
@@ -58,12 +49,11 @@ name = KumoFS
 software-type = kumofs
 sla-computer_guid = ${slap-parameter:kumofs-computer-guid}
 
-[request-tidstorage]
+[request-zope-base]
 <=request-common
-name = TidStorage
-return = url-login
+return =
+  zope-address-list
 config =
-  json
   mysql-url
   memcached-url
   cloudooo-url
@@ -73,7 +63,10 @@ config =
   smtp-url
   bt5
   bt5-repository-url
-config-json = ${slap-parameter:json}
+  zodb-dict
+  tidstorage-ip
+  tidstorage-port
+  zope-instance-number
 config-mysql-url = ${request-mariadb:connection-url}
 config-memcached-url = ${request-memcached:connection-url}
 config-cloudooo-url = ${request-cloudooo:connection-url}
@@ -83,18 +76,10 @@ config-sphinx-url-sphinx-sql = ${request-sphinx:connection-url-sphinx-sql}
 config-bt5 = ${slap-parameter:bt5}
 config-bt5-repository-url = ${slap-parameter:bt5-repository-url}
 config-smtp-url = ${slap-parameter:smtp-url}
-software-type = tidstorage
-sla-computer_guid = ${slap-parameter:tidstorage-computer-guid}
-
-[request-varnish]
-<=request-common
-name = Varnish
-config = tidstorage-url
-config-tidstorage-url = ${request-tidstorage:connection-url-login}
-config-web-checker-mail-address = ${slap-parameter:web-checker-mail-address}
-config-web-checker-smtp-host = ${slap-parameter:web-checker-smtp-host}
-software-type = varnish
-sla-computer_guid = ${slap-parameter:varnish-computer-guid}
+config-zodb-dict = ${request-zodb:connection-zodb-dict}
+config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
+config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
+software-type = production-zope
 
 [slap-parameter]
 # Default value if no computer_guid is specified for each type
@@ -103,8 +88,7 @@ mariadb-computer-guid = ${slap-connection:computer-id}
 cloudooo-computer-guid = ${slap-connection:computer-id}
 memcached-computer-guid = ${slap-connection:computer-id}
 kumofs-computer-guid = ${slap-connection:computer-id}
-tidstorage-computer-guid = ${slap-connection:computer-id}
-varnish-computer-guid = ${slap-connection:computer-id}
+zodb-computer-guid = ${slap-connection:computer-id}
 cloudooo-json =
 bt5 = erp5_full_text_myisam_catalog
   erp5_configurator_standard
@@ -117,4 +101,151 @@ smtp-url = smtp://localhost:25/
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc/run
+
+[request-zodb]
+< = request-common
+return =
+  zodb-storage-type
+# (str)
+# Storage type, suitable for zope.conf (ex: "zeoclient").
+  zodb-dict
+# (dict)
+# key (str)
+#   (same as config-zodb-dict)
+# value (dict)
+#   Storage-type-specific parameters, suitable for zope.conf.
+#   key & value: str
+#   Ex: {'server': 'example.com:8100', 'storage': '1'}
+  tidstorage-ip
+# self-explaining
+  tidstorage-port
+# self-explaining
+config =
+  zodb-dict
+# (dict)
+# ZODBs to provide.
+# key (str)
+#   Zope internal name for this ZODB. Used to tell mountpoints apart.
+# value (dict)
+#   key (str)
+#     Possible keys and associated value types:
+#     "storage_ident" (str, optional)
+#       Storage identifier. All zodbs requested with the same storage_ident value
+#       are provided by the same service (ex: same ZEO process). Might not be
+#       supported by all storage_types, silently ignored if so.
+#       Defaults to "default" if not provided.
+#     "storage_dict" (dict, optional)
+#       Storage-type-specific parameters. For example, it can be used to tell where
+#       a ZEO filestorage database is located.
+  tidstorage-dict
+# (dict, optional)
+# Backup parameters for tidstorage-related backup scripts.
+# Set to None if no backup is desired.
+# key (str)
+#   Possible keys and associated value types:
+#   "zodb_dict" (dict, mandatory)
+#     key (str)
+#       (same as zodb-dict)
+#     value (dict)
+#       key (str)
+#         Possible keys and associated value types:
+#         "url_substitute" (str, mandatory)
+#           Value to substitute to "%s" in tidstorage-base-url-template to
+#           trigger a no-op change in storage.
+#         "path" (str, optional)
+#           Path to store backups of this zodb into.
+#   "timestamp_path" (str, optional)
+#     Path to backup timestamp file.
+  tidstorage-base-url-template
+# (str, None)
+# URL template base used by TIDStorage to modify all mountpoints upon startup.
+# ex: 'http://example.com:1080/%s/serialize'
+# Note that it needs to contain exactly one "%s" conversion specification,
+# which gets substituted with ZODB mountpoint by TIDStorage server.
+# Ignored when tidstorage-dict is None.
+{# TODO: make libslap able to generate nested, complex values without having to resort to SR-level parsing -#}
+config-zodb-dict = {{ json_module.loads(slapparameter_dict['zodb-dict']) }}
+config-tidstorage-dict = {{ slapparameter_dict.get('tidstorage-dict') }}
+# TODO: auto-generate from one zope (distributing node)
+config-tidstorage-base-url-template = {{ slapparameter_dict.get('tidstorage-base-url-template') }}
+software-type = {{ slapparameter_dict['zodb-software-type'] }}
+sla-computer_guid = ${slap-parameter:zodb-computer-guid}
+
+{% set i = 0 -%}
+{% set zope_family_dict = {} -%}
+{% for instance_count, zope_parameter_dict in json_module.loads(slapparameter_dict.get('zope-partition-list', '[[1, {}]]')) -%}
+{%   set name = 'zope-' ~ i -%}
+{%   set i = i + 1 -%}
+{%   do zope_family_dict.setdefault(zope_parameter_dict.get('family'), []).append(name) -%}
+[{{ name }}]
+< = request-zope-base
+name = {{ name }}
+config-zope-instance-count = {{ instance_count }}
+sla-computer_guid = {{ zope_parameter_dict.get('computer_guid', '${slap-connection:computer-id}') }}
+{% endfor %}
+[request-backend-base]
+< = request-common
+return = url
+config =
+  zope-address-list
+  haproxy-maxconn
+  haproxy-server-check-path
+  apache-access-control-string
+  apache-ssl-authentication
+  apache-backend-path
+  ca
+# XXX: should those really be same for all backends ?
+config-haproxy-maxconn = {{ slapparameter_dict.get('haproxy-maxconn', 1) }}
+config-haproxy-server-check-path = {{ slapparameter_dict.get('haproxy-server-check-path', '/') }}
+config-apache-access-control-string = {{ slapparameter_dict.get('apache-access-control-string', '') }}
+config-apache-ssl-authentication = {{ slapparameter_dict.get('apache-ssl-authentication', '0') }}
+config-apache-backend-path = {{ slapparameter_dict.get('apache-backend-path', '/') % {'site_id': slapparameter_dict['site_id']} }}
+config-ca = {{ json_module.loads(slapparameter_dict['ca']) }}
+
+[request-frontend-base]
+{% if 'frontend-software-url' in slapparameter_dict -%}
+< = request-common
+software-url = {{ slapparameter_dict['frontend-software-url'] }}
+software-type = {{ slapparameter_dict.get('frontend-software-type', 'RootSoftwareInstance') }}
+sla = instance_guid
+sla-instance_guid = {{ slapparameter_dict['frontend-instance-guid'] }}
+slave = true
+{% set config_dict = {
+  'type': 'zope',
+} -%}
+{% if 'frontend-domain' in slapparameter_dict -%}
+{%   do config_dict.__setitem__('custom_domain', slapparameter_dict['frontend-domain']) -%}
+{% endif -%}
+config = url {{ config_dict.keys() | join(' ') }}
+{% for name, value in config_dict.items() -%}
+config-{{ name }} = {{ value }}
+{% endfor -%}
+return = site_url
+{% endif -%}
+
+{% set i = 0 -%}
+{% for zope_section_id_list in zope_family_dict.values() -%}
+{%   set backend_name = 'backend-' ~ i -%}
+{%   set frontend_name = 'frontend-' ~ i -%}
+{%   do part_list.append(frontend_name) -%}
+{%   set i = i + 1 -%}
+[{{ backend_name }}]
+< = request-backend-base
+name = {{ backend_name }}
+config-zope-address-list =
+{% for zope_section_id in zope_section_id_list %}{{ '    ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}{% endfor %}
+
+[{{ frontend_name }}]
+< = request-frontend-base
+name = {{ frontend_name }}
+config-url = {{ '${' ~ backend_name ~ ':connection-url}' }}
+{% endfor %}
+[buildout]
+parts =
+  basedirectory
+  {{ part_list | join('\n  ') }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
 {% endif %}

software/erp5/instance-logrotate-base.cfg.in

+[buildout]
+parts =
+  cron-entry-logrotate
+
+[cron]
+recipe = slapos.cookbook:cron
+cron-entries = ${logrotate-directory:cron-entries}
+dcrond-binary = {{ dcron_location }}/sbin/crond
+crontabs = ${logrotate-directory:crontabs}
+cronstamps = ${logrotate-directory:cronstamps}
+catcher = ${cron-simplelogger:wrapper}
+binary = ${logrotate-directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = ${logrotate-directory:bin}/cron_simplelogger
+log = ${logrotate-directory:log}/cron.log
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+logrotate-entries = ${logrotate-directory:logrotate-entries}
+backup = ${logrotate-directory:logrotate-backup}
+logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate
+gzip-binary = {{ gzip_location }}/bin/gzip
+gunzip-binary = {{ gzip_location }}/bin/gunzip
+wrapper = ${logrotate-directory:bin}/logrotate
+conf = ${logrotate-directory:etc}/logrotate.conf
+state-file = ${logrotate-directory:srv}/logrotate.status
+
+[cron-entry-logrotate]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = logrotate
+frequency = 0 0 * * *
+command = ${logrotate:wrapper}
+
+[logrotate-directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = ${:etc}/cron.d
+cronstamps = ${:etc}/cronstamps
+crontabs = ${:etc}/crontabs
+logrotate-backup = ${:backup}/logrotate
+logrotate-entries = ${:etc}/logrotate.d
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+etc = ${buildout:directory}/etc
+services = ${:etc}/run
+log = ${buildout:directory}/var/log
+

software/erp5/instance-production-zope.cfg.in

+{% if slap_software_type == software_type -%}
+{% set site_id = slapparameter_dict['site-id'] -%}
+{% set part_list = [] -%}
+{% set publish_list = [] -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
+{#
+XXX: This template only supports exactly one IPv4 and one IPv6 per
+partition. No more (undefined result), no less (IndexError).
+-#}
+{% set ipv4 = (ipv4_set | list)[0] -%}
+{% set ipv6 = (ipv6_set | list)[0] -%}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+instance = ${:srv}/erp5shared
+instance-constraint = ${:instance}/Constraint
+instance-document = ${:instance}/Document
+instance-etc = ${:instance}/etc
+instance-etc-package-include = ${:instance}/etc/package-include
+instance-extensions = ${:instance}/Extensions
+instance-import = ${:instance}/import
+instance-lib = ${:instance}/lib
+instance-products = ${:instance}/Products
+instance-propertysheet = ${:instance}/PropertySheet
+instance-tests = ${:instance}/tests
+log = ${:var}/log
+run = ${:var}/run
+services = ${:etc}/run
+srv = ${buildout:directory}/srv
+tmp = ${buildout:directory}/tmp
+var = ${buildout:directory}/var
+
+[binary-link]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:bin}
+link-binary =
+  {{ parameter_dict['coreutils'] }}/bin/basename
+  {{ parameter_dict['coreutils'] }}/bin/cat
+  {{ parameter_dict['coreutils'] }}/bin/cp
+  {{ parameter_dict['coreutils'] }}/bin/ls
+  {{ parameter_dict['coreutils'] }}/bin/tr
+  {{ parameter_dict['coreutils'] }}/bin/uname
+  {{ parameter_dict['git'] }}/bin/git
+  {{ parameter_dict['graphviz'] }}/bin/dot
+  {{ parameter_dict['grep'] }}/bin/grep
+  {{ parameter_dict['imagemagick'] }}/bin/convert
+  {{ parameter_dict['imagemagick'] }}/bin/identify
+  {{ parameter_dict['mariadb'] }}/bin/mysql
+  {{ parameter_dict['mariadb'] }}/bin/mysqldump
+  {{ parameter_dict['pdftk'] }}/bin/pdftk
+  {{ parameter_dict['sed'] }}/bin/sed
+  {{ parameter_dict['tesseract'] }}/bin/tesseract
+  {{ parameter_dict['w3m'] }}/bin/w3m
+  {{ parameter_dict['poppler'] }}/bin/pdfinfo
+  {{ parameter_dict['poppler'] }}/bin/pdftotext
+  {{ parameter_dict['poppler'] }}/bin/pdftohtml
+  {{ parameter_dict['dmtx-utils'] }}/bin/dmtxwrite
+
+[certificate-authority-common]
+requests-directory = ${directory:requests}
+ca-dir = ${directory:ca-dir}
+ca-private = ${directory:private}
+ca-certs = ${directory:certs}
+ca-newcerts = ${directory:newcerts}
+ca-crl = ${directory:crl}
+
+[certificate-authority]
+< = certificate-authority-common
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = {{ openssl }}/bin/openssl
+wrapper = ${directory:services}/ca
+
+[stunnel-cert]
+< = certificate-authority-common
+recipe = slapos.cookbook:certificate_authority.request
+key-file = ${directory:stunnel-conf}/stunnel.key
+cert-file = ${directory:stunnel-conf}/stunnel.crt
+executable = ${stunnel:wrapper}
+wrapper = ${directory:services}/stunnel
+
+[stunnel-base]
+recipe = slapos.cookbook:stunnel
+stunnel-binary = {{ stunnel }}/bin/stunnel
+wrapper = ${directory:bin}/stunnel-${:name}
+log-file = ${directory:log}/stunnel-${:name}.log
+config-file = ${directory:etc}/stunnel-${:name}.conf
+pid-file = ${directory:run}/stunnel-${:name}.pid
+client = false
+key-file = ${stunnel-cert:key-file}
+cert-file = ${stunnel-cert:cert-file}
+remote-address = {{ ipv6 }}
+remote-port = ${:local-port}
+local-address = {{ ipv4 }}
+
+[zope-base]
+recipe = slapos.cookbook:generic.zope.zeo.client
+user = zope
+ip = {{ ipv4 }}
+timezone = {{ slapparameter_dict['timezone'] }}
+tidstorage-ip = ${tidstorage:ip}
+tidstorage-port = ${tidstorage:port}
+instance-etc = ${directory:instance-etc}
+bt5-repository = ${directory:var}/bt5_repository
+tmp-path = ${directory:tmp}
+bin-path = ${directory:bin}
+site-zcml = ${:instance-etc}/site.zcml
+inituser = ${directory:instance}/inituser
+runzope-binary = {{ bin_directory }}/runzope
+bt5-repository-list =
+
+[deadlock-debugger-password]
+recipe = slapos.cookbook:pwgen.stable
+
+[zope-conf-parameter-base]
+ip = {{ ipv4 }}
+site-id = {{ site_id }}
+zodb-list = {{ dumps(zodb_connection_list) }}
+# XXX: products won't be needed as soon as all ERP5 (and products-deps)
+# products will be eggified so then it will be possible to use them thanks to
+# availability in software's eggs
+products = {{ parameter_dict['products'] }}
+
+[zope-conf-base]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['zope-conf-template'] }}
+extra-context =
+context =
+  key instance directory:instance
+  key instance_products directory:instance-products
+  raw deadlock_path /manage_debug_threads
+  key deadlock_debugger_password deadlock-debugger-password:password
+  key tidstorage_ip tidstorage:ip
+  key tidstorage_port tidstorage:port
+  key promise_path erp5-promise:promise-path
+  ${:extra-context}
+
+[logrotate-entry-base]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:logrotate-backup}
+
+{% set zope_dummy_list = [] -%}
+{% macro zope(
+  name,
+  publish,
+  thread_amount=1,
+  timerserver_interval=5,
+  longrequest_logger_file='',
+  longrequest_logger_timeout='',
+  longrequest_logger_interval=''
+) -%}
+{% set conf_name = name ~ '-conf' -%}
+{% set conf_parameter_name = conf_name ~ '-param' -%}
+[{{ conf_parameter_name }}]
+< = zope-conf-parameter-base
+pid-file = ${directory:run}/{{ name }}.pid
+lock-file = ${directory:run}/{{ name }}.lock
+{% do zope_dummy_list.append(None) -%}
+{% set offset = zope_dummy_list | length -%}
+{#
+XXX: port base hardcoded
+-#}
+port = {{ 2000 + offset }}
+thread-amount = {{ thread_amount }}
+{% if timerserver_interval -%}
+timerserver-interval = {{ timerserver_interval }}
+{% endif -%}
+event-log = ${directory:log}/{{ name }}-event.log
+z2-log = ${directory:log}/{{ name }}-Z2.log
+
+[{{ conf_name }}]
+< = zope-conf-base
+rendered = ${directory:etc}/{{ name }}.conf
+extra-context =
+  section parameter_dict {{ conf_parameter_name }}
+
+[{{ section(name) }}]
+< = zope-base
+longrequest-logger-file = {{ longrequest_logger_file }}
+longrequest-logger-timeout = {{ longrequest_logger_timeout }}
+longrequest-logger-interval = {{ longrequest_logger_interval }}
+wrapper = ${directory:services}/{{ name }}
+configuration-file = {{ '${' ~ conf_name ~ ':rendered}' }}
+port = {{ '${' ~ conf_parameter_name ~ ':port}' }}
+
+{% set stunnel_name = 'stunnel-' ~ name -%}
+[{{ stunnel_name }}]
+{% if publish -%}
+< = stunnel-base
+name = {{ name }}
+local-port = {{ '${' ~ name ~ ':port}' }}
+{%   do publish_list.append(stunnel_name) -%}
+{% else -%}
+# Dummy entry to keep logrotate section template simple
+log-file =
+{% endif %}
+[{{ section('logrotate-entry-' ~ name) }}]
+< = logrotate-entry-base
+name = {{ name }}
+log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ stunnel_name ~ ':log-file}' }}
+post = {{ bin_directory }}/killpidfromfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} SIGUSR2
+{% endmacro -%}
+
+#{ % for ... in ... -%}
+#{ {    zope(...) }}
+#{ %  endfor -%}
+{{ zope("foo", False) }}
+{{ zope("bar", True) }}
+
+[publish-zope]
+recipe = slapos.cookbook:publish
+url-list =
+  {{ publish_list | join('\n  ') }}
+
+# TODO: move to another place - or remove altogether ?
+#[erp5-bootstrap]
+#recipe = slapos.cookbook:erp5.bootstrap
+#runner-path = ${directory:services}/erp5-bootstrap
+#mysql-url = {{ slapparameter_dict['mysql-url'] }}
+#zope-url = http://${zope-admin:user}:${zope-admin:password}@${zope-admin:ip}:${zope-admin:port}/{{ site_id }}
+
+[erp5-promise]
+recipe = slapos.cookbook:erp5.promise
+promise-path = ${directory:etc}/erp5promise.cfg
+kumofs-url = {{ slapparameter_dict['kumofs-url'] }}
+memcached-url = {{ slapparameter_dict['memcached-url'] }}
+cloudooo-url = {{ slapparameter_dict['cloudooo-url'] }}
+smtp-url = ${slap-parameter:smtp-url}
+bt5 = ${slap-parameter:bt5}
+bt5-repository-url = ${slap-parameter:bt5-repository-url}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+extends =
+  {{ parameter_dict['instance-logrotate-cfg'] }}
+parts +=
+  binary-link
+  erp5-promise
+  erp5-bootstrap
+  {{ part_list | join('\n  ') }}
+{% endif %}

software/erp5/instance-zeo.cfg.in

+{% if software_type == slap_software_type -%}
+{% set current_port = 2000 -%}
+{% set part_list = [] -%}
+{% set storage_dict = {} -%}
+{% set zodb_dict = {} -%}
+{% set tidstorage_dict = slapparameter_dict.get('tidstorage-dict') -%}
+{% set known_tid_storage_identifier_dict = {} -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+{% macro zodb(base_name, storage_type, port) -%}
+{% endmacro -%}
+{#
+XXX: This template only supports exactly one IPv6 per
+partition. No more (undefined result), no less (IndexError).
+-#}
+{% set ipv6 = (ipv6_set | list)[0] -%}
+{% set tidstorage_ip = ipv6 -%}
+{% set tidstorage_port = 6001 -%}
+{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
+[buildout]
+extends =
+  {{ parameter_dict['instance-logrotate-cfg'] }}
+parts +=
+  publish
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[zeo-base]
+recipe = slapos.cookbook:zeo
+log-path = ${directory:log}/${:base-name}.log
+pid-path = ${directory:run}/${:base-name}.pid
+conf-path = ${directory:etc}/${:base-name}.conf
+wrapper-path = ${directory:services}/${:base-name}
+binary-path = {{ bin_directory }}/runzeo
+# TODO: check ipv6 actually works
+# TODO: setup ZEO secure auth & cryptography
+ip = {{ ipv6 }}
+
+{% for export_id, mountpoint_dict in slapparameter_dict['zodb-dict'].items() -%}
+{%   do storage_dict.setdefault(
+       mountpoint_dict.get('storage_ident', 'default'), []
+     ).append((
+       export_id,
+       mountpoint_dict.get('storage_dict', {}),
+     )) -%}
+{% endfor -%}
+{% for storage_ident, export_list in storage_dict.items() -%}
+{%   set key_template = "((('" ~ tidstorage_ip ~ "', " ~ tidstorage_port ~ "),), '%s')" -%}
+{%   for export_id, storage_dict in export_list -%}
+{%     do zodb_dict.__setitem__(export_id, {
+         'server': ipv6 ~ ':' ~ current_port,
+         'storage': export_id,
+       }) -%}
+{#     TODO: generate sane defaults for storage_dict['path'] and tidstorage_dict[export_id]['path'] -#}
+{%     if tidstorage_dict -%}
+{%       do known_tid_storage_identifier_dict.__setitem__(key_template % (export_id, ), (
+           storage_dict['path'],
+           tidstorage_dict[export_id]['path'],
+           tidstorage_dict[export_id]['url_substitute'],
+         )) -%}
+{%     endif -%}
+{%   endfor -%}
+[{{ section(base_name) }}]
+< = zeo-base
+base-name = zeo-{{ storage_ident }}
+port = {{ current_port }}
+storage = {{ dumps(export_list) }}
+{%   set current_port = current_port + 1 -%}
+{% endfor -%}
+
+{% if tidstorage_dict -%}
+[tidstorage]
+recipe = slapos.cookbook:tidstorage
+known-tid-storage-identifier-dict = {{ known_tid_storage_identifier_dict }}
+base-url = {{ slapparameter_dict['tidstorage-base-url-template'] }}
+configuration-path = ${directory:etc}/tidstorage.py
+ip = {{ tidstorage_ip }}
+port = {{ tidstorage_port }}
+timestamp-file-path = ${directory:tidstorage}/repozo_tidstorage_timestamp.log
+logfile-name = ${directory:log}/tidstorage.log
+pidfile-name = ${directory:run}/tidstorage.pid
+status-file = ${directory:tidstorage}/tidstorage.tid
+tidstorage-repozo-binary = {{ bin_directory }}/tidstorage_repozo
+tidstoraged-binary = {{ bin_directory }}/tidstoraged
+repozo-binary = {{ bin_directory }}/repozo
+tidstorage-wrapper = ${directory:services}/tidstoraged
+repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo
+# TODO: stunnel/TLS
+
+[cron-entry-tidstorage-backup]
+# TODO:
+# - configurable periodicity
+# - configurable full/incremental
+# - configurable retention
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = tidstorage
+frequency = 0 0 * * *
+command = ${tidstorage:repozo-wrapper}
+
+[logrotate-tidstorage]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:logrotate-backup}
+name = tidstorage
+log = ${tidstorage:logfile-name}
+post = {{ bin_directory }}/killpidfromfile ${tidstorage:pidfile-name} SIGHUP
+
+{% else -%}
+[tidstorage]
+# Dummy section, backups are disabled
+ip =
+port =
+
+{% endif -%}
+[publish]
+recipe = slapos.cookbook:publish
+zodb-storage-type = zeoclient
+zodb-dict = {{ dumps(zodb_dict) }}
+tidstorage-ip = ${tidstorage:ip}
+tidstorage-port = ${tidstorage:port}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+services = ${:etc}/run
+srv = ${buildout:directory}/run
+tidstorage = ${:srv}/tidstorage
+var = ${buildout:directory}/var
+log = ${:var}/log
+run = ${:var}/run
+
+{% endif %}

software/erp5/instance.cfg.in

@@ -25,47 +25,6 @@ context =
     key slapparameter_dict slap-parameters:configuration
     ${:extra-context}
 
-[dynamic-template-tidstorage-parameters]
-apache = {{ apache_location }}
-haproxy = {{ haproxy_location }}
-dcron = {{ dcron_location }}
-logrotate = {{ logrotate_location }}
-gzip = {{ gzip_location }}
-openssl = {{ openssl_location }}
-coreutils = {{ coreutils_location }}
-git = {{ git_location }}
-graphviz = {{ graphviz_location }}
-grep = {{ grep_location }}
-imagemagick = {{ imagemagick_location }}
-librsvg = {{ librsvg_location }}
-mariadb = {{ mariadb_location }}
-pdftk = {{ pdftk_location }}
-sed = {{ sed_location }}
-tesseract = {{ tesseract_location }}
-w3m = {{ w3m_location }}
-aspell = {{ aspell_location }}
-poppler = {{ poppler_location }}
-dmtx-utils = {{ dmtx_utils_location }}
-buildout-bin-directory = {{ buildout_bin_directory }}
-zope-conf-template = {{ template_zope_conf }}
-
-[dynamic-template-tidstorage]
-< = jinja2-template-base
-template = {{ template_tidstorage }}
-filename = instance-tidstorage.cfg
-extensions = jinja2.ext.do
-extra-context =
-    section parameter_dict dynamic-template-tidstorage-parameters
-    import json_module json
-    raw zope_port_base 12000
-    raw zeo_port_base 15000
-    raw haproxy_port_base 11000
-    raw apache_port_base 10000
-    key ipv4_set slap-parameters:ipv4
-    key ipv6_set slap-parameters:ipv6
-# Must match the key id in [switch-softwaretype] which uses this section.
-    raw software_type tidstorage
-
 [dynamic-template-cloudooo-parameters]
 file = {{ file_location }}
 fontconfig = {{ fontconfig_location }}
@@ -181,30 +140,98 @@ extra-context =
     key openssl_location template-erp5-development-parameters:openssl
     key template_zope dynamic-template-zope:rendered
 
-[dynamic-template-production-parameters]
+[dynamic-template-erp5-production-parameters]
 local-bt5-repository = {{ local_bt5_repository }}
 
-[dynamic-template-production]
+[dynamic-template-erp5-production]
 < = jinja2-template-base
 template = {{ template_erp5_production }}
 filename = instance-erp5-production.cfg
+extensions = jinja2.ext.do
 extra-context =
-    key local_bt5_repository dynamic-template-production-parameters:local-bt5-repository
+    key local_bt5_repository dynamic-template-erp5-production-parameters:local-bt5-repository
+    import json_module json
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type production
 
+[dynamic-template-balancer-parameters]
+apache = {{ apache_location }}
+openssl = {{ openssl_location }}
+haproxy = {{ haproxy_location }}
+instance-logrotate-cfg = {{ template_logrotate_base }}
+
+[dynamic-template-balancer]
+< = jinja2-template-base
+template = {{ template_balancer }}
+filename = instance-balancer.cfg
+extra-context =
+    key ipv4_set slap-parameters:ipv4
+    key ipv6_set slap-parameters:ipv6
+    section parameter_dict dynamic-template-balancer-parameters
+# Must match the key id in [switch-softwaretype] which uses this section.
+# XXX: a better name should be searched
+    raw software_type balancer
+
+[dynamic-template-zeo-parameters]
+buildout-bin-directory = {{ buildout_bin_directory }}
+instance-logrotate-cfg = {{ template_logrotate_base }}
+
+[dynamic-template-zeo]
+< = jinja2-template-base
+template = {{ template_zeo }}
+filename = instance-zeo.cfg
+extensions = jinja2.ext.do
+extra-context =
+    key ipv6_set slap-parameters:ipv6
+    section parameter_dict dynamic-template-zeo-parameters
+#    key instance_stunnel
+# Must match the key id in [switch-softwaretype] which uses this section.
+    raw software_type zeo
+
+[dynamic-template-production-zope-parameters]
+zope-conf-template = {{ template_zope_conf }}
+products = {{ product_list }}
+instance-logrotate-cfg = {{ template_logrotate_base }}
+git = {{ git_location }}
+graphviz = {{ graphviz_location }}
+buildout-bin-directory = {{ buildout_bin_directory }}
+coreutils = {{ coreutils_location }}
+grep = {{ grep_location }}
+imagemagick = {{ imagemagick_location }}
+#mariadb = {{ mariadb_location }} ?
+pdftk = {{ pdftk_location }}
+sed = {{ sed_location }}
+tesseract = {{ tesseract_location }}
+w3m = {{ w3m_location }}
+poppler = {{ poppler_location }}
+dmtx-utils = {{ dmtx_utils_location }}
+
+[dynamic-template-production-zope]
+< = jinja2-template-base
+template = {{ template_production_zope }}
+filename = instance-production-zope.cfg
+extensions = jinja2.ext.do
+extra-context =
+    key ipv6_set slap-parameters:ipv6
+    section parameter_dict dynamic-template-production-zope
+#    key instance_stunnel
+# Must match the key id in [switch-softwaretype] which uses this section.
+    raw software_type production-zope
+
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = ${template-erp5-development:rendered}
-production = ${dynamic-template-production:rendered}
+production = ${dynamic-template-erp5-production:rendered}
 kumofs = {{ template_kumofs }}
 memcached = {{ template_memcached }}
 cloudooo = ${dynamic-template-cloudooo:rendered}
 zope = ${dynamic-template-zope:rendered}
 mariadb = {{ template_mariadb }}
 sphinx = {{ template_sphinx }}
-tidstorage = ${dynamic-template-tidstorage:rendered}
 varnish = ${dynamic-template-varnish:rendered}
+balancer = ${dynamic-template-balancer:rendered}
+zeo = ${dynamic-template-zeo:rendered}
+production-zope = ${dynamic-template-production-zope:rendered}
 
 [slap-connection]
 # part to migrate to new - separated words

software/erp5/software.cfg

@@ -107,13 +107,6 @@ extra-context =
     key kumo_location kumo:location
     key logrotate_location logrotate:location
 
-
-[template-tidstorage]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-tidstorage.cfg.in
-md5sum = 520e39f2612a045b42c0ad914bfdc5a9
-mode = 640
-
 [template-cloudooo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
@@ -134,7 +127,7 @@ mode = 640
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 9e966770123ab09c018533ad9ac8a742
+md5sum = f9217ba04cd1f6f7bbfb5f8d758494d1
 extra-context =
     key apache_location apache:location
     key aspell_location aspell:location
@@ -172,15 +165,18 @@ extra-context =
     key pdftk_location pdftk:location
     key poppler_location poppler:location
     key sed_location sed:location
+    key template_balancer template-balancer:target
     key template_cloudooo template-cloudooo:target
     key template_erp5_development template-erp5-development:target
     key template_erp5_production template-erp5-production:target
     key template_kumofs template-kumofs:rendered
+    key template_logrotate_base template-logrotate-base:rendered
     key template_mariadb template-mariadb:rendered
     key template_memcached template-memcached:rendered
+    key template_production_zope template-production-zope:target
     key template_sphinx template-sphinx:rendered
-    key template_tidstorage template-tidstorage:target
     key template_varnish template-varnish:target
+    key template_zeo template-zeo:target
     key template_zope template-zope:target
     key template_zope_conf template-zope-conf:target
     key tesseract_location tesseract:location
@@ -206,9 +202,36 @@ mode = 640
 [template-erp5-production]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-erp5-production.cfg.in
-md5sum = 33beda33ae8105b61956fa53f2643303
+md5sum = eabeba496634ebcb4898c219d7d50776
+mode = 640
+
+[template-zeo]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-zeo.cfg.in
+md5sum = 98ee5c8802c58cfca792813a71cebf52
+mode = 640
+
+[template-production-zope]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-production-zope.cfg.in
+md5sum = b340d3f6a6eb1cd3dfaf87d00e8d6029
 mode = 640
 
+[template-balancer]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-balancer.cfg.in
+md5sum = d460b32d46f56723d1d2de364610ef9f
+mode = 640
+
+[template-logrotate-base]
+< = template-jinja2-base
+filename = instance-logrotate-base.cfg
+md5sum = 58cb9b7690a56d65502cb37aba9860ef
+extra-context =
+    key dcron_location dcron:location
+    key gzip_location gzip:location
+    key logrotate_location logrotate:location
+
 [template-varnish]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-varnish.cfg.in