erp5: by default, create site automatically

Since erp5@6c6b77a0, this is now a safe
operation, in that a mistake while moving databases won't result in a loss of
the catalog if the zodb is empty.

This new implementation relies on
erp5@6bb4c56e, which means that zopes
decide on their own to create the site at startup if there's none, and the
'erp5-bootstrap' service is removed from the root partition.

setup.py

@@ -101,7 +101,6 @@ setup(name=name,
           'duplicity = slapos.recipe.duplicity:Recipe',
           'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
           'equeue = slapos.recipe.equeue:Recipe',
-          'erp5.bootstrap = slapos.recipe.erp5_bootstrap:Recipe',
           'erp5.promise = slapos.recipe.erp5_promise:Recipe',
           'erp5.test = slapos.recipe.erp5_test:Recipe',
           'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',

slapos/recipe/erp5_bootstrap/__init__.py

-##############################################################################
-#
-# Copyright (c) 2012-2014 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-from slapos.recipe.librecipe import GenericBaseRecipe
-import sys
-import urlparse
-
-class Recipe(GenericBaseRecipe):
-  """
-  Instanciate ERP5 in Zope
-  """
-
-  def install(self):
-    mysql = urlparse.urlsplit(self.options['mysql-url'])
-    zope = urlparse.urlsplit(self.options['zope-url'])
-    # Note: raises when there is more than a single element in path, as it's
-    # not supported by manage_addERP5Site anyway.
-    _, zope_path = zope.path.split('/')
-    return [self.createExecutable(
-      self.options['runner-path'],
-      self.substituteTemplate(
-        self.getTemplateFilename('erp5_bootstrap.in'),
-        {
-          'python_path': sys.executable,
-          'base_url': urlparse.urlunsplit((zope.scheme, zope.netloc, '', '', '')),
-          'site_id': zope_path,
-          'sql_connection_string': '%(database)s@%(hostname)s:%(port)s %(username)s %(password)s' % {
-            'database': mysql.path.split('/')[1],
-            'hostname': mysql.hostname,
-            'port': mysql.port,
-            'username': mysql.username,
-            'password': mysql.password
-          },
-        },
-    ))]

slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in

-#!%(python_path)s
-import urllib
-
-import ssl
-if hasattr(ssl, '_create_unverified_context'):
-  ssl._create_default_https_context = ssl._create_unverified_context
-
-def isSuccess(response):
-  return 200 <= response.code < 300
-
-base_url = %(base_url)r
-response = urllib.urlopen(base_url + '/isERP5SitePresent')
-if isSuccess(response) and response.read() == '':
-  mysql_url = %(sql_connection_string)r
-  response = urllib.urlopen(
-    base_url + '/manage_addProduct/ERP5/manage_addERP5Site',
-    data=urllib.urlencode({
-      'id': %(site_id)r,
-      'erp5_catalog_storage': 'erp5_mysql_innodb_catalog',
-      'erp5_sql_connection_string': mysql_url,
-      'cmf_activity_sql_connection_string': mysql_url,
-    }),
-  )
-  if not isSuccess(response):
-    raise ValueError('Failed creating site, status=%%i: %%s' %% (response.code, response.read()))
-  print "ERP5 site created."

software/erp5/instance-erp5-input-schema.json

@@ -13,7 +13,7 @@
       "type": "object"
     },
     "site-id": {
-      "description": "ERP5Site object's id",
+      "description": "ERP5Site object's id. An empty value disables automatic site creation.",
       "default": "erp5",
       "type": "string"
     },

stack/erp5/buildout.cfg

@@ -210,12 +210,6 @@ link-binary =
 [template-mariadb-initial-setup]
 <= download-base
 
-[template-create-erp5-site]
-<= download-base
-
-[template-create-erp5-site-real]
-<= download-base
-
 [template-postfix]
 < = download-base
 
@@ -303,8 +297,6 @@ context =
     key template_apache_conf template-apache-backend-conf:target
     key template_balancer template-balancer:target
     key template_cloudooo template-cloudooo:target
-    key template_create_erp5_site template-create-erp5-site:target
-    key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
     key template_jupyter_cfg instance-jupyter-notebook:rendered

stack/erp5/buildout.hash.cfg

@@ -39,7 +39,7 @@ md5sum = 76f9e8c8cdc352081e34539d8fc17026
 
 [template-zope-conf]
 filename = zope.conf.in
-md5sum = 77ab4f229a92e02603028a0bd3772edd
+md5sum = bbea91bc8f0b2d455d9824928abfad5b
 
 [site-zcml]
 filename = site.zcml
@@ -53,14 +53,6 @@ md5sum = d814b984abf2dc444af2a0bc6287e7f5
 filename = mariadb_initial_setup.sql.in
 md5sum = 6465212fdc7fe9076a0c929d9f14da14
 
-[template-create-erp5-site]
-filename = instance-create-erp5-site.cfg.in
-md5sum = 1186c5804bdc679d8a31ac70cd85d51f
-
-[template-create-erp5-site-real]
-filename = instance-create-erp5-site-real.cfg.in
-md5sum = 79f789360e71146486c82a7a10834bae
-
 [template-postfix]
 filename = instance-postfix.cfg.in
 md5sum = e217f8b55f22d99ee2352eba6807d6f6
@@ -79,7 +71,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template]
 filename = instance.cfg.in
-md5sum = 7b4ebfd2ef31ed57a7129020c8ccde67
+md5sum = 258146a9d979b9981d18875fcac4de73
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -95,7 +87,7 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = a2377d5c53fd2a441ea713b428e4844b
+md5sum = 27d26c6380883cf3bd7b2f003f7888d8
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-create-erp5-site-real.cfg.in

-[directory]
-recipe = slapos.cookbook:mkdirectory
-etc = ${buildout:directory}/etc
-services = ${:etc}/run
-promise = ${:etc}/promise
-
-[erp5-bootstrap]
-recipe = slapos.cookbook:erp5.bootstrap
-runner-path = ${directory:services}/erp5-bootstrap
-{# Note: a random domain name will be picked if several point to the same IP -#}
-{% set reverse_hosts = {} -%}
-{% for x, y in publish['hosts-dict'].iteritems() -%}
-{%   do reverse_hosts.__setitem__(y, x) -%}
-{% endfor -%}
-{# XXX: Expect the first database to be the one to use for catalog. -#}
-{% set mysql_parsed = urlparse.urlparse(publish['mariadb-database-list'][0]) -%}
-mysql-url = {{ dumps(urlparse.urlunparse(mysql_parsed[:1] + (mysql_parsed.username + ":" + mysql_parsed.password + "@" + reverse_hosts.get(mysql_parsed.hostname, mysql_parsed.hostname) + ':' ~ mysql_parsed.port, ) + mysql_parsed[2:])) }}
-{# Pick the first http[s] family found, they should be all equivalent anyway. -#}
-{% set family_list = [] -%}
-{% for key, value in publish.items() -%}
-{%   if key.startswith('family-') and value.startswith('http') -%}
-{%     do family_list.append(value.split('://', 1)) -%}
-{%   endif -%}
-{%  endfor -%}
-zope-url = {{ dumps(family_list[0][0] + '://' + publish['inituser-login'] + ':' + publish_early['inituser-password'] + '@' + family_list[0][1] + '/' + publish['site-id']) }}
-
-[promise-erp5-site]
-recipe = slapos.cookbook:check_url_available
-url = ${erp5-bootstrap:zope-url}
-path = ${directory:promise}/erp5-site
-dash_path = {{ parameter_dict['dash-location'] }}/bin/dash
-curl_path = {{ parameter_dict['curl-location'] }}/bin/curl
-
-[buildout]
-parts = promise-erp5-site
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}

stack/erp5/instance-create-erp5-site.cfg.in

-{# To create the script (wrapper) which creates the ERP5Site object, pieces
- # of what is published by extended file are required. Because they are not
- # available at the time the file you are reading is rendered, and because
- # those values are composed (lists, dicts...) of which items are needed,
- # they cannot be accessed. Instead, make buildout provide these values to
- # a second template, rendered at a convenient time.
--#}
-[instance-create-erp5-site-real-parameters]
-dash-location = {{ parameter_dict['dash-location'] }}
-curl-location = {{ parameter_dict['curl-location'] }}
-
-[instance-create-erp5-site-real]
-recipe = slapos.recipe.template:jinja2
-template = {{ parameter_dict['template-create-erp5-site-real'] }}
-rendered = ${buildout:directory}/instance-create-erp5-site-real.cfg
-extensions = jinja2.ext.do
-mode = 644
-context =
-  import urlparse urlparse
-  section publish publish
-  section publish_early publish-early
-  section parameter_dict instance-create-erp5-site-real-parameters
-  key eggs_directory buildout:eggs-directory
-  key develop_eggs_directory buildout:develop-eggs-directory
-
-[instance-create-erp5-site-real-run]
-recipe = slapos.recipe.build
-script =
-  import subprocess, sys
-  subprocess.check_call([
-    sys.argv[0],
-    "buildout:directory=${buildout:directory}",
-    "buildout:installed=.installed-${:_buildout_section_name_}.cfg",
-    "-Uoc", self.options['run'],
-  ])
-run = ${instance-create-erp5-site-real:rendered}
-slapos_promise =
-
-[buildout]
-extends = {{ parameter_dict['instance-erp5'] }}
-parts +=
-  instance-create-erp5-site-real-run

stack/erp5/instance-zope.cfg.in

@@ -222,6 +222,23 @@ once = ${:rendered}_done
 [zope-conf-parameter-base]
 ip = {{ ipv4 }}
 site-id = {{ site_id }}
+{% if site_id -%}
+mysql-url = {{ slapparameter_dict['mysql-url-list'][0] }}
+inituser = {{ slapparameter_dict['inituser-login'] }}
+{%  set mysql = urlparse.urlsplit(slapparameter_dict['mysql-url-list'][0]) -%}
+sql-connection-string = {{ '%s@erp5-catalog-0:%s %s %s' % (
+    mysql.path.split('/')[1], mysql.port, mysql.username, mysql.password) }}
+bt5 = {{ slapparameter_dict['bt5'] }}
+bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }}
+home = ${buildout:directory}
+# We only want to change the hostname to 'erp5-cloudooo' if we use the internal
+# cloudooo. We plan to remove the ability to have an internal one, so this
+# heuristic is enough.
+{%  set cloudooo = urlparse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
+cloudooo-url = {{ (cloudooo if cloudooo.port == None else
+  cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
+
+{% endif -%}
 {% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
 {% for name, zodb in zodb_dict.iteritems() -%}
 {%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
@@ -249,7 +266,6 @@ deadlock-debugger-password = {{ dumps(slapparameter_dict['deadlock-debugger-pass
 tidstorage-ip = {{ dumps(slapparameter_dict['tidstorage-ip']) }}
 tidstorage-port = {{ dumps(slapparameter_dict['tidstorage-port']) }}
 {% endif -%}
-promise-path = ${erp5-promise:promise-path}
 {% set thread_amount = slapparameter_dict['thread-amount'] -%}
 thread-amount = {{ thread_amount }}
 {% set webdav = slapparameter_dict['webdav'] -%}
@@ -308,8 +324,10 @@ longrequest-logger-file =
 [{{ conf_name }}]
 < = zope-conf-base
 rendered = ${directory:etc}/{{ name }}.conf
+extensions = jinja2.ext.do
 context =
   section parameter_dict {{ conf_parameter_name }}
+  import os os
 
 [{{ section(name) }}]
 < = runzope-base
@@ -381,16 +399,6 @@ hard to guess.
 hosts-dict = {{ dumps(hosts_dict) }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
-[erp5-promise]
-recipe = slapos.cookbook:erp5.promise
-promise-path = ${directory:etc}/erp5promise.cfg
-kumofs-url = {{ dumps(slapparameter_dict['kumofs-url']) }}
-memcached-url = {{ dumps(slapparameter_dict['memcached-url']) }}
-cloudooo-url = {{ dumps(slapparameter_dict['cloudooo-url']) }}
-smtp-url = {{ dumps(slapparameter_dict['smtp-url']) }}
-bt5 = {{ dumps(slapparameter_dict['bt5']) }}
-bt5-repository-url = {{ dumps(slapparameter_dict['bt5-repository-url']) }}
-
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ next_port() }}
@@ -402,6 +410,5 @@ extends =
   {{ logrotate_cfg }}
   {{ parameter_dict['template-monitor'] }}
 parts +=
-  erp5-promise
   {{ part_list | join('\n  ') }}
   publish

stack/erp5/instance.cfg.in

@@ -194,19 +194,6 @@ filename = instance-mariadb.cfg
 extra-context =
     section parameter_dict dynamic-template-mariadb-parameters
 
-[dynamic-template-create-erp5-site-parameters]
-instance-erp5 = ${dynamic-template-erp5:rendered}
-template-create-erp5-site-real = {{ template_create_erp5_site_real }}
-dash-location = {{ dash_location }}
-curl-location = {{ curl_location }}
-
-[dynamic-template-create-erp5-site]
-<= jinja2-template-base
-template = {{ template_create_erp5_site }}
-filename = instance-create-erp5-site.cfg
-extra-context =
-    section parameter_dict dynamic-template-create-erp5-site-parameters
-
 # we need this value to be present in a section,
 # for slapos.cookbook:switch-softwaretype to work
 [dynamic-template-jupyter]
@@ -218,7 +205,6 @@ override = {{ dumps(override_switch_softwaretype |default) }}
 
 # Public software types
 default = dynamic-template-erp5:rendered
-create-erp5-site = dynamic-template-create-erp5-site:rendered
 # BBB
 RootSoftwareInstance = ${:default}
 # Internal software types

stack/erp5/zope.conf.in

@@ -69,10 +69,24 @@ products {{ parameter_dict['instance-products'] }}
 </timer-server>
 
 {% endif -%}
-{% set promise_path = parameter_dict['promise-path'] -%}
-{% if promise_path -%}
-<product-config /{{ parameter_dict['site-id'] }}>
-  promise_path {{ promise_path }}
+{% set sql_connection_string = parameter_dict.get('sql-connection-string') -%}
+{% if sql_connection_string -%}
+{%   set bt5_repository_url = [] -%}
+{%   set sr_path = os.path.realpath(parameter_dict['home'] + '/software_release') + '/' -%}
+{%   for url in parameter_dict['bt5-repository-url'].split() -%}
+{%     if url.startswith(sr_path) -%}
+{%       set url = '~/software_release/' + url[len(sr_path):] -%}
+{%     endif -%}
+{%     do bt5_repository_url.append(url) -%}
+{%   endfor -%}
+<product-config initsite>
+  id {{ parameter_dict['site-id'] }}
+  owner {{ parameter_dict['inituser'] }}
+  erp5_sql_connection_string {{ sql_connection_string }}
+  cmf_activity_sql_connection_string {{ sql_connection_string }}
+  bt5_repository_url {{ ' '.join(bt5_repository_url) }}
+  bt5 {{ parameter_dict['bt5'] }}
+  cloudooo_url {{ parameter_dict['cloudooo-url'] }}
 </product-config>
 
 {% endif -%}