Revert "stack/erp5: completely remove cloudooo"

This reverts commit 40108537.

Using the public cloudooo for all unit tests makes things
unreliable, we have every day many tests failing due to this. It
would be required to reapply this change to :
- make ERP5 more resilient to network failures when there is
  issues connecting to cloudooo
- find some automated ways to have a cloudooo per testnode machine,
  with idea of having possibility to request "where is the
  nearest service doing that"
- possibly having a CDN of cloudooo, but this does not prevent having
  a cloudooo per testnode

software/erp5/README.rst

@@ -18,8 +18,9 @@ front-end instance which is expected to contains the RewriteRules_ (or
 equivalent) needed to relocate Zope's urls via its VirtualHostMonster_. See the
 ``frontend`` erp5 instance parameter.
 
-ERP5 defaults connect to the public cloudooo on https://cloudooo.erp5.net/.
-See the ``cloudooo`` Software Release to setup a cloudooo cluster if necessary.
+Included cloudooo partition is **deprecated**. It is not recommended for
+intensive usage. See the ``cloudooo`` Software Release to setup a cloudooo
+cluster, more suitable for intensive usage.
 
 Replication
 ===========
@@ -87,6 +88,7 @@ This software release assigns the following port ranges by default:
   ====================  ==========
   memcached-persistent  2000-2009
   memcached-volatile    2010-2019
+  cloudooo              2020-2024
   smtp                  2025-2029
   neo (admin & master)  2050-2051
   mariadb               2099

software/erp5/instance-cloudooo-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "required": ["tcpv4-port"],
+  "properties": {
+    "tcpv4-port": {
+      "allOf": [{
+        "$ref": "#/definitions/tcpv4port"
+      }, {
+        "description": "Start allocating ports at this value, going upward"
+      }]
+    }
+  }
+}

software/erp5/instance-erp5-input-schema.json

@@ -160,10 +160,12 @@
       },
       "type": "object"
     },
-    "cloudooo-url": {
-      "description": "Format conversion service URL",
-      "pattern": "^https?://",
-      "type": "string"
+    "cloudooo": {
+      "description": "Format conversion service",
+      "additionalProperties": {
+        "$ref": "./instance-cloudooo-schema.json#/properties"
+      },
+      "type": "object"
     },
     "smtp": {
       "description": "Mail queuing and relay service",

software/erp5/instance-erp5-output-schema.json

@@ -39,6 +39,11 @@
       "pattern": "^memcached://",
       "type": "string"
     },
+    "cloudooo-url": {
+      "description": "Conversion service access information - DEPRECATED",
+      "pattern": "^(http|https)://",
+      "type": "string"
+    },
     "mariadb-database-list": {
       "description": "Relational database access information",
       "items": {

stack/erp5/buildout.cfg

@@ -24,6 +24,7 @@ extends =
   ../../component/libdmtx/buildout.cfg
   ../../component/libffi/buildout.cfg
   ../../component/libpng/buildout.cfg
+  ../../component/libreoffice-bin/buildout.cfg
   ../../component/matplotlib/buildout.cfg
   ../../component/mesa/buildout.cfg
   ../../component/numpy/buildout.cfg
@@ -56,6 +57,7 @@ extends =
   ../../component/bash/buildout.cfg
   ../../component/wget/buildout.cfg
   ../../component/aspell/buildout.cfg
+  ../../component/cloudooo/buildout.cfg
   ../../component/jsl/buildout.cfg
   ../../component/6tunnel/buildout.cfg
   ../../component/findutils/buildout.cfg
@@ -91,6 +93,7 @@ parts +=
   libdmtx
   dmtx-utils
   kumo
+  libreoffice-bin
   w3-validator
   tesseract
   scipy
@@ -124,6 +127,10 @@ parts +=
   ocrb-fonts
   android-fonts
 
+# Cloudooo specific part
+  cloudooo-develop
+  cloudooo
+
 # get git repositories
   erp5
   genbt5list
@@ -185,6 +192,9 @@ link-binary =
 [template-kumofs]
 <= download-base
 
+[template-cloudooo]
+<= download-base
+
 [template-zope-conf]
 <= download-base
 
@@ -233,7 +243,6 @@ context =
     key dbus_glib_location dbus-glib:location
     key dbus_location dbus:location
     key dcron_location dcron:location
-    key default_cloudooo_url erp5-defaults:cloudooo-connection-url
     key erp5_location erp5:location
     key file_location file:location
     key findutils_location findutils:location
@@ -259,6 +268,7 @@ context =
     key libexpat_location libexpat:location
     key libffi_location libffi:location
     key libpng12_location libpng12:location
+    key libreoffice_bin_location libreoffice-bin:location
     key librsvg_location librsvg:location
     key libxcb_location libxcb:location
     key local_bt5_repository local-bt5-repository:list
@@ -281,6 +291,7 @@ context =
     key stunnel_location stunnel:location
     key template_apache_conf template-apache-backend-conf:target
     key template_balancer template-balancer:target
+    key template_cloudooo template-cloudooo:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
     key template_jupyter_cfg instance-jupyter-notebook:rendered
@@ -365,7 +376,6 @@ repository_id_list = erp5
 
 # ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
 [erp5-defaults]
-cloudooo-connection-url = https://cloudooo.erp5.net/
 # Jupyter is by default disabled in ERP5
 jupyter-enable-default = false
 
@@ -592,6 +602,10 @@ scripts = zodbanalyze
 recipe = zc.recipe.egg:develop
 setup = ${erp5:location}
 
+[cloudooo-repository]
+branch =
+revision = 00604a3a5a69082f7c607665cc6badd5104a32b9
+
 [slapos-deps-eggs]
 recipe = zc.recipe.egg
 eggs +=
@@ -616,6 +630,9 @@ pysvn = 1.7.10+SlapOSPatched002
 python-ldap = 2.4.32+SlapOSPatched001
 python-magic = 0.4.12+SlapOSPatched001
 
+# specify dev version to be sure that an old released version is not used
+cloudooo = 1.2.5-dev
+
 # use newer version than specified in ZTK
 PasteDeploy = 1.5.2
 argparse = 1.4.0

stack/erp5/buildout.hash.cfg

@@ -29,6 +29,10 @@ md5sum = 5c11abdef0c84cdef91f999490786244
 filename = instance-kumofs.cfg.in
 md5sum = 091d3c3feb2d543d176b0fadb11c07dc
 
+[template-cloudooo]
+filename = instance-cloudoo.cfg.in
+md5sum = 76f9e8c8cdc352081e34539d8fc17026
+
 [template-zope-conf]
 filename = zope.conf.in
 md5sum = bbea91bc8f0b2d455d9824928abfad5b
@@ -63,7 +67,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template]
 filename = instance.cfg.in
-md5sum = 9adb0aea4248a21bfe4dd48df82f864d
+md5sum = f95174eee2de45d1e3ccb1d5f88355f1
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -71,7 +75,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 41fb33a46826712bca404d77cf50c795
+md5sum = 909c8eb4f1f2f2d58ad982cec67228bc
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-cloudoo.cfg.in

+{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
+{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
+[buildout]
+extends =
+  {{ parameter_dict['template-monitor'] }}
+parts +=
+  publish
+  cloudooo-instance
+  resiliency-exclude-file
+  promise
+  promise-openoffice
+  {% if use_ipv6 %}promise-tunnel{% endif %}
+  xvfb-instance
+  wkhtmltopdf-on-xvfb
+
+[publish]
+recipe = slapos.cookbook:publish.serialised
+{% if use_ipv6 -%}
+url = http://[${ipv6toipv4:ipv6}]:${ipv6toipv4:ipv6-port}/
+{% else -%}
+url = http://${cloudooo-instance:ip}:${cloudooo-instance:port}/
+{% endif -%}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+
+[cloudooo-instance]
+recipe = slapos.cookbook:generic.cloudooo
+
+# Network options
+ip = {{ (ipv4_set | list)[0] }}
+{% set tcpv4_port = slapparameter_dict['tcpv4-port'] -%}
+port = {{ tcpv4_port }}
+openoffice-port = {{ tcpv4_port + 1 }}
+
+# Paths
+configuration-file = ${directory:etc}/cloudooo.cfg
+wrapper = ${directory:service}/cloudooo
+
+# Paths: Data
+data-directory = ${directory:cloudooo-data}
+
+environment =
+  LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
+  FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
+  PATH = ${binary-link:target-directory}
+
+# Binary information
+# cloudooo specific configuration
+ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
+ooo-paster = {{ bin_directory }}/cloudooo_paster
+ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
+
+[fontconfig-instance]
+recipe = slapos.cookbook:fontconfig
+conf-path = ${directory:etc}/font.conf
+font-system-folder = {{ parameter_dict['fonts'] }}
+font-folder = ${directory:font}
+service-folder = ${directory:service}
+
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = {{ 'inline:{{ "**\\n" }}' }}
+rendered = ${directory:srv}/exporter.exclude
+
+[promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/cloudooo
+hostname = ${cloudooo-instance:ip}
+port = ${cloudooo-instance:port}
+
+[promise-openoffice]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/openoffice
+hostname = ${cloudooo-instance:ip}
+port = ${cloudooo-instance:openoffice-port}
+
+{% if use_ipv6 -%}
+[promise-tunnel]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/tunnel
+hostname = ${ipv6toipv4:ipv6}
+port = ${ipv6toipv4:ipv6-port}
+
+[ipv6toipv4]
+recipe = slapos.cookbook:ipv6toipv4
+runner-path = ${directory:service}/${:base-name}
+6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel
+shell-path = {{ parameter_dict['dash'] }}/bin/dash
+ipv4 = ${cloudooo-instance:ip}
+ipv6 = {{ (ipv6_set | list)[0] }}
+ipv6-port = ${cloudooo-instance:port}
+ipv4-port = ${cloudooo-instance:port}
+base-name = cloudooo-tunnel
+
+{% endif -%}
+# rest of parts are candidates for some generic stuff
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+run = ${:var}/run
+service = ${:etc}/run
+promise = ${:etc}/promise
+cloudooo-data = ${:srv}/cloudooo
+font = ${:srv}/font
+framebuffer = ${:srv}/framebuffer
+
+[monitor-instance-parameter]
+monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
+monitor-httpd-port = {{ tcpv4_port + 2 }}
+monitor-title = {{ slapparameter_dict['name'] }}
+password = {{ slapparameter_dict['monitor-passwd'] }}
+
+[binary-link]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:bin}
+link-binary =
+  {{ parameter_dict['coreutils'] }}/bin/basename
+  {{ parameter_dict['coreutils'] }}/bin/cat
+  {{ parameter_dict['coreutils'] }}/bin/cp
+  {{ parameter_dict['coreutils'] }}/bin/ls
+  {{ parameter_dict['coreutils'] }}/bin/tr
+  {{ parameter_dict['coreutils'] }}/bin/uname
+  {{ parameter_dict['coreutils'] }}/bin/dirname
+# wrapper recipe needs the head command
+  {{ parameter_dict['coreutils'] }}/bin/head
+  {{ parameter_dict['imagemagick'] }}/bin/convert
+  {{ parameter_dict['imagemagick'] }}/bin/identify
+  {{ parameter_dict['poppler'] }}/bin/pdfinfo
+  {{ parameter_dict['poppler'] }}/bin/pdftotext
+  {{ parameter_dict['poppler'] }}/bin/pdftohtml
+  {{ parameter_dict['onlyoffice-x2t'] }}/x2t
+
+[xvfb-instance]
+recipe = slapos.cookbook:xvfb
+runner-path = ${directory:service}/xvfb
+xvfb-path = {{ parameter_dict["xserver"] }}/bin/Xvfb
+fbdir-path = ${directory:framebuffer}
+tmp-path = ${directory:run}
+shell-path = {{ parameter_dict["dash"] }}/bin/dash
+
+[wkhtmltopdf-on-xvfb]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/wkhtmltopdf
+environment =
+  DISPLAY=:0
+  TMPDIR=${directory:run}
+command-line = {{ parameter_dict['wkhtmltopdf'] }}/wkhtmltopdf --use-xserver
+parameters-extra = true

stack/erp5/instance-erp5.cfg.in

@@ -44,6 +44,7 @@ config-name = {{ name }}
 
 {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {% if has_posftix -%}
 {{   request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
@@ -149,7 +150,7 @@ return =
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-caucase-url = ${request-caucase:connection-http-url}
-config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
+config-cloudooo-url = ${request-cloudooo:connection-url}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
 config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}

stack/erp5/instance.cfg.in

@@ -6,6 +6,52 @@ extends =
 [jinja2-template-base]
 mode = 644
 
+[dynamic-template-cloudooo-parameters]
+cairo = {{ cairo_location }}
+coreutils = {{ dumps(coreutils_location) }}
+cups = {{ cups_location }}
+dbus = {{ dbus_location }}
+dbus-glib = {{ dbus_glib_location }}
+file = {{ file_location }}
+fontconfig = {{ fontconfig_location }}
+freetype = {{ freetype_location }}
+glib = {{ glib_location }}
+glu = {{ glu_location }}
+imagemagick = {{ dumps(imagemagick_location) }}
+libICE = {{ libICE_location }}
+libSM = {{ libSM_location }}
+libX11 = {{ libX11_location }}
+libXau = {{ libXau_location }}
+libXdmcp = {{ libXdmcp_location }}
+libXext = {{ libXext_location }}
+libXrender = {{ libXrender_location }}
+libexpat = {{ libexpat_location }}
+libffi = {{ libffi_location }}
+libpng12 = {{ libpng12_location }}
+libxcb = {{ libxcb_location }}
+mesa = {{ mesa_location }}
+onlyoffice-x2t = {{ dumps(onlyoffice_x2t_location) }}
+pixman = {{ pixman_location }}
+poppler = {{ dumps(poppler_location) }}
+wkhtmltopdf = {{ dumps(wkhtmltopdf_location) }}
+xdamage = {{ xdamage_location }}
+xfixes = {{ xfixes_location }}
+xserver = {{ dumps(xserver_location) }}
+zlib = {{ zlib_location }}
+libreoffice-bin = {{ libreoffice_bin_location }}
+fonts = {{ fonts_location }}
+buildout-bin-directory = {{ buildout_bin_directory }}
+6tunnel = {{ sixtunnel_location }}
+dash = {{ dash_location }}
+template-monitor = {{ dumps(template_monitor) }}
+
+[dynamic-template-cloudooo]
+<= jinja2-template-base
+template = {{ template_cloudooo }}
+filename = instance-cloudoo.cfg
+extra-context =
+    section parameter_dict dynamic-template-cloudooo-parameters
+
 [dynamic-template-postfix-parameters]
 bin-directory = {{ bin_directory }}
 buildout-bin-directory = {{ buildout_bin_directory }}
@@ -26,7 +72,6 @@ extra-context =
     import urllib urllib
 
 [dynamic-template-erp5-parameters]
-default-cloudooo-url =  {{ dumps(default_cloudooo_url) }}
 jupyter-enable-default = {{ jupyter_enable_default }}
 local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
 template-monitor = {{ dumps(template_monitor) }}
@@ -36,7 +81,6 @@ template-monitor = {{ dumps(template_monitor) }}
 template = {{ template_erp5 }}
 filename = instance-erp5.cfg
 extra-context =
-    key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key template_monitor dynamic-template-erp5-parameters:template-monitor
@@ -166,6 +210,7 @@ RootSoftwareInstance = ${:default}
 # Internal software types
 kumofs = dynamic-template-kumofs:rendered
 caucase = dynamic-template-caucase:rendered
+cloudooo = dynamic-template-cloudooo:rendered
 mariadb = dynamic-template-mariadb:rendered
 balancer = dynamic-template-balancer:rendered
 postfix = dynamic-template-postfix:rendered