stack/erp5 += WCFS service (technology preview)

See merge request !939

software/erp5/instance-erp5-input-schema.json

@@ -477,6 +477,16 @@
       },
       "type": "object"
     },
+    "wcfs": {
+      "description": "Parameters for wendelin.core filesystem",
+      "properties": {
+	"enable": {
+	  "description": "Whether to enable WCFS filesystem and use it to access ZBigArray/ZBigFile data. In WCFS mode wendelin.core clients (Zope/ERP5 processes) share in-RAM cache for in-ZODB data without duplicating it for every client. This cache sharing does not affect correctness as isolation property is continued to be provided to every client.",
+	  "default": false,
+	  "type": "boolean"
+	}
+      }
+    },
     "wendelin-core-zblk-fmt": {
       "description": "In wendelin.core there are 2 formats for storing data, so called ZBlk0 and ZBlk1. See https://lab.nexedi.com/nexedi/wendelin.core/blob/2e5e1d3d/bigfile/file_zodb.py#L19 for more details.",
       "default": "",

software/erp5/test/test/test_wcfs.py

+# Copyright (C) 2021  Nexedi SA and Contributors.
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import json
+import os.path
+import unittest
+
+from slapos.grid.utils import md5digest
+
+from . import ERP5InstanceTestCase, setUpModule as _setUpModule
+from .test_erp5 import TestPublishedURLIsReachableMixin
+
+# skip tests when software release is built with wendelin.core 1.
+def setUpModule():
+  _setUpModule()
+
+  cls = ERP5InstanceTestCase
+  if not os.path.exists(
+    os.path.join(
+      cls.slap.software_directory,
+      md5digest(cls.getSoftwareURL()),
+      'bin', 'wcfs')):
+    raise unittest.SkipTest("built with wendelin.core 1")
+
+
+class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test Wendelin Core File System
+  """
+  __partition_reference__ = 'wcfs'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'wcfs': {'enable': True}})}
+
+  def test_wcfs_accessible(self):
+    """Verify that wcfs filesystem is basically accessible.
+
+       - we can read .wcfs/zurl
+       - its content is equal to published `serving-zurl`
+    """
+    zurl = json.loads(
+             self.getComputerPartition('wcfs').getConnectionParameter('_')
+           )['serving-zurl']
+
+    mntpt = lookupMount(zurl)
+    zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
+    self.assertEqual(zurl_, zurl)
+
+
+# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
+def lookupMount(zurl):
+  for line in readfile('/proc/mounts').splitlines():
+    # <zurl> <mountpoint> fuse.wcfs ...
+    zurl_, mntpt, typ, _ = line.split(None, 3)
+    if typ != 'fuse.wcfs':
+      continue
+    if zurl_ == zurl:
+      return mntpt
+  raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
+
+# readfile returns content of file @path.
+def readfile(path):
+  with open(path, 'r') as f:
+    return f.read()

software/slapos-master/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = f078d72db690805eab90a4577066b920
+md5sum = eba73f868a5bbfb942323eadcd88cb16
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -9,6 +9,8 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
+{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
 {% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
 {% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
 {% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
 {%   endif -%}
 {% endfor -%}
 
+[request-zodb-base]
+config-zodb-dict = {{ dumps(zodb_dict) }}
+{% for server_type, server_dict in storage_dict.iteritems() -%}
+{%   if server_type == 'neo' -%}
+config-neo-cluster = ${publish-early:neo-cluster}
+config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-masters = ${publish-early:neo-masters}
+{%   else -%}
+config-zodb-zeo = ${request-zodb:connection-storage-dict}
+config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
+config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
+{%   endif -%}
+{% endfor -%}
+
 
 {% set zope_address_list_id_dict = {} -%}
 {% if zope_partition_dict -%}
 
 [request-zope-base]
 <= request-common
+   request-zodb-base
 return =
   zope-address-list
   hosts-dict
@@ -233,20 +250,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
 config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
 config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
-config-zodb-dict = {{ dumps(zodb_dict) }}
 config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
 config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
-{%   if server_type == 'neo' -%}
-config-neo-cluster = ${publish-early:neo-cluster}
-config-neo-name = {{ server_dict.keys()[0] }}
-config-neo-masters = ${publish-early:neo-masters}
-{%   else -%}
-config-zodb-zeo = ${request-zodb:connection-storage-dict}
-config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
-config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-{%   endif -%}
-{% endfor -%}
+config-wcfs_enable = {{ wcfs_enable }}
 software-type = zope
 
 {% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
@@ -348,6 +354,17 @@ config-url = ${request-jupyter:connection-url}
 {%   endif -%}
 {%- endif %}
 
+{% if wcfs_enable -%}
+{# request WCFS connected to ZODB -#}
+{%   do root_common.section('request-wcfs') -%}
+{{   request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
+
+[request-wcfs]
+<= request-common
+   request-zodb-base
+
+{%- endif %}
+
 [directory]
 recipe = slapos.cookbook:mkdirectory
 {% if slapparameter_dict.get('shared-certificate-authority-path', '') -%}

stack/erp5/buildout.cfg

@@ -185,6 +185,7 @@ context =
     key instance_common_cfg instance-common:rendered
     key jsl_location jsl:location
     key jupyter_enable_default erp5-defaults:jupyter-enable-default
+    key wcfs_enable_default erp5-defaults:wcfs-enable-default
     key kumo_location kumo:location
     key local_bt5_repository local-bt5-repository:list
     key logrotate_location logrotate:location
@@ -222,7 +223,9 @@ context =
     key template_postfix_aliases template-postfix-aliases:target
     key template_postfix_main_cf template-postfix-main-cf:target
     key template_postfix_master_cf template-postfix-master-cf:target
+    key instance_wcfs_cfg_in instance-wcfs.cfg.in:target
     key template_zeo template-zeo:target
+    key template_zodb_base template-zodb-base:target
     key template_zope template-zope:target
     key template_zope_conf template-zope-conf:target
     key template_fonts_conf template-fonts-conf:output
@@ -237,6 +240,9 @@ context =
 [template-zeo]
 <= download-base
 
+[template-zodb-base]
+<= download-base
+
 [template-zope]
 <= download-base
 link-binary =
@@ -281,6 +287,9 @@ fontconfig-includes =
 [template-rsyslogd-cfg]
 <= download-base
 
+[instance-wcfs.cfg.in]
+<= download-base
+
 [erp5-bin]
 <= erp5
 repository = https://lab.nexedi.com/nexedi/erp5-bin.git
@@ -320,6 +329,8 @@ repository_id_list = erp5 erp5-bin erp5-doc
 cloudooo-connection-url = https://cloudooo.erp5.net/
 # Jupyter is by default disabled in ERP5
 jupyter-enable-default = false
+# WCFS is by default disabled in ERP5
+wcfs-enable-default = false
 
 [erp5]
 recipe = slapos.recipe.build:gitclone
@@ -558,6 +569,7 @@ scripts =
   runzeo
   tidstoraged
   tidstorage_repozo
+  wcfs
   web_checker_utility
 
 extra-paths =

stack/erp5/buildout.hash.cfg

@@ -70,19 +70,23 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
 
 [template]
 filename = instance.cfg.in
-md5sum = b5ac16fdeed8863e465e955ba6d1e12a
+md5sum = 8bd7f89b7c1e453ecc952659a01347e6
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = ac155f28a096747fc267f32a1cec46e1
+md5sum = 79f092c54998918423257f4b676425ea
 
 [template-zeo]
 filename = instance-zeo.cfg.in
 md5sum = 0648e38bd5d3a15bb9f93264932740b9
 
+[template-zodb-base]
+filename = instance-zodb-base.cfg.in
+md5sum = bc821f9f9696953b10a03ad7b59a1936
+
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 9fa66b93fbf6a40aa8136c651ad9f539
+md5sum = e495f7225a49f61c501ccc496a976d63
 
 [template-balancer]
 filename = instance-balancer.cfg.in
@@ -95,3 +99,7 @@ md5sum = 3f4f7e49c504cbf610fc5dc462713dfc
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in
 md5sum = 5cf0316fdd17a940031e4083bbededd8
+
+[instance-wcfs.cfg.in]
+filename = instance-wcfs.cfg.in
+md5sum = 945e8e4552a6bdf228b9609567b09399

stack/erp5/instance-erp5.cfg.in

@@ -9,6 +9,8 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
+{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
 {% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
 {% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
 {% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
 {%   endif -%}
 {% endfor -%}
 
+[request-zodb-base]
+config-zodb-dict = {{ dumps(zodb_dict) }}
+{% for server_type, server_dict in storage_dict.iteritems() -%}
+{%   if server_type == 'neo' -%}
+config-neo-cluster = ${publish-early:neo-cluster}
+config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-masters = ${publish-early:neo-masters}
+{%   else -%}
+config-zodb-zeo = ${request-zodb:connection-storage-dict}
+config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
+config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
+{%   endif -%}
+{% endfor -%}
+
 
 {% set zope_address_list_id_dict = {} -%}
 {% if zope_partition_dict -%}
 
 [request-zope-base]
 <= request-common
+   request-zodb-base
 return =
   zope-address-list
   hosts-dict
@@ -238,20 +255,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
 config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
 config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
-config-zodb-dict = {{ dumps(zodb_dict) }}
 config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
 config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
-{%   if server_type == 'neo' -%}
-config-neo-cluster = ${publish-early:neo-cluster}
-config-neo-name = {{ server_dict.keys()[0] }}
-config-neo-masters = ${publish-early:neo-masters}
-{%   else -%}
-config-zodb-zeo = ${request-zodb:connection-storage-dict}
-config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
-config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-{%   endif -%}
-{% endfor -%}
+config-wcfs_enable = {{ wcfs_enable }}
 software-type = zope
 
 {% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
@@ -353,6 +359,17 @@ config-url = ${request-jupyter:connection-url}
 {%   endif -%}
 {%- endif %}
 
+{% if wcfs_enable -%}
+{# request WCFS connected to ZODB -#}
+{%   do root_common.section('request-wcfs') -%}
+{{   request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
+
+[request-wcfs]
+<= request-common
+   request-zodb-base
+
+{%- endif %}
+
 {% set balancer_ret_dict = {'monitor-base-url': False} -%}
 {% for family in zope_family_dict -%}
 {%   do balancer_ret_dict.__setitem__(family, False) -%}

stack/erp5/instance-wcfs.cfg.in

+{# instance that runs WCFS service associated with ZODB storage  #}
+{% from "instance_zodb_base" import zodb_dict  with context %}
+
+
+{# build zurl to connect to configured ZODB #}
+{% if len(zodb_dict) != 1 -%}
+{%   do assert(False, ("WCFS supports only single ZODB storage", zodb_dict)) -%}
+{% endif -%}
+
+{% set db_name, zodb = zodb_dict.popitem() -%}
+{% set z = zodb['storage-dict'] -%}
+{% if zodb['type'] == 'zeo' -%}
+{%   set zurl = ('zeo://%s?storage=%s' % (z['server'], z['storage'])) -%}
+{% elif zodb['type'] == 'neo' -%}
+{%   set zurl = ('neo://%s@%s' % (z.pop('name'), z.pop('master_nodes'))) -%}
+{%   set argv = [] -%}
+{%   set i = 0 -%}
+{%   for k,v in z|dictsort -%}
+{%     do argv.append('%s=%s' % (k,v)) -%}
+{%   endfor -%}
+{%   if len(argv) > 0 -%}
+{%     set zurl = zurl + '&' + '?'.join(argv) -%}
+{%   endif -%}
+{% else -%}
+{%   do assert(False, ("unsupported ZODB type", zodb)) -%}
+{% endif -%}
+
+
+[buildout]
+extends = {{ template_monitor }}
+parts +=
+  wcfs
+  wcfs-promise
+  publish
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+log = ${:var}/log
+run = ${:var}/run
+services = ${:etc}/run
+service-on-watch = ${:etc}/service
+srv = ${buildout:directory}/srv
+tmp = ${buildout:directory}/tmp
+var = ${buildout:directory}/var
+
+[wcfs]
+recipe = slapos.cookbook:wrapper
+command-line = {{ bin_directory }}/wcfs serve -log_dir=${directory:log} {{ zurl }}
+wrapper-path = ${directory:service-on-watch}/wcfs
+
+[wcfs-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = ${:_buildout_section_name_}.py
+config-command = {{ bin_directory }}/wcfs status {{ zurl }}
+
+[publish]
+recipe = slapos.cookbook:publish.serialised
+serving-zurl = {{ zurl }}

stack/erp5/instance-zodb-base.cfg.in

+{# base for instances that need to access ZODB storage #}
+{# provides zodb_dict #}
+
+{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
+
+{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
+{% for name, zodb in zodb_dict.iteritems() -%}
+{%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
+{%   if zodb['type'] == 'zeo' -%}
+{%     do storage_dict.update(zeo_dict.get(name, ())) -%}
+{%   else -%}
+{%     if name == slapparameter_dict.get('neo-name') -%}
+{%       do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
+                                name=slapparameter_dict['neo-cluster']) -%}
+{%     endif -%}
+{{     assert(storage_dict['master_nodes'], name) }}
+{%     if storage_dict.pop('ssl', 1) -%}
+{%       do storage_dict.update(ca='~/etc/ca.crt',
+                                cert='~/etc/neo.crt',
+                                key='~/etc/neo.key') -%}
+{%     endif -%}
+{%   endif -%}
+{% endfor -%}

stack/erp5/instance-zope.cfg.in

+{% from "instance_zodb_base" import zodb_dict  with context %}
 {% set wsgi = slapparameter_dict['wsgi'] -%}
 {% set webdav = slapparameter_dict['webdav'] -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 {% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
 {% set site_id = slapparameter_dict['site-id'] -%}
-{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
 {% set instance_index_list = range(slapparameter_dict['instance-count']) -%}
 {% set node_id_base = slapparameter_dict['name'] -%}
 {% set selenium_server_configuration_dict = slapparameter_dict.get('selenium-server-configuration-dict', None) -%}
@@ -63,6 +63,12 @@ environment +=
   FONTCONFIG_FILE=${fontconfig-conf:rendered}
 {% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
   WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
+{% endif %}
+  WENDELIN_CORE_WCFS_AUTOSTART=no
+{% if slapparameter_dict['wcfs_enable'] %}
+  WENDELIN_CORE_VIRTMEM=r:wcfs+w:uvmm
+{% else %}
+  WENDELIN_CORE_VIRTMEM=rw:uvmm
 {% endif %}
   ${:environment-extra}
 
@@ -257,24 +263,6 @@ cloudooo-url = {{ (cloudooo if cloudooo.port == None else
   cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
 
 {% endif -%}
-{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
-{% for name, zodb in zodb_dict.iteritems() -%}
-{%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
-{%   if zodb['type'] == 'zeo' -%}
-{%     do storage_dict.update(zeo_dict.get(name, ())) -%}
-{%   else -%}
-{%     if name == slapparameter_dict.get('neo-name') -%}
-{%       do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
-                                name=slapparameter_dict['neo-cluster']) -%}
-{%     endif -%}
-{{     assert(storage_dict['master_nodes'], name) }}
-{%     if storage_dict.pop('ssl', 1) -%}
-{%       do storage_dict.update(ca='~/etc/ca.crt',
-                                cert='~/etc/neo.crt',
-                                key='~/etc/neo.key') -%}
-{%     endif -%}
-{%   endif -%}
-{% endfor -%}
 developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
 publisher-timeout = {{ dumps(slapparameter_dict['publisher-timeout']) }}
 activity-timeout = {{ dumps(slapparameter_dict['activity-timeout']) }}

stack/erp5/instance.cfg.in

@@ -31,11 +31,13 @@ extra-context =
 [dynamic-template-erp5-parameters]
 default-cloudooo-url =  {{ dumps(default_cloudooo_url) }}
 jupyter-enable-default = {{ jupyter_enable_default }}
+wcfs-enable-default = {{ wcfs_enable_default }}
 local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
 
 [context]
 root-common = {{ root_common }}
 caucase-jinja2-library = {{ caucase_jinja2_library }}
+template-zodb-base = {{ template_zodb_base }}
 
 [dynamic-template-erp5]
 <= jinja2-template-base
@@ -44,6 +46,7 @@ filename = instance-erp5.cfg
 extra-context =
     key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
+    key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
     import re re
@@ -125,6 +128,8 @@ extra-context =
     import urlparse urlparse
     import hashlib hashlib
     import itertools itertools
+import-list =
+    file instance_zodb_base context:template-zodb-base
 
 [dynamic-template-kumofs-parameters]
 <= default-dynamic-template-parameters
@@ -190,6 +195,15 @@ mode = 644
 [dynamic-template-jupyter]
 rendered = {{ template_jupyter_cfg }}
 
+[dynamic-template-wcfs]
+<= jinja2-template-base
+template = {{ instance_wcfs_cfg_in }}
+filename = instance_wcfs.cfg
+extra-context =
+    section parameter_dict dynamic-template-zope-parameters
+import-list =
+    file instance_zodb_base context:template-zodb-base
+
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 override = {{ dumps(override_switch_softwaretype |default) }}
@@ -208,6 +222,7 @@ zodb-zeo = dynamic-template-zeo:rendered
 zodb-neo = neo:rendered
 zope = dynamic-template-zope:rendered
 jupyter = dynamic-template-jupyter:rendered
+wcfs = dynamic-template-wcfs:rendered
 
 # Keep cloudooo backward compatibility
 cloudooo = dynamic-template-legacy:output