ERP5: new 'private-dev-shm' zope partition option for wendelin.core

Current version of wendelin.core is limited in that there's no cooperation
in memory allocation between several processes sharing the same /dev/shm.
Over time, the distribution of memory among processes becomes more and more
inequal until some of them get killed due to allocation failure.

This is a temporary solution that relies on user namespaces.

slapos/recipe/librecipe/execute.py

@@ -28,8 +28,25 @@ def _wait_files_creation(file_list):
         if event.name in directory:
           directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
 
+def _libc():
+  from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util
+  libc = CDLL(util.find_library('c'), use_errno=True)
+  libc_mount = libc.mount
+  libc_mount.argtypes = c_char_p, c_char_p, c_char_p, c_ulong, c_char_p
+  def mount(source, target, filesystemtype, mountflags, data):
+    if libc_mount(source, target, filesystemtype, mountflags, data):
+      e = get_errno()
+      raise OSError(e, os.strerror(e))
+  libc_unshare = libc.unshare
+  libc_unshare.argtypes = c_int,
+  def unshare(flags):
+    if libc_unshare(flags):
+      e = get_errno()
+      raise OSError(e, os.strerror(e))
+  return mount, unshare
+
 def generic_exec(args, extra_environ=None, wait_list=None,
-                 pidfile=None, reserve_cpu=False,
+                 pidfile=None, reserve_cpu=False, private_dev_shm=None,
                  #shebang_workaround=False, # XXX: still needed ?
                  ):
   args = list(args)
@@ -62,6 +79,18 @@ def generic_exec(args, extra_environ=None, wait_list=None,
   if wait_list:
     _wait_files_creation(wait_list)
 
+  if private_dev_shm:
+    mount, unshare = _libc()
+    CLONE_NEWNS   = 0x00020000
+    CLONE_NEWUSER = 0x10000000
+    uid = os.getuid()
+    gid = os.getgid()
+    unshare(CLONE_NEWUSER |CLONE_NEWNS)
+    with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
+    with open('/proc/self/uid_map',   'wb') as f: f.write('%s %s 1' % (uid, uid))
+    with open('/proc/self/gid_map',   'wb') as f: f.write('%s %s 1' % (gid, gid))
+    mount('tmpfs', '/dev/shm', 'tmpfs', 0, 'size=' + private_dev_shm)
+
   if extra_environ:
     env = os.environ.copy()
     env.update(extra_environ)

slapos/recipe/wrapper.py

@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe):
 
     :param lines wait-for-files: list of files to wait for
     :param str pidfile: path to pidfile ensure exclusivity for the process
+    :param str private-dev-shm: size of private /dev/shm, using user namespaces
     :param bool reserve-cpu: command will ask for an exclusive CPU core
     """
     def install(self):
@@ -44,6 +45,7 @@ class Recipe(GenericBaseRecipe):
         wrapper_path = self.options['wrapper-path']
         wait_files = self.options.get('wait-for-files')
         pidfile = self.options.get('pidfile')
+        private_dev_shm = self.options.get('private-dev-shm')
 
         environment = {}
         for line in (self.options.get('environment') or '').splitlines():
@@ -57,6 +59,8 @@ class Recipe(GenericBaseRecipe):
           kw['wait_list'] = wait_files.split()
         if pidfile:
           kw['pidfile'] = pidfile
+        if private_dev_shm:
+          kw['private_dev_shm'] = private_dev_shm
         if self.isTrueValue(self.options.get('reserve-cpu')):
           kw['reserve_cpu'] = True
 

software/erp5/instance-erp5-input-schema.json

@@ -119,6 +119,10 @@
               "default": 5,
               "type": "number"
             },
+            "private-dev-shm": {
+              "description": "Size of private /dev/shm for wendelin.core. If sysctl kernel.unprivileged_userns_clone exists, it must be set to 1.",
+              "type": "string"
+            },
             "ssl-authentication": {
               "title": "Enable SSL Client authentication on this zope instance.",
               "description": "If set to true, will set SSL Client verification to required on apache VirtualHost which allow to access this zope instance.",

stack/erp5/buildout.hash.cfg

@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 02ed5d9b74c70789004d01dd2ecde7b1
+md5sum = 1d6735a803c9d28930bf2ad00706c06b
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -87,7 +87,7 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = fd7e8c507cef1950e6c0347ce2a01021
+md5sum = e08c00d5973916d796bf08aa78dba34a
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -205,6 +205,7 @@ name = {{ partition_name }}
 {{ root_common.sla(partition_name) }}
 config-name = {{ dumps(custom_name) }}
 config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
+config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
 config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
 config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}

stack/erp5/instance-zope.cfg.in

@@ -206,6 +206,7 @@ environment =
 {% endif %}
 parameters-extra = true
 command-line = '{{ parameter_dict['userhosts'] }}' '{{ bin_directory }}/runzope' -C '${:configuration-file}'
+private-dev-shm = {{ slapparameter_dict['private-dev-shm'] }}
 
 [{{ section('zcml') }}]
 recipe = slapos.cookbook:copyfilelist

[Roque]

After this change, when I requested a erp5 instance without the new private-dev-shm parameter, the instantiation failed here: https://lab.nexedi.com/nexedi/slapos/blob/master/stack/erp5/instance-zope.cfg.in#L225, while processing "Installing dynamic-template-zope." for the zope partition, due to: UndefinedError: 'dict object' has no attribute 'private-dev-shm'. As workaround, I requested the instance with the new parameter set in "".

[Julien Muchembled]

Your workaround makes no sense to me. It looks a coincidence.

[Julien Muchembled]

An explanation might be that node instance considered there was no change (despite the upgrade of SR) and skipped the root partition. And when you changed parameters (no matter how), it forced reinstanciation of the root partition and the new parameter was propagated to the zope partition.

If you remove the parameter you added, it would still work.

[Roque]

Yes, probably something like that occurred. I re-requested the instance and it is working fine. Sorry for the inconvenience.