ERP5: new parameters to override ZODB settings for any zope

software/erp5/instance-erp5-input-schema.json

@@ -257,7 +257,7 @@
       "type": "object"
     },
     "zodb": {
-      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
+      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/4/src/ZODB/component.xml for extra options.",
       "items": {
         "required": [
           "type"
@@ -282,8 +282,13 @@
                 "type": "boolean"
               }
             },
+            "patternProperties": {
+              ".!$": {
+                "$ref": "#/properties/zodb/items/patternProperties/.!$"
+              }
+            },
             "additionalProperties": {
-              "type": "string"
+              "$ref": "#/properties/zodb/items/additionalProperties"
             },
             "type": "object"
           },
@@ -318,8 +323,27 @@
             }
           }
         ],
+        "patternProperties": {
+          ".!$": {
+            "description": "Override with the value of the first item whose zope id matches against the pattern.",
+            "items": {
+              "items": [
+                {
+                  "description": "Override pattern (Python regular expression).",
+                  "type": "string"
+                },
+                {
+                  "description": "Override value (parameter for maching nodes).",
+                  "type": ["integer", "string"]
+                }
+              ],
+              "type": "array"
+            },
+            "type": "array"
+          }
+        },
         "additionalProperties": {
-          "type": "string"
+          "type": ["integer", "string"]
         },
         "type": "object"
       },

software/erp5/test/test/__init__.py

@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
     """Return the output paramters from the root partition"""
     return json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
+
+  def getComputerPartition(self, partition_reference):
+    for computer_partition in self.slap.computer.getComputerPartitionList():
+      if partition_reference == computer_partition.getInstanceParameter(
+          'instance_title'):
+        return computer_partition
+
+  def getComputerPartitionPath(self, partition_reference):
+    partition_id = self.getComputerPartition(partition_reference).getId()
+    return os.path.join(self.slap._instance_root, partition_id)

software/erp5/test/test/test_erp5.py

@@ -195,8 +195,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     """
     # self.computer_partition_root_path is the path of root partition.
     # we want to assert that no scripts exist in any partition.
-    bin_programs = [os.path.basename(path) for path in
-      glob.glob("{}/../*/bin/*".format(self.computer_partition_root_path))]
+    bin_programs = map(os.path.basename,
+      glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
 
     self.assertTrue(bin_programs) # just to check the glob was correct.
     self.assertNotIn('runUnitTest', bin_programs)
@@ -210,8 +210,99 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     apache_process = psutil.Process(process_info['pid'])
     self.assertEqual(
         sorted([socket.AF_INET, socket.AF_INET6]),
-        sorted([
+        sorted(
             c.family
             for c in apache_process.connections()
             if c.status == 'LISTEN'
-        ]))
+        ))
+
+class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test override zope node parameters
+  """
+  __partition_reference__ = 'override'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    # The following example includes the most commonly used options,
+    # but not necessarily in a meaningful way.
+    return {'_': json.dumps({
+      "zodb": [{
+        "type": "zeo",
+        "server": {},
+        "cache-size-bytes": "20MB",
+        "cache-size-bytes!": [
+          ("bb-0", 1<<20),
+          ("bb-.*", "500MB"),
+        ],
+        "pool-timeout": "10m",
+        "storage-dict": {
+          "cache-size!": [
+            ("a-.*", "50MB"),
+          ],
+        },
+      }],
+      "zope-partition-dict": {
+          "a": {
+              "instance-count": 3,
+          },
+          "bb": {
+              "instance-count": 5,
+              "port-base": 2300,
+          },
+      },
+    })}
+
+  def test_zope_conf(self):
+    zeo_addr = json.loads(
+        self.getComputerPartition('zodb').getConnectionParameter('_')
+      )["storage-dict"]["root"]["server"]
+
+    def checkParameter(line, kw):
+      k, v = line.split()
+      self.assertFalse(k.endswith('!'), k)
+      try:
+        expected = kw.pop(k)
+      except KeyError:
+        if k == 'server':
+          return
+      self.assertIsNotNone(expected)
+      self.assertEqual(str(expected), v)
+
+    def checkConf(zodb, storage):
+      zodb["mount-point"] = "/"
+      zodb["pool-size"] = 4
+      zodb["pool-timeout"] = "10m"
+      storage["storage"] = "root"
+      storage["server"] = zeo_addr
+      with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
+        conf = map(str.strip, f.readlines())
+      i = conf.index("<zodb_db root>") + 1
+      conf = iter(conf[i:conf.index("</zodb_db>", i)])
+      for line in conf:
+        if line == '<zeoclient>':
+          for line in conf:
+            if line == '</zeoclient>':
+              break
+            checkParameter(line, storage)
+          for k, v in storage.iteritems():
+            self.assertIsNone(v, k)
+          del storage
+        else:
+          checkParameter(line, zodb)
+      for k, v in zodb.iteritems():
+        self.assertIsNone(v, k)
+
+    partition = self.getComputerPartitionPath('zope-a')
+    for zope in xrange(3):
+      checkConf({
+          "cache-size-bytes": "20MB",
+        }, {
+          "cache-size": "50MB",
+        })
+    partition = self.getComputerPartitionPath('zope-bb')
+    for zope in xrange(5):
+      checkConf({
+          "cache-size-bytes": "500MB" if zope else 1<<20,
+        }, {
+          "cache-size": None,
+        })

stack/erp5/buildout.hash.cfg

@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
 
 [template-zope-conf]
 filename = zope.conf.in
-md5sum = 114e0ac43281b943931754ed317ebc36
+md5sum = 762897486b1e7e28b614224a9a577125
 
 [site-zcml]
 filename = site.zcml
@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = d53a7dbc374374b5c312e3f7e2c0a0ff
+md5sum = 520b6bf3461dddc9c8b862e50b14465d
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = b1685783f4c93da918ccc83702559e6f
+md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -267,6 +267,11 @@ timerserver-interval = {{ dumps(timerserver_interval) }}
 [zope-conf-base]
 < = jinja2-template-base
 template = {{ parameter_dict['zope-conf-template'] }}
+extensions =
+    jinja2.ext.do
+    jinja2.ext.loopcontrols
+import-list =
+    rawfile root_common {{ root_common }}
 
 {% macro zope(
   index,
@@ -312,10 +317,10 @@ longrequest-logger-file =
 [{{ conf_name }}]
 < = zope-conf-base
 rendered = ${directory:etc}/{{ name }}.conf
-extensions = jinja2.ext.do
 context =
   section parameter_dict {{ conf_parameter_name }}
   import os os
+  import re re
 
 [{{ section(name) }}]
 < = runzope-base

stack/erp5/instance.cfg.in

@@ -112,6 +112,7 @@ template = {{ template_zope }}
 filename = instance-zope.cfg
 extra-context =
     key buildout_directory buildout:directory
+    key root_common context:root-common
     section parameter_dict dynamic-template-zope-parameters
     import urlparse urlparse
     import hashlib hashlib

stack/erp5/zope.conf.in

+{% set slapparameter_dict = {} %}{# dummy -#}
+{% import "root_common" as root_common with context -%}
+{% set node_id = parameter_dict['node-id'] -%}
 # Note: Environment is setup in running wrapper script, as zope.conf is read
 # too late for some components.
 %define INSTANCE {{ parameter_dict['instance'] }}
@@ -65,7 +68,7 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 
 {% endif -%}
 <product-config CMFActivity>
-  node-id {{ parameter_dict['node-id'] }}
+  node-id {{ node_id }}
 </product-config>
 
 {% set timerserver_interval = parameter_dict['timerserver-interval'] -%}
@@ -145,13 +148,15 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 <zodb_db {{ name }}>
 {%- set storage_type = type_dict[zodb_dict.pop('type')] %}
 {%- set storage_dict = zodb_dict.pop('storage-dict') %}
+{%- do root_common.apply_overrides(zodb_dict, node_id) %}
 {%- for key, value in zodb_dict.iteritems() %}
   {{ key }} {{ value }}
-{%-   endfor %}
+{%- endfor %}
   <{{ storage_type }}>
-{%-   for key, value in storage_dict.iteritems() %}
+{%- do root_common.apply_overrides(storage_dict, node_id) %}
+{%- for key, value in storage_dict.iteritems() %}
     {{ key }} {{ value }}
-{%-   endfor %}
+{%- endfor %}
   </{{ storage_type }}>
 </zodb_db>