NEO: new schema for node parameters

software/neoppod/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance-common]
 filename = instance-common.cfg.in
-md5sum = 0a3a54fcc7be0bbd63cbd64f006ceebc
+md5sum = 80599fcc6e5d07270d7900aebfd62139
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = ccc6e33412259415ec6c3452d37b77cc
+md5sum = c03fbfc9df9edc1ef60be970e0627c5e
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -38,7 +38,7 @@ md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
 
 [template-neo]
 filename = instance.cfg.in
-md5sum = 83dc9faca482b2ddbd3d5fa968af7c33
+md5sum = 9e63e16eda75e73ad4ffb50afde0505d
 
 [cluster]
 filename = cluster.cfg.in

software/neoppod/instance-common.cfg.in

@@ -15,7 +15,9 @@ cert = ${slap-connection:cert-file}
 recipe = slapos.recipe.template:jinja2
 filename = ${:_buildout_section_name_}.cfg
 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
-extensions = jinja2.ext.do
+extensions =
+    jinja2.ext.do
+    jinja2.ext.loopcontrols
 extra-context =
 context =
     key ipv4_set slap-configuration:ipv4

software/neoppod/instance-neo-input-schema.json

 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
   "additionalProperties": false,
   "required": [
@@ -72,71 +72,95 @@
     "_key": {
       "type": "string"
     },
-    "node-list": {
-      "description": "List of dictionaries containing parameters for each node.",
-      "items": {
-        "description": "Dictionary containing parameters required to configure individual nodes.",
-        "default": {},
-        "additionalProperties": false,
-        "properties": {
-          "admin": {
-            "description": "Port of admin node. 0 to disable.",
-            "default": 2050,
-            "type": "integer"
-          },
-          "master": {
-            "description": "Port of master node. 0 to disable.",
-            "default": 2051,
-            "type": "integer"
-          },
-          "storage-count": {
-            "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
-            "default": 1,
-            "type": "integer"
-          },
-          "data-deduplication": {
-            "description": "Set the --dedup option for storage nodes.",
-            "default": false,
-            "type": "boolean"
-          },
-          "disable-drop-partitions": {
-            "description": "Set the --disable-drop-partitions option for storage nodes.",
-            "default": false,
-            "type": "boolean"
-          },
-          "storage-type": {
-            "description": "Storage type. Defaults to MySQL if available, else SQLite.",
-            "enum": [
-              "MySQL",
-              "SQLite"
-            ],
-            "type": "string"
-          },
-          "mysql": {
-            "description": "Dictionary containing parameters for MySQL.",
-            "default": {},
-            "properties": {
-              "relaxed-writes": {
-                "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
-                "default": false,
-                "type": "boolean"
-              }
+        "node-count": {
+          "description": "Number of nodes to deploy.",
+          "default": 1,
+          "type": "integer"
+        },
+        "node": {
+          "description": "Default node parameters.",
+          "default": {},
+          "additionalProperties": false,
+          "properties": {
+            "admin": {
+              "description": "Port of admin node. 0 to disable.",
+              "default": 2050,
+              "type": "integer"
+            },
+            "master": {
+              "description": "Port of master node. 0 to disable.",
+              "default": 2051,
+              "type": "integer"
+            },
+            "storage-count": {
+              "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
+              "default": 1,
+              "type": "integer"
             },
-            "additionalProperties": {
-              "description": "To configure important parameters like innodb_buffer_pool_size, tokudb_cache_size, etc.",
+            "data-deduplication": {
+              "description": "Set the --dedup option for storage nodes.",
+              "default": false,
+              "type": "boolean"
+            },
+            "disable-drop-partitions": {
+              "description": "Set the --disable-drop-partitions option for storage nodes.",
+              "default": false,
+              "type": "boolean"
+            },
+            "storage-type": {
+              "description": "Storage type. Defaults to MySQL if available, else SQLite.",
+              "enum": [
+                "MySQL",
+                "SQLite"
+              ],
               "type": "string"
             },
-            "type": "object"
+            "mysql": {
+              "description": "Dictionary containing parameters for MySQL.",
+              "default": {},
+              "properties": {
+                "relaxed-writes": {
+                  "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
+                  "default": false,
+                  "type": "boolean"
+                }
+              },
+              "additionalProperties": {
+                "description": "To configure important parameters like innodb_buffer_pool_size, tokudb_cache_size, etc.",
+                "type": "string"
+              },
+              "type": "object"
+            },
+            "engine": {
+              "description": "Configures storage engine, currently only InnoDB, RocksDB and TokuDB are supported. Defaults to NEO's default.",
+              "type": "string"
+            }
           },
-          "engine": {
-            "description": "Configures storage engine, currently only InnoDB, RocksDB and TokuDB are supported. Defaults to NEO's default.",
-            "type": "string"
-          }
+          "type": "object"
         },
-        "type": "object"
-      },
-      "type": "array"
-    }
+        "node!": {
+          "description": "Node parameters are overridden by the value of the first item whose partition reference matches against the pattern.",
+          "items": {
+            "items": [
+              {
+                "description": "Override pattern (Python regular expression).",
+                "type": "string"
+              },
+              {
+                "allOf": [
+                  {
+                    "$ref": "#/properties/node"
+                  },
+                  {
+                    "description": "Override value (parameters for maching nodes)."
+                  }
+                ]
+              }
+            ],
+            "type": "array"
+          },
+          "type": "array"
+        }
   },
   "type": "object"
 }

software/neoppod/instance.cfg.in

@@ -8,6 +8,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
 <= jinja2-template-base
 template = {{ cluster }}
 extra-context =
+    import re re
     import urlparse urlparse
 import-list =
     rawfile root_common {{ root_common }}

software/neoppod/root-common.cfg.in

@@ -16,6 +16,20 @@ sla-computer_guid = ${slap-connection:computer-id}
 {% endif -%}
 {% endmacro -%}
 
+{% macro apply_overrides(dict_, reference) -%}
+{% for key in list(dict_) -%}
+{%   if key.endswith('!') -%}
+{%     for pattern, value in dict_.pop(key, ()) -%}
+{%       set m = re.match(pattern, reference) -%}
+{%       if m and m.group() == reference %}{# PY3: fullmatch -#}
+{%         do dict_.__setitem__(key[:-1], value) -%}
+{%         break -%}
+{%       endif -%}
+{%     endfor -%}
+{%   endif -%}
+{% endfor -%}
+{% endmacro -%}
+
 {% macro common_section() -%}
 [request-common-base]
 recipe = slapos.cookbook:request.serialised
@@ -52,7 +66,22 @@ config-ssl = {{ dumps((
 config-upstream-cluster = {{ dumps(parameter_dict.get('upstream-cluster', '')) }}
 config-upstream-masters = {{ dumps(parameter_dict.get('upstream-masters', '')) }}
 software-type = {{ software_type }}
-{% set node_list = parameter_dict.get('node-list', ({},)) -%}
+
+{% set node_list = parameter_dict.get('node-list') -%}
+{% if node_list == None -%}
+{%   set node_list = [] -%}
+{%   for i in range(parameter_dict.get('node-count', 1)) -%}
+{%     set x = parameter_dict.copy() -%}
+{%     do apply_overrides(x, prefix ~ i) -%}
+{%     do node_list.append(x.get('node', {})) -%}
+{%   endfor -%}
+{% else %}{# BBB -#}
+{%   do assert('node-count' not in parameter_dict) -%}
+{%   do assert('node' not in parameter_dict) -%}
+{%   do assert('node!' not in parameter_dict) -%}
+{% endif -%}
+{% do assert(node_list) -%}
+
 {% set storage_count = [] -%}
 {% for node in node_list -%}
 {%   do storage_count.append(node.get('storage-count', 1)) -%}

stack/erp5/buildout.hash.cfg

@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = f6c6820f9b3653d0d5c29708606fc591
+md5sum = d53a7dbc374374b5c312e3f7e2c0a0ff
 
 [monitor-template-dummy]
 filename = dummy.cfg

stack/erp5/instance.cfg.in

@@ -46,6 +46,7 @@ extra-context =
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
+    import re re
     import urlparse urlparse
 import-list =
     file root_common context:root-common