publish_early: new '-update' option, keep published values out of buildout installed file

The new option will be used to fix/optimize
propagation of parameters in the NEO SR.

slapos/recipe/publish_early.py

@@ -25,9 +25,23 @@
 #
 ##############################################################################
 
-import slapos.slap
-from slapos.recipe.librecipe import unwrap, wrap
-from slapos.recipe.librecipe import GenericSlapRecipe
+from collections import defaultdict
+from .librecipe import unwrap, wrap, GenericSlapRecipe
+
+def patchOptions(options, override):
+  def get(option, *args, **kw):
+    try:
+      return override[option]
+    except KeyError:
+      return options_get(option, *args, **kw)
+  try:
+    options_get = options._get
+  except AttributeError:
+    options_get = options.get
+    options.get = get
+  else:
+    options._get = get
+
 
 class Recipe(GenericSlapRecipe):
   """
@@ -43,6 +57,8 @@ class Recipe(GenericSlapRecipe):
     -init =
       foo gen-foo:x
       bar gen-bar:y
+    -update =
+      baz update-baz:z
     bar = z
 
     [gen-foo]
@@ -58,31 +74,75 @@ class Recipe(GenericSlapRecipe):
   (and in this case, it is published immediately as a way to save the value).
   ${publish-early:bar} is forced to 'z' (${gen-bar:y} ignored):
   a line like 'bar = z' is usually rendered conditionally with Jinja2.
+
+  The '-update' option has the same syntax than '-init'. The recipes of the
+  specified sections must implement 'publish_early(publish_dict)':
+  - it is always called, just before early publishing
+  - publish_dict is a dict with already published values
+  - 'publish_early' can change published values by modifying publish_dict.
+
+  In the above example:
+  - publish_dict is {'z': ...}
+  - during the execution of 'publish_early', other sections can access the
+    value with ${update-baz:z}
+  - once [publish-early] is initialized, the value should be accessed with
+    ${publish-early:bar} ([update-baz] does not have it if it's accessed
+    before [publish-early])
   """
   def __init__(self, buildout, name, options):
     GenericSlapRecipe.__init__(self, buildout, name, options)
-    published_dict = None
-    publish = False
-    publish_dict = {}
-    for line in options['-init'].splitlines():
+    init = defaultdict(dict)
+    update = defaultdict(dict)
+    for d, k in (init, '-init'), (update, '-update'):
+      for line in options.get(k, '').splitlines():
         if line:
           k, v = line.split()
           if k not in options:
-          if published_dict is None:
+            section, v = v.split(':')
+            d[section][k] = v
+    if init or update:
       self.slap.initializeConnection(self.server_url, self.key_file,
         self.cert_file)
       computer_partition = self.slap.registerComputerPartition(
         self.computer_id, self.computer_partition_id)
-            published_dict = unwrap(
-              computer_partition.getConnectionParameterDict())
+      published_dict = unwrap(computer_partition.getConnectionParameterDict())
+
+      publish = False
+      publish_dict = {}
+      for section, init in init.iteritems():
+        for k, v in init.iteritems():
           try:
             publish_dict[k] = published_dict[k]
           except KeyError:
-            section, key = v.split(":")
-            publish_dict[k] = self.buildout[section][key]
+            publish_dict[k] = buildout[section][v]
+            publish = True
+
+      for section, update in update.iteritems():
+        override = {}
+        for k, v in update.iteritems():
+          try:
+            override[v] = published_dict[k]
+          except KeyError:
+            pass
+        section = buildout[section]
+        patchOptions(section, override)
+        old = override.copy()
+        section.recipe.publish_early(override)
+        if override != old:
           publish = True
+        for k, v in update.iteritems():
+          try:
+            publish_dict[k] = override[v]
+          except KeyError:
+            pass
+
       if publish:
         computer_partition.setConnectionDict(wrap(publish_dict))
-    options.update(publish_dict)
+
+      publish = [k for k in options
+        if k != 'recipe' and not k.startswith('-')]
+      publish += publish_dict
+      publish_dict['-publish'] = ' '.join(publish)
+      patchOptions(options, publish_dict)
 
   install = update = lambda self: None

[Julien Muchembled]

The patchOptions function is broken. It fails when the value is not a string. For example, this breaks the expansion of https://lab.nexedi.com/nexedi/slapos/blob/ba6c33312bfca1dc1bccfb6d410059b900c787ef/stack/erp5/instance-mariadb.cfg.in#L147

[Rafael Monnerat]

Maybe it would be better start a test suite on this one, to cover the basics maybe.