Update Release Candidate

CHANGES.rst

 Changes
 =======
+
+1.0.118 (2019-08-13)
+--------------------
+
+* NEO: new recipe to fix/optimize propagation of the 'masters' parameter
+* publish_early: new '-update' option, keep published values out of buildout installed file
+* publish: new -publish option to list explicitly options to publish
+* re6stnet: Fix typo
+* librecipe: Try to reuse existing file to avoid excessive IO on update and other minor optimisations
+* certificate_authority: unique_subject = no
+* wrapper: handle "=" in environment variables' content
+ 
+
 1.0.92 (2019-02-21)
 -------------------
 

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.92'
+version = '1.0.118'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read() + "\n" + \
     open("CHANGES.rst").read() + "\n"
@@ -134,6 +134,7 @@ setup(name=name,
           'mydumper = slapos.recipe.mydumper:Recipe',
           'mysql = slapos.recipe.mysql:Recipe',
           'nbdserver = slapos.recipe.nbdserver:Recipe',
+          'neoppod.cluster = slapos.recipe.neoppod:Cluster',
           'neoppod.admin = slapos.recipe.neoppod:Admin',
           'neoppod.master = slapos.recipe.neoppod:Master',
           'neoppod.storage = slapos.recipe.neoppod:Storage',

slapos/recipe/neoppod.py

@@ -26,8 +26,38 @@
 ##############################################################################
 import os
 import shlex
-from slapos.recipe.librecipe import GenericBaseRecipe
 from zc.buildout import UserError
+from .librecipe import GenericBaseRecipe
+
+class Cluster(object):
+
+  def __init__(self, buildout, name, options):
+    self.buildout = buildout
+    self.options = options
+
+  def publish_early(self, publish_dict):
+    masters = publish_dict.setdefault('masters', '')
+    result_dict = {
+      'connection-admin': [],
+      'connection-master': [],
+    }
+    node_list = []
+    for node in sorted(self.options['nodes'].split()):
+      node = self.buildout[node]
+      node_list.append(node)
+      for k, v in result_dict.iteritems():
+        x = node[k]
+        if x:
+          v.append(x)
+    publish_dict['admins'] = ' '.join(result_dict.pop('connection-admin'))
+    x = ' '.join(result_dict.pop('connection-master'))
+    if masters != x:
+      publish_dict['masters'] = x
+      for node in node_list:
+        node['config-masters'] = x
+        node.recipe.__init__(self.buildout, node.name, node)
+
+  install = update = lambda self: None
 
 class NeoBaseRecipe(GenericBaseRecipe):
 

slapos/recipe/publish.py

@@ -44,9 +44,14 @@ class Recipe(GenericSlapRecipe):
   def _install(self):
     publish_dict = {}
     for name in self._extend_set:
-      for k, v in self.buildout[name].iteritems():
-        if k != 'recipe' and not k.startswith('-'):
-          publish_dict[k] = v
+      section = self.buildout[name]
+      try:
+        publish = section['-publish'].split()
+      except KeyError:
+        publish = (k for k in section
+          if k != 'recipe' and not k.startswith('-'))
+      for k in publish:
+        publish_dict[k] = section[k]
     self._setConnectionDict(publish_dict, self.options.get('-slave-reference'))
     return []
 

slapos/recipe/publish_early.py

@@ -25,9 +25,23 @@
 #
 ##############################################################################
 
-import slapos.slap
-from slapos.recipe.librecipe import unwrap, wrap
-from slapos.recipe.librecipe import GenericSlapRecipe
+from collections import defaultdict
+from .librecipe import unwrap, wrap, GenericSlapRecipe
+
+def patchOptions(options, override):
+  def get(option, *args, **kw):
+    try:
+      return override[option]
+    except KeyError:
+      return options_get(option, *args, **kw)
+  try:
+    options_get = options._get
+  except AttributeError:
+    options_get = options.get
+    options.get = get
+  else:
+    options._get = get
+
 
 class Recipe(GenericSlapRecipe):
   """
@@ -43,6 +57,8 @@ class Recipe(GenericSlapRecipe):
     -init =
       foo gen-foo:x
       bar gen-bar:y
+    -update =
+      baz update-baz:z
     bar = z
 
     [gen-foo]
@@ -58,31 +74,75 @@ class Recipe(GenericSlapRecipe):
   (and in this case, it is published immediately as a way to save the value).
   ${publish-early:bar} is forced to 'z' (${gen-bar:y} ignored):
   a line like 'bar = z' is usually rendered conditionally with Jinja2.
+
+  The '-update' option has the same syntax than '-init'. The recipes of the
+  specified sections must implement 'publish_early(publish_dict)':
+  - it is always called, just before early publishing
+  - publish_dict is a dict with already published values
+  - 'publish_early' can change published values by modifying publish_dict.
+
+  In the above example:
+  - publish_dict is {'z': ...}
+  - during the execution of 'publish_early', other sections can access the
+    value with ${update-baz:z}
+  - once [publish-early] is initialized, the value should be accessed with
+    ${publish-early:bar} ([update-baz] does not have it if it's accessed
+    before [publish-early])
   """
   def __init__(self, buildout, name, options):
     GenericSlapRecipe.__init__(self, buildout, name, options)
-    published_dict = None
-    publish = False
-    publish_dict = {}
-    for line in options['-init'].splitlines():
+    init = defaultdict(dict)
+    update = defaultdict(dict)
+    for d, k in (init, '-init'), (update, '-update'):
+      for line in options.get(k, '').splitlines():
         if line:
           k, v = line.split()
           if k not in options:
-          if published_dict is None:
+            section, v = v.split(':')
+            d[section][k] = v
+    if init or update:
       self.slap.initializeConnection(self.server_url, self.key_file,
         self.cert_file)
       computer_partition = self.slap.registerComputerPartition(
         self.computer_id, self.computer_partition_id)
-            published_dict = unwrap(
-              computer_partition.getConnectionParameterDict())
+      published_dict = unwrap(computer_partition.getConnectionParameterDict())
+
+      publish = False
+      publish_dict = {}
+      for section, init in init.iteritems():
+        for k, v in init.iteritems():
           try:
             publish_dict[k] = published_dict[k]
           except KeyError:
-            section, key = v.split(":")
-            publish_dict[k] = self.buildout[section][key]
+            publish_dict[k] = buildout[section][v]
+            publish = True
+
+      for section, update in update.iteritems():
+        override = {}
+        for k, v in update.iteritems():
+          try:
+            override[v] = published_dict[k]
+          except KeyError:
+            pass
+        section = buildout[section]
+        patchOptions(section, override)
+        old = override.copy()
+        section.recipe.publish_early(override)
+        if override != old:
           publish = True
+        for k, v in update.iteritems():
+          try:
+            publish_dict[k] = override[v]
+          except KeyError:
+            pass
+
       if publish:
         computer_partition.setConnectionDict(wrap(publish_dict))
-    options.update(publish_dict)
+
+      publish = [k for k in options
+        if k != 'recipe' and not k.startswith('-')]
+      publish += publish_dict
+      publish_dict['-publish'] = ' '.join(publish)
+      patchOptions(options, publish_dict)
 
   install = update = lambda self: None

software/agent/software.cfg

@@ -44,7 +44,6 @@ eggs =
 
 [versions]
 ecdsa = 0.13
-erp5.util = 0.4.51
 gitdb = 0.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0

software/build-rina/software.cfg

@@ -84,5 +84,4 @@ packages +=
   python-dev
 
 [versions]
-erp5.util = 0.4.51
 slapos.recipe.template = 4.3

software/buildout-testing/software.cfg

@@ -59,5 +59,4 @@ template =
     raw runTestSuite_py ${buildout:bin-directory}/${runTestSuite_py:interpreter}
 
 [versions]
-erp5.util = 0.4.51
 slapos.recipe.template = 4.3

software/cdn-me/software.cfg

@@ -38,6 +38,5 @@ eggs =
 [versions]
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
-erp5.util = 0.4.51
 plone.recipe.command = 1.1
 slapos.recipe.template = 4.3

software/erp5testnode/software.cfg

@@ -61,7 +61,6 @@ output = ${buildout:directory}/template-default.cfg
 mode = 0644
 
 [versions]
-erp5.util = 0.4.58
 slapos.recipe.template = 4.3
 dnspython = 1.15.0
 PyXML = 0.8.5

software/erp5testnode/testsuite/deploy-test/software.cfg

@@ -73,6 +73,3 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/template.cfg
 mode = 0644
-
-[versions]
-erp5.util = 0.4.51

software/jstestnode/software.cfg

@@ -115,7 +115,7 @@ output = ${buildout:directory}/template-nginx.cfg.in
 output = ${buildout:directory}/runTestSuite.in
 
 [versions]
-erp5.util = 0.4.51
+
 slapos.recipe.template = 4.3
 selenium = 3.14.1
 urllib3 = 1.24

software/kvm/software.cfg

@@ -200,7 +200,6 @@ context =
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-erp5.util = 0.4.51
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4
 pycurl = 7.43.0

software/neoppod/software.cfg

@@ -35,7 +35,6 @@ eggs = erp5.util
 interpreter = ${:_buildout_section_name_}
 
 [versions]
-erp5.util = 0.4.51
 # To match ERP5
 ZConfig = 2.9.3
 zc.lockfile = 1.0.2

software/neotest/software.cfg

@@ -124,7 +124,6 @@ ZODB3 = 3.11.0
 numpy = 1.16.4
 zope.testing = 4.6.2
 pygolang = 0.0.0.dev4
-erp5.util = 0.4.51
 
 # Required by:
 # ZEO==4.3.1

software/nextcloud/apache-httpd.conf.in

@@ -25,4 +25,7 @@
         Dav off
       </IfModule>
   </Directory>
+  ErrorLog "{{ parameter_dict['log-dir'] }}/nextcloud-error.log"
+  CustomLog "{{ parameter_dict['log-dir'] }}/nextcloud-access.log" combined
+
 </VirtualHost>
\ No newline at end of file

software/nextcloud/nextcloud-instance.cfg.in

@@ -131,7 +131,7 @@ mode = 744
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = nextcloud-backup
-frequency = 0 0 * * * *
+frequency = 0 0 * * *
 command = ${nextcloud-backup:output}
 
 [nextcloud-optimize]

software/nextcloud/software.cfg

@@ -24,7 +24,7 @@ md5sum = a2281f86f6a26a8ff40a57a495505977
 [template-apache-httpd]
 <= nc-download-base
 filename = apache-httpd.conf.in
-md5sum = f3bca64bf991526fd8221035a86aacbf
+md5sum = 839258624e273aac71a96516bf34c7e6
 
 [template-nextcloud-config.json]
 <= nc-download-base
@@ -36,7 +36,7 @@ recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/nextcloud-instance.cfg.in
 rendered = ${buildout:directory}/instance-nextcloud.cfg
 extensions = jinja2.ext.do
-md5sum = 5c721c7eb8a06147b849a65ba92becba
+md5sum = 59e9c65e655cf9cf144d97dd36863ede
 context = 
           key gzip_location          gzip:location
           key python3_location       python3.6.6:location

software/nginx-push-stream/software.cfg

@@ -46,7 +46,6 @@ mode = 0644
 
 [versions]
 slapos.recipe.template = 4.3
-erp5.util = 0.4.51
 inotifyx = 0.2.2
 gitdb2 = 2.0.3
 smmap2 = 2.0.3

software/powerdns/software.cfg

@@ -67,7 +67,6 @@ cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 slapos.recipe.template = 4.3
 dnspython = 1.15.0
-erp5.util = 0.4.53
 passlib = 1.7.1
 GitPython = 2.1.11
 lockfile = 0.12.2

software/slapos-master/software-shared.cfg

+[buildout]
+extends = software.cfg
+shared-parts = /opt/slapgrid/shared-parts
+eggs-directory = /opt/slapgrid/shared-eggs
+abi-tag-eggs = true

software/slapos-sr-testing/software.cfg

@@ -142,9 +142,6 @@ output = ${buildout:directory}/template.cfg
 mode = 640
 
 [versions]
-# Recent erp5.util is needed
-erp5.util = 0.4.58
-
 # Various needed versions
 Pillow = 5.3.0
 PyNaCl = 1.3.0

software/wendelin/software-shared.cfg

+[buildout]
+extends = software.cfg
+shared-parts = /opt/slapgrid/shared-parts
+eggs-directory = /opt/slapgrid/shared-eggs
+abi-tag-eggs = true

software/wendelin/software.cfg

@@ -8,6 +8,7 @@ extends =
   ../../software/erp5/software.cfg
 parts +=
   wendelin
+  erp5-bin
   scipy
   msgpack-python
   msgpack-numpy
@@ -60,10 +61,14 @@ initialization =
   ${testrunner:initialization}
  
 [erp5_repository_list]
-repository_id_list += wendelin
+repository_id_list += 
+  wendelin
+  erp5-bin
 
 [local-bt5-repository]
-list += ${wendelin:location}/bt5
+list += 
+  ${wendelin:location}/bt5
+  ${erp5-bin:location}/bt5
 
 # Jupyter is by default enabled in Wendelin
 [erp5-defaults]
@@ -75,6 +80,12 @@ git-executable = ${git:location}/bin/git
 repository = https://lab.nexedi.com/nexedi/wendelin.git
 branch = master
 
+[erp5-bin]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+repository = https://lab.nexedi.com/nexedi/erp5-bin.git
+branch = master
+
 [versions]
 msgpack = 0.6.1
 msgpack-numpy = 0.4.4.3

stack/cloudooo.cfg

@@ -91,7 +91,3 @@ slapos.recipe.template = 4.3
 # PasteScript==2.0
 # cloudooo==1.2.5.dev0
 PasteDeploy = 1.5.2
-
-# Required by:
-# cloudooo==1.2.5.dev0
-erp5.util = 0.4.51

stack/erp5/buildout.hash.cfg

@@ -78,7 +78,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 7dd00dedef4cc4320ec6977a7e2dc110
+md5sum = 079244ece03662c7ceadb68570b69d2f
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -397,11 +397,11 @@ recipe = slapos.cookbook:publish-early
 {%- if has_posftix %}
   smtpd-sasl-password gen-smtpd-sasl-password:passwd
 {%- endif %}
-{% if test_runner_enabled -%}
-{%   for zope_family_name in zope_family_name_list %}
+{%- if test_runner_enabled %}
+{%-   for zope_family_name in zope_family_name_list %}
   {{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default
-{%   endfor -%}
-{% endif -%}
+{%-   endfor %}
+{%- endif %}
 {%- if neo %}
   neo-cluster gen-neo-cluster:name
 {%-  if neo[0] %}

stack/lamp/buildout.cfg

@@ -119,7 +119,7 @@ context =
 <= template-download-base
 filename = instance-apache-php.cfg.in
 output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
-md5sum = 7edcbca22e800f2c90559298c44d5a48
+md5sum = 44796786448fd0319dde14923fbd798b
 
 [instance-lamp]
 <= template-download-base

stack/lamp/instance-apache-php.cfg.in

@@ -129,6 +129,7 @@ port = ${apache-network-configuration:listening-port}
 url = https://[${:ip}]:${:port}/
 error-log = ${directory:httpd-log}/error.log
 access-log = ${directory:httpd-log}/access.log
+log-dir = ${directory:httpd-log}
 php-ini-dir = ${directory:php-ini-dir}
 cert-file = ${ca-directory:certs}/httpd.crt
 key-file = ${ca-directory:certs}/httpd.key
@@ -153,7 +154,7 @@ wait-for-files =
 
 [apache-graceful]
 recipe = collective.recipe.template
-output = ${directory:bin}/apache-httpd-graceful
+output = ${directory:scripts}/apache-httpd-graceful
 mode = 700
 input = inline:
   #!/bin/sh
@@ -162,7 +163,7 @@ input = inline:
 [logrotate-entry-apache]
 <= logrotate-entry-base
 name = apache
-log = ${apache-php-configuration:error-log} ${apache-php-configuration:access-log}
+log = ${apache-php-configuration:log-dir}/*.log
 frequency = daily
 rotate-num = 30
 

stack/slapos.cfg

@@ -136,8 +136,8 @@ pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
 six = 1.11.0
-slapos.cookbook = 1.0.92
-slapos.core = 1.4.25
+slapos.cookbook = 1.0.118
+slapos.core = 1.4.26
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.19
@@ -154,7 +154,7 @@ CacheControl = 0.12.5
 msgpack = 0.6.1
 
 # Required by:
-# slapos.core==1.4.25
+# slapos.core==1.4.26
 Flask = 0.12
 
 # Required by:
@@ -195,7 +195,7 @@ enum34 = 1.1.6
 
 # Required by:
 # slapos.toolbox==0.94
-erp5.util = 0.4.51
+erp5.util = 0.4.59
 
 # Required by:
 # slapos.toolbox==0.94
@@ -218,7 +218,7 @@ pyrsistent = 0.14.5
 ipaddress = 1.0.18
 
 # Required by:
-# slapos.cookbook==1.0.92
+# slapos.cookbook==1.0.118
 jsonschema = 3.0.0a3
 
 # Required by:
@@ -226,7 +226,7 @@ jsonschema = 3.0.0a3
 lockfile = 0.12.2
 
 # Required by:
-# slapos.core==1.4.25
+# slapos.core==1.4.26
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
@@ -259,7 +259,7 @@ python-dateutil = 2.7.3
 rpdb = 0.1.5
 
 # Required by:
-# slapos.core==1.4.25
+# slapos.core==1.4.26
 supervisor = 3.3.3
 
 # Required by:
@@ -267,11 +267,11 @@ supervisor = 3.3.3
 tzlocal = 1.5.1
 
 # Required by:
-# slapos.core==1.4.25
+# slapos.core==1.4.26
 uritemplate = 3.0.0
 
 # Required by:
-# slapos.core==1.4.25
+# slapos.core==1.4.26
 zope.interface = 4.3.3
 
 [networkcache]