Update Release Candidate

CHANGES.rst

 Changes
 =======
+1.0.92 (2019-02-21)
+-------------------
+
+* plugin recipe: improve recipe to correctly generate promise with parameters which contain control characters
 
 1.0.85 (2018-12-28)
 -----------------------

component/groonga/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = false
-url = https://packages.groonga.org/source/groonga/groonga-8.0.6.tar.gz
-md5sum = 532dab4d5d625942852d021283e974ee
+url = https://packages.groonga.org/source/groonga/groonga-9.0.0.tar.gz
+md5sum = 5475818c734dfc6414d209babea90921
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/mariadb/buildout.cfg

@@ -26,8 +26,8 @@ parts =
 [mariadb]
 recipe = slapos.recipe.cmmi
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.2.17
-md5sum = 97dac7c5c288dbbbdd97768972daeb2e
+version = 10.2.22
+md5sum = f390235995b72b4c50948a43eb7e41fe
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.2.16_create_system_tables__no_test.patch#3fd5f9febabdb42d4b6653969a0194f9
@@ -79,8 +79,8 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = https://packages.groonga.org/source/mroonga/mroonga-8.06.tar.gz
-md5sum = 5933363420cb66fcc89e25485072f1b8
+url = https://packages.groonga.org/source/mroonga/mroonga-9.00.tar.gz
+md5sum = a1deff08a3649d8370436f1c903ed432
 pre-configure = set -e
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.85'
+version = '1.0.92'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read() + "\n" + \
     open("CHANGES.rst").read() + "\n"

slapos/recipe/promise_plugin.py

@@ -24,19 +24,18 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import logging, os, sys
+import json
+import re
+import logging, os
 import zc.buildout.easy_install
 from slapos.recipe.librecipe import GenericBaseRecipe
 
 script_template = '''# This script is auto generated by slapgrid, do not edit!
+import json
 import sys
-sys.path[0:0] = [
-  %(path)s
-]
+sys.path[0:0] = %(path)s
 
-extra_config_dict = {
-  %(config)s
-}
+extra_config_dict = json.loads("""%(config)s""", strict=False)
 
 # We want to cleanup all imported modules from slapos namespace, because
 # they will conflict with slapos.core.
@@ -53,8 +52,6 @@ for module in sys.modules.keys():
   if 'slapos' in module or 'pkg_resources' in module:
     del sys.modules[module]
 
-import slapos.grid.promise
-
 %(content)s
 '''
 
@@ -79,11 +76,18 @@ class Recipe(GenericBaseRecipe):
     )
     if cache_storage is None:
       cache_storage = {}
+      try:
         setattr(
           self.buildout,
           self._WORKING_SET_CACHE_NAME,
           cache_storage
         )
+      except AttributeError:
+        if type(self.buildout) == type({}):
+          # failed to set attribute in test mode, cache not used
+          pass
+        else:
+          raise
     return cache_storage
 
   def install(self):
@@ -102,30 +106,43 @@ class Recipe(GenericBaseRecipe):
       develop_eggs_dir,
     )
     if cache_key not in cache_storage:
+      if develop_eggs_dir and eggs_dir:
         working_set = zc.buildout.easy_install.working_set(
           egg_list,
           [develop_eggs_dir, eggs_dir]
         )
         cache_storage[cache_key] = working_set
+      else:
+        working_set = set()
     else:
       working_set = cache_storage[cache_key]
 
-    content = self.options['content'].strip()
+    regex = r"^[\w_\-\.\s]+$"
+    import_path = self.options.get('import', '').strip()
+    if import_path:
+      if not re.search(regex, import_path):
+        raise ValueError("Import path %r is not a valid" % import_path)
+      content_string = "from %s import RunPromise" % import_path
+    else:
+      # old parameter for compatibility
+      content_string = self.options['content'].strip()
+      if not re.search(regex, content_string):
+        raise ValueError("Promise content %r is not valid" % content_string)
+
     output = self.options['output']
-    mode = self.options.get('mode', '0600')
-    path_list_string = ""
+    mode = self.options.get('mode', '0644')
+    path_list = []
     for dist in working_set:
-      path_list_string += '  "%s",\n' % dist.location
+      path_list.append(dist.location)
 
-    content_string = '\n'.join([line.lstrip() for line in content.split('\n')])
-    config_string = ""
+    config_dict = dict()
     for key in self.options:
       if key.startswith('config-'):
-        config_string += "  '%s': '%s',\n" % (key[7:], self.options[key])
+        config_dict[key[7:]] = self.options[key]
 
-    option_dict = dict(path=path_list_string.strip(),
+    option_dict = dict(path=json.dumps(path_list, indent=2),
                        content=content_string,
-                       config=config_string.strip())
+                       config=json.dumps(config_dict, indent=2, sort_keys=True))
     with open(output, 'w') as f:
       f.write(script_template % option_dict)
 

slapos/test/recipe/test_plugin.py

+import os, shutil, tempfile, unittest
+from slapos.recipe import promise_plugin
+from slapos.test.utils import makeRecipe
+import stat, json
+
+class TestPromisePlugin(unittest.TestCase):
+
+  def setUp(self):
+    self.tmp = tempfile.mkdtemp()
+    self.output = os.path.join(self.tmp, 'output.py')
+    self.options = options = {
+      'output': self.output,
+      'eggs': 'slapos.cookbook'
+    }
+
+  def tearDown(self):
+    shutil.rmtree(self.tmp)
+
+  def test_parameters(self):
+    self.options['mode'] = '0644'
+    self.options['import'] = 'slapos.promise.plugin.check_site_available'
+    self.options['config-param1'] = "YY^@12"
+    self.options['config-param2'] = "23'91'"
+    self.options['config-param3'] = None
+    self.options['config-param4'] = """param
+in multi line
+123444
+"""
+    recipe = makeRecipe(
+            promise_plugin.Recipe,
+            options=self.options,
+            name='plugin')
+    recipe.install()
+
+    self.assertTrue(os.path.exists(self.output))
+    with open(self.output) as f:
+      content = f.read()
+    self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
+    self.assertEqual(stat.S_IMODE(os.stat(self.output).st_mode), int('644', 8))
+
+    expected_dict = dict(
+      param1=self.options['config-param1'],
+      param2=self.options['config-param2'],
+      param3=self.options['config-param3'],
+      param4=self.options['config-param4'],
+    )
+    self.assertIn('extra_config_dict = json.loads("""%s""", strict=False)' % json.dumps(expected_dict, indent=2, sort_keys=True), content)
+
+  def test_no_module_set(self):
+    recipe = makeRecipe(
+            promise_plugin.Recipe,
+            options=self.options,
+            name='plugin')
+    with self.assertRaises(KeyError):
+      recipe.install()
+
+  def test_default(self):
+    self.options['import'] = 'slapos.promise.plugin.check_site_available'
+    recipe = makeRecipe(
+            promise_plugin.Recipe,
+            options=self.options,
+            name='plugin')
+    recipe.install()
+
+    self.assertTrue(os.path.exists(self.output))
+    self.assertEqual(stat.S_IMODE(os.stat(self.output).st_mode), int('644', 8))
+    with open(self.output) as f:
+      content = f.read()
+    self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
+    self.assertIn('extra_config_dict = json.loads("""{}""", strict=False)', content)
+
+
+  def test_bad_parameters(self):
+    self.options['import'] = 'slapos.promise.plugin.check_site_available'
+    self.options['config-param1; print "toto"'] = """#xxxx"\nimport os; os.stat(f)"""
+    self.options['config-param2\n@domething'] = '"#$$*PPP\n\n p = 2*5; print "result is %s" % p'
+    recipe = makeRecipe(
+            promise_plugin.Recipe,
+            options=self.options,
+            name='plugin')
+    recipe.install()
+
+    self.assertTrue(os.path.exists(self.output))
+    with open(self.output) as f:
+      content = f.read()
+
+    expected_param1 = '"param1; print \\"toto\\"": "#xxxx\\"\\nimport os; os.stat(f)",'
+    expected_param2 = '"param2\\n@domething": "\\"#$$*PPP\\n\\n p = 2*5; print \\"result is %s\\" % p"'
+    self.assertIn(expected_param1, content)
+    self.assertIn(expected_param2, content)
+
+  def test_bad_module_path(self):
+    self.options['import'] = 'slapos.promise.plugin.check_site_available; print "toto"'
+    recipe = makeRecipe(
+            promise_plugin.Recipe,
+            options=self.options,
+            name='plugin')
+    with self.assertRaises(ValueError) as p:
+      recipe.install()
+
+    self.assertEqual(p.exception.message, "Import path %r is not a valid" % self.options['import'])
+
+  def test_bad_content(self):
+    self.options['content'] = 'from slapos.plugin.check_site_available import toto; print "toto"'
+    recipe = makeRecipe(
+            promise_plugin.Recipe,
+            options=self.options,
+            name='plugin')
+    with self.assertRaises(ValueError) as p:
+      recipe.install()
+
+    self.assertEqual(p.exception.message, "Promise content %r is not valid" % self.options['content'])
+

software/caddy-frontend/test/test.py

@@ -2411,9 +2411,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
-    open(
-      os.path.join(self.software_path, 'bin', 'is-icmp-packet-lost'), 'w'
-    ).write('echo "$@"')
+    fname = os.path.join(self.software_path, 'bin', 'is-icmp-packet-lost')
+    self.assertTrue(os.path.isfile(fname))
+    open(fname, 'w').write('echo "$@"')
     # call the monitor for this partition
     monitor_file = glob.glob(
       os.path.join(
@@ -2453,9 +2453,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
-    open(
-      os.path.join(self.software_path, 'bin', 'is-icmp-packet-lost'), 'w'
-    ).write('echo "$@"')
+    fname = os.path.join(self.software_path, 'bin', 'is-icmp-packet-lost')
+    self.assertTrue(os.path.isfile(fname))
+    open(fname, 'w').write('echo "$@"')
     # call the monitor for this partition
     monitor_file = glob.glob(
       os.path.join(
@@ -2495,10 +2495,10 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
-    open(
-      os.path.join(
-        self.software_path, 'bin', 'check-re6st-optimal-status'), 'w'
-    ).write('echo "$@"')
+    fname = os.path.join(
+      self.software_path, 'bin', 'check-re6st-optimal-status')
+    self.assertTrue(os.path.isfile(fname))
+    open(fname, 'w').write('echo "$@"')
     # call the monitor for this partition
     monitor_file = glob.glob(
       os.path.join(
@@ -3836,10 +3836,10 @@ https://www.google.com {}""",
     self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
-    open(
-      os.path.join(
-        self.software_path, 'bin', 'check-re6st-optimal-status'), 'w'
-    ).write('echo "$@"')
+    fname = os.path.join(
+      self.software_path, 'bin', 'check-re6st-optimal-status')
+    self.assertTrue(os.path.isfile(fname))
+    open(fname, 'w').write('echo "$@"')
     # call the monitor for this partition
     monitor_file = glob.glob(
       os.path.join(
@@ -4030,9 +4030,10 @@ https://www.google.com {}""",
     self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
-    open(
-      os.path.join(self.software_path, 'bin', 'is-icmp-packet-lost'), 'w'
-    ).write('echo "$@"')
+    fname = os.path.join(
+      self.software_path, 'bin', 'is-icmp-packet-lost')
+    self.assertTrue(os.path.isfile(fname))
+    open(fname, 'w').write('echo "$@"')
     # call the monitor for this partition
     monitor_file = glob.glob(
       os.path.join(
@@ -4072,9 +4073,10 @@ https://www.google.com {}""",
     self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
-    open(
-      os.path.join(self.software_path, 'bin', 'is-icmp-packet-lost'), 'w'
-    ).write('echo "$@"')
+    fname = os.path.join(
+      self.software_path, 'bin', 'is-icmp-packet-lost')
+    self.assertTrue(os.path.isfile(fname))
+    open(fname, 'w').write('echo "$@"')
     # call the monitor for this partition
     monitor_file = glob.glob(
       os.path.join(

software/kvm/buildout.hash.cfg

@@ -59,7 +59,7 @@ md5sum = 2036bf145f472f62ef8dee5e729328fd
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 9e40246b4bc4f968f0631016c939b014
+md5sum = c8ca875bf246997137552538ab9d8b1a
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/template/template-kvm-run.in

@@ -359,6 +359,8 @@ for nbd_ip, nbd_port in nbd_list:
       print 'Warning : Nbd is not available.'
     else:
       # NBD is available
+      # We close the NBD socket else qemu won't be able to use it apparently
+      s.close()
       kvm_argument_list.extend([
           '-drive',
           'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])

stack/erp5/zope-versions.cfg

 [versions]
-Zope2 = 2.13.28
+Zope2 = 2.13.29
 AccessControl = 2.13.16
 Acquisition = 2.13.12
 DateTime = 2.12.8
@@ -22,8 +22,8 @@ Products.Sessions = 3.0
 Products.StandardCacheManagers = 2.13.1
 Products.TemporaryFolder = 3.0
 Products.ZCTextIndex = 2.13.5
-Products.ZCatalog = 2.13.29
-Pygments = 2.2.0
+Products.ZCatalog = 2.13.30
+Pygments = 2.3.1
 Record = 2.13.0
 RestrictedPython = 3.6.0
 Sphinx = 1.0.8
@@ -32,20 +32,22 @@ ZODB3 = 3.10.7
 ZServer = 3.0
 ZopeUndo = 2.12.0
 docutils = 0.12
+filelock = 3.0.10
 initgroups = 2.13.0
 mechanize = 0.2.5
-mr.developer = 1.38
-pluggy = 0.6.0
-py = 1.5.2
+mr.developer = 1.34
+pluggy = 0.8.1
+py = 1.7.0
 pytz = 2017.2
 repoze.retry = 1.2
 repoze.tm2 = 1.0
 repoze.who = 2.0
-six = 1.11.0
+six = 1.12.0
 tempstorage = 2.12.2
-tox = 2.9.1
+toml = 0.10.0
+tox = 3.7.0
 transaction = 1.1.1
-z3c.checkversions = 0.5
+z3c.checkversions = 1.1
 zExceptions = 2.13.0
 zLOG = 2.11.2
 zc.buildout = 2.3.1

stack/slapos.cfg

@@ -134,7 +134,7 @@ pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
 six = 1.11.0
-slapos.cookbook = 1.0.85
+slapos.cookbook = 1.0.92
 slapos.core = 1.4.18
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
@@ -214,7 +214,7 @@ pyrsistent = 0.14.5
 ipaddress = 1.0.18
 
 # Required by:
-# slapos.cookbook==1.0.85
+# slapos.cookbook==1.0.92
 jsonschema = 3.0.0a3
 
 # Required by: