Update Release Candidate

component/curl/buildout.cfg

@@ -14,8 +14,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.56.0.tar.xz
-md5sum = 18ebc36c5dc9317d4a0b5db94a4e12ad
+url = http://curl.haxx.se/download/curl-7.60.0.tar.xz
+md5sum = a889cd11b4ae8794b7030472cb4df0a0
 configure-options =
   --disable-static
   --disable-ldap

component/mariadb/buildout.cfg

@@ -28,8 +28,8 @@ parts =
 [mariadb]
 recipe = slapos.recipe.cmmi
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.1.32
-md5sum = 389ce891cf00957748ba98b09f433c14
+version = 10.1.33
+md5sum = 164e4cbcfe4c6be235a133de91f866af
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3

component/pcre/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [pcre]
 recipe = slapos.recipe.cmmi
-url = https://ftp.pcre.org/pub/pcre/pcre-8.41.tar.bz2
-md5sum = c160d22723b1670447341b08c58981c1
+url = https://ftp.pcre.org/pub/pcre/pcre-8.42.tar.bz2
+md5sum = 085b6aa253e0f91cae70b3cdbe8c1ac2
 configure-options =
   --disable-static
   --enable-unicode-properties

component/proftpd/0001-mod_rlimit-don-t-change-PR_SET_DUMPABLE-flag.patch

+From ec7c3929df82855cc708f2074eb163590ecaeeaf Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
+Date: Wed, 23 May 2018 08:55:35 +0200
+Subject: [PATCH] mod_rlimit: don't change PR_SET_DUMPABLE flag
+
+In our case we don't really care if the process leaves a coredump and
+are interested in being able to inspect running processes for tests.
+---
+ modules/mod_rlimit.c | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/modules/mod_rlimit.c b/modules/mod_rlimit.c
+index 6788e4cee..2584260af 100644
+--- a/modules/mod_rlimit.c
++++ b/modules/mod_rlimit.c
+@@ -568,6 +568,8 @@ static int rlimit_set_core(int scope) {
+ #if !defined(PR_DEVEL_COREDUMP) && \
+     defined(HAVE_PRCTL) && \
+     defined(PR_SET_DUMPABLE)
++
++ #if 0 /* SlapOS patch: keep PR_SET_DUMPABLE for tests */
+   if (max == 0) {
+     /* Really, no core dumps please. On Linux, there are exceptions made
+      * even when setting RLIMIT_CORE = 0; see:
+@@ -582,6 +584,7 @@ static int rlimit_set_core(int scope) {
+         strerror(errno));
+     }
+   }
++#endif /* SlapOS patch */
+ #endif /* no --enable-devel=coredump and HAVE_PRCTL and PR_SET_DUMPABLE */
+ 
+   errno = xerrno;
+-- 
+2.11.0
+

component/proftpd/buildout.cfg

@@ -11,6 +11,7 @@ extends =
   ../curl/buildout.cfg
   ../libtool/buildout.cfg
   ../git/buildout.cfg
+  ../patch/buildout.cfg
 
 # proftpd server
 [proftpd-environment]
@@ -37,7 +38,10 @@ environment =
   install_group=${proftpd-grp:GROUP}
 cppflags=-I${zlib:location}/include -I${openssl:location}/include
 ldflags=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
-
+patch-binary = ${patch:location}/bin/patch
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/0001-mod_rlimit-don-t-change-PR_SET_DUMPABLE-flag.patch#6e58a7a429ff96a51812dc9835e4c227
 
 # mod_auth_web: a proftpd module to authenticate users against an HTTP service
 [proftpd-mod_auth_web-repository]
@@ -63,13 +67,13 @@ command =
 location=${proftpd:location}/libexec/mod_auth_web.so
 
 
-# ftpasswod: a perl script to manage a proftpd AuthUserFile
+# ftpasswd: a perl script to manage a proftpd AuthUserFile
 [ftpasswd]
 recipe = slapos.recipe.build:download
 url = https://raw.githubusercontent.com/proftpd/proftpd/v1.3.6/contrib/ftpasswd
 md5sum = 4a47df2cab86d8de7077a445bb416f31
 download-only = true
-mode = 0700
+mode = 0755
 
 
 [proftpd-output]

component/wget/buildout.cfg

 [buildout]
 extends =
   ../gettext/buildout.cfg
+  ../lunzip/buildout.cfg
   ../openssl/buildout.cfg
   ../patch/buildout.cfg
   ../pcre/buildout.cfg
   ../perl/buildout.cfg
   ../pkgconfig/buildout.cfg
-  ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
 parts =
   wget
 
 [wget]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/wget/wget-1.18.tar.xz
-md5sum = af9ca95a4bb8ac4a9bf10aeae66fa5ec
+url = http://ftp.gnu.org/gnu/wget/wget-1.19.5.tar.lz
+md5sum = 4ee511011a15f4207bea8b2993c343ee
 configure-options =
   --enable-ipv6
   --enable-opie
@@ -23,14 +23,8 @@ configure-options =
   --with-libssl-prefix=${openssl:location}
   --with-zlib-lib=${zlib:location}
 
-patch-options =
-  -p1
-
-patches =
-  ${:_profile_base_location_}/wget-doc.makefile.patch#0d23cf1ee81268a94699aebbb26058e6
-
 environment =
-  PATH=${gettext:location}/bin:${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  PATH=${gettext:location}/bin:${lunzip:location}/bin:${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${openssl:location}/lib
   CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include -I${pcre:location}/include

component/wget/wget-doc.makefile.patch

---- a/Makefile.am	2012-05-13 16:44:48.000000000 +0200
-+++ b/Makefile.am	2013-06-21 15:05:07.519274542 +0200
-@@ -41,7 +41,7 @@
- ACLOCAL_AMFLAGS = -I m4
- 
- # subdirectories in the distribution
--SUBDIRS = lib src doc po tests util testenv
-+SUBDIRS = lib src po tests util testenv
- 
- EXTRA_DIST = MAILING-LIST \
-              msdos/config.h msdos/Makefile.DJ \
---- a/Makefile.in	2012-08-05 22:17:17.000000000 +0200
-+++ b/Makefile.in	2013-06-21 15:05:04.351269286 +0200
-@@ -1322,7 +1322,7 @@
- ACLOCAL_AMFLAGS = -I m4
-
- # subdirectories in the distribution
--SUBDIRS = lib src doc po tests util testenv
-+SUBDIRS = lib src po tests util testenv
- EXTRA_DIST = MAILING-LIST \
-              msdos/config.h msdos/Makefile.DJ \
-              msdos/Makefile.WC ABOUT-NLS \

software/erp5/README.rst

@@ -14,9 +14,8 @@ front-end instance which is expected to contains the RewriteRules_ (or
 equivalent) needed to relocate Zope's urls via its VirtualHostMonster_. See the
 ``frontend`` erp5 instance parameter.
 
-Included cloudooo partition is **deprecated**. It is not recommended for
-intensive usage. See the ``cloudooo`` Software Release to setup a cloudooo
-cluster, more suitable for intensive usage.
+ERP5 defaults connect to the public cloudooo on https://cloudooo.erp5.net/.
+See the ``cloudooo`` Software Release to setup a cloudooo cluster if necessary.
 
 Replication
 ===========
@@ -84,7 +83,6 @@ This software release assigns the following port ranges by default:
   ====================  ==========
   memcached-persistent  2000-2009
   memcached-volatile    2010-2019
-  cloudooo              2020-2024
   smtp                  2025-2029
   neo (admin & master)  2050-2051
   mariadb               2099

software/erp5/instance-cloudooo-schema.json

-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "required": ["tcpv4-port"],
-  "properties": {
-    "tcpv4-port": {
-      "allOf": [{
-        "$ref": "#/definitions/tcpv4port"
-      }, {
-        "description": "Start allocating ports at this value, going upward"
-      }]
-    }
-  }
-}

software/erp5/instance-erp5-input-schema.json

@@ -187,12 +187,10 @@
       },
       "type": "object"
     },
-    "cloudooo": {
-      "description": "Format conversion service",
-      "additionalProperties": {
-        "$ref": "./instance-cloudooo-schema.json#/properties"
-      },
-      "type": "object"
+    "cloudooo-url": {
+      "description": "Format conversion service URL",
+      "pattern": "^https?://",
+      "type": "string"
     },
     "cloudooo-retry-count": {
       "description": "Define retry count for cloudooo in network error case in test",

software/erp5/instance-erp5-output-schema.json

@@ -39,11 +39,6 @@
       "pattern": "^memcached://",
       "type": "string"
     },
-    "cloudooo-url": {
-      "description": "Conversion service access information - DEPRECATED",
-      "pattern": "^(http|https)://",
-      "type": "string"
-    },
     "mariadb-database-list": {
       "description": "Relational database access information",
       "items": {

software/erp5/instance-mariadb-schema.json

@@ -127,6 +127,13 @@
       "description": "Contents of odbc.ini file, see unixodbc document",
       "default": "",
       "type": "string"
+    },
+    "environment-variables": {
+      "description": "Extra environment variables for mysqld may be required to use third party ODBC libraries for CONNECT storage engine.",
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
     }
 
   }

software/erp5testnode/testsuite/proftpd/buildout.hash.cfg

@@ -15,6 +15,6 @@
 # not need these here).
 
 [template]
-filename = instance.cfg
-md5sum = 9b4cc727c43d0daaec5b01cf4b9f7a15
+filename = instance.cfg.in
+md5sum = 20c5afac025e97c2937852f4d84f7cd3
 

software/erp5testnode/testsuite/proftpd/instance.cfg → software/erp5testnode/testsuite/proftpd/instance.cfg.in

@@ -25,6 +25,7 @@ repository = ${slapos-repository:location}
 [create-directory]
 recipe = slapos.cookbook:mkdirectory
 bin = $${buildout:directory}/bin
+working-dir = $${buildout:directory}/tmp/
 
 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper
@@ -40,3 +41,4 @@ environment =
   PATH=${buildout:bin-directory}:/usr/bin/:/bin/
   LOCAL_IPV4=$${slap-configuration:ipv4-random}
   GLOBAL_IPV6=$${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=$${create-directory:working-dir}

software/erp5testnode/testsuite/proftpd/software.cfg

@@ -67,3 +67,7 @@ mode = 640
 slapos.test.proftpd =
 erp5.util =
 #erp5.util = 0.4.51
+
+pyasn1 = 0.4.2
+slapos.recipe.template = 4.3
+pysftp = 0.2.9

software/gitlab/software.cfg

@@ -41,6 +41,8 @@ parts =
     gitlab/vendor/bundle
     github-markup-patch
     gitlab-workhorse
+    git-go123
+    git-errors
     git-backup
 
 #   for instance
@@ -142,7 +144,7 @@ location = ${buildout:parts-directory}/gitlab-shell
 repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
 # gitlab 8.8 wants gitlab-workhorse 0.7.1
 # 0.7.1 + NXD patches
-revision = v0.7.1-5-gd23a3247829fc3200e3dc784dcd57b5a0febac48
+revision = v0.7.1-5-gc368233516b78856484aff31f9141c880b800670
 location = ${buildout:parts-directory}/gitlab-workhorse
 
 # Patch github markup to not call "python2 -S /path/to/rest2html" but only "python2 /path/to/rest2html"
@@ -191,10 +193,18 @@ go.importpath = github.com/libgit2/git2go
 # branch 'next' is required by git-backup
 revision = next-g53594d7581617dbae7bb5960b4ac5f0ff513c184
 
+[git-go123]
+<= go-git-repository
+go.importpath = lab.nexedi.com/kirr/go123
+
+[git-errors]
+<= go-git-repository
+go.importpath = github.com/pkg/errors
+
 [git-backup-repository]
 <= go-git-repository
 go.importpath = lab.nexedi.com/kirr/git-backup
-revision = 3ba6cf73ba224c40f67f1fb87c855b915eb91f58
+revision = 0b8d834bd00b19169f65d407eab8980076779b9d
 
 
 [git-backup]

software/monitor/instance-monitor.cfg.jinja2

@@ -7,6 +7,7 @@ parts =
   cron-network-bench
   symlink-re6st-logs
   symlink-collected-logs
+  python-symlink
   monitor-collect-csv-wrapper
   monitor-base
   monitor-check-memory-usage
@@ -77,6 +78,11 @@ context =
   section slave_information slap-configuration
   key monitor_private_hash slap-configuration:private-hash  
 
+[python-symlink]
+recipe = plone.recipe.command
+command = ln -sf {{ buildout_bin }}/pythonwitheggs ${monitor-directory:bin}/python
+update-command = ${:command}
+
 [monitor-collect-csv-wrapper]
 recipe = slapos.cookbook:wrapper
 command-line = 

software/monitor/software.cfg

@@ -28,7 +28,7 @@ mode = 0644
 recipe = slapos.recipe.build:download 
 url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
 destination = ${buildout:directory}/template-base-monitor.cfg
-md5sum = db36ed784c690be892bf1834444c1fe7
+md5sum = 79125819f20f4f18a301b806daed2ceb
 mode = 0644
 
 [template-monitor-distributor]

software/nayuos/scripts/cros_full_build.in

@@ -356,7 +356,7 @@ for hashfunction in md5sum sha1sum sha256sum sha512sum; do
 done
 
 for file in $(ls *.img); do
-  gzip -9 ${file}
+  gzip -9 -f ${file}
 done
 
 exit 0

software/neoppod/instance-neo-storage-mysql.cfg.in

@@ -142,6 +142,7 @@ context =
     section directory         directory
     section my_cnf_parameters my-cnf-parameters
     raw     bin_directory     {{ bin_directory }}
+    raw     prepend_path      ${mysqld:mysql-base-directory}/bin
 {% endif -%}
 
 [buildout]

software/neoppod/runTestSuite.in

@@ -12,6 +12,9 @@ SUMMARY_RE = re.compile(
   r' (.*) (?P<expected_count>\d+|\.) (.*) (?P<skip_count>\d+|\.)'
   r' (.*) (?P<duration>\d+(\.\d*)?|\.\d+)s', re.MULTILINE)
 
+PATH = os.environ['PATH']
+PATH = '{{ prepend_path }}' + (PATH and ':' + PATH)
+
 # NEO specific environment
 TEMP_DIRECTORY  = '{{directory.tmp}}'
 NEO_DB_SOCKET = '{{my_cnf_parameters.socket}}'
@@ -90,7 +93,8 @@ def main():
 
     args = [RUN_NEO_TESTS_COMMAND, '-ufz']
     command = ' '.join(args)
-    env = {'TEMP': temp,
+    env = {'PATH': PATH,
+           'TEMP': temp,
            'NEO_TESTS_ADAPTER': adapter,
            'NEO_TEST_ZODB_FUNCTIONAL': '1',
            'NEO_DB_USER': 'root'}

software/neoppod/software-common.cfg

@@ -105,7 +105,7 @@ md5sum = 1fee10f02c2fa2a581e21878ca0fd704
 
 [instance-neo-storage-mysql]
 <= download-base-neo
-md5sum = 366e51c0dbd85e511a31e403b8704735
+md5sum = 9ca3283659f98713ffe3b08ba9442ff1
 
 [template-neo-my-cnf]
 <= download-base-neo

software/neoppod/software.cfg

@@ -26,7 +26,7 @@ md5sum = ee8401a4e7d82bf488a57e3399f9ce48
 [runTestSuite.in]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_buildout_section_name_}
-md5sum = 982de37ea52c10938e46037adf2894c1
+md5sum = d8bba690cf950613d5576ff23813df59
 
 [runTestSuite_py]
 recipe = zc.recipe.egg

software/proftpd/buildout.hash.cfg

@@ -20,9 +20,9 @@ md5sum = 8ed5b4a7940db47ccb386c2f4e3e7273
 
 [instance-default]
 filename = instance-default.cfg.in
-md5sum = 89e145188fe2a223f22f57e5c0c242e4
+md5sum = e2e67390753ec63babcc8d4ce9fc6230
 
 [proftpd-config-file]
 filename = proftpd-config-file.cfg.in
-md5sum = 46cbe4ea65b445822d3da32c76c7d67d
+md5sum = a7c0f4607c378b640379cc258a8aadfa
 

software/proftpd/instance-default.cfg.in

@@ -62,6 +62,8 @@ url = sftp://[${:host}]:${:sftp-port}
 data-dir = ${directory:proftpd-dir}
 user=${proftpd-userinfo:pw-name}
 group=${proftpd-userinfo:gr-name}
+scoreboard-file=${directory:var}/proftpd.scoreboard
+pid-file=${directory:var}/proftpd.pid
 sftp-log=${directory:log}/proftpd-sftp.log
 xfer-log=${directory:log}/proftpd-xfer.log
 ban-log=${directory:log}/proftpd-ban.log
@@ -70,6 +72,7 @@ ssh-host-dsa-key=${ssh-host-dsa-key:output}
 ssh-host-ecdsa-key=${ssh-host-ecdsa-key:output}
 ssh-authorized-keys-dir = ${directory:ssh-authorized-keys-dir}
 ban-table=${directory:srv}/proftpd-ban-table
+control-socket=${directory:var}/proftpd.sock
 auth-user-file=${auth-user-file:output}
 
 recipe = slapos.cookbook:wrapper

software/proftpd/proftpd-config-file.cfg.in

@@ -9,6 +9,9 @@ Port {{ proftpd['sftp-port'] }}
 User {{ proftpd['user'] }}
 Group {{ proftpd['group'] }}
 
+ScoreboardFile {{ proftpd['scoreboard-file'] }}
+PidFile {{ proftpd['pid-file'] }}
+
 Umask 022
 AllowOverwrite on
 
@@ -63,5 +66,6 @@ BanTable {{ proftpd['ban-table'] }}
 # 5 failed login attemps in 5 minutes -> ban for 20 minutes
 BanOnEvent MaxLoginAttempts 5/00:05:00 00:20:00 "Too many Failed Login Attempts"
 BanControlsACLs all allow user {{ proftpd['user'] }}
-
-
+# This depends on a control socket
+ControlsSocket {{ proftpd['control-socket'] }}
+ControlsSocketOwner {{ proftpd['user'] }} {{ proftpd['group'] }}

software/proftpd/test/setup.py

@@ -45,6 +45,8 @@ setup(name=name,
         'slapos.core',
         'erp5.util',
         'pysftp',
+        'supervisor',
+        'psutil',
         ],
       zip_safe=True,
       test_suite='test',

software/proftpd/test/test.py

@@ -33,6 +33,7 @@ import StringIO
 import subprocess
 
 import pysftp
+import psutil
 from paramiko.ssh_exception import SSHException
 from paramiko.ssh_exception import AuthenticationException
 
@@ -204,3 +205,29 @@ class TestInstanceParameterPort(ProFTPdTestCase):
     self.assertTrue(self._getConnection())
 
 
+class TestFilesAndSocketsInInstanceDir(ProFTPdTestCase):
+  """Make sure the instance only have files and socket in software dir.
+  """
+  def setUp(self):
+    """sets `self.proftpdProcess` to `psutil.Process` for the running proftpd in the instance.
+    """
+    all_process_info = self.getSupervisorRPCServer().supervisor.getAllProcessInfo()
+    # there is only one process in this instance
+    process_info, = [p for p in all_process_info if p['name'] != 'watchdog']
+    process = psutil.Process(process_info['pid'])
+    self.assertEqual('proftpd', process.name()) # sanity check
+    self.proftpdProcess = process
+
+  def test_only_write_file_in_instance_dir(self):
+    self.assertEqual(
+        [],
+        [f for f in self.proftpdProcess.open_files()
+         if f.mode != 'r'
+         if not f.path.startswith(self.computer_partition_root_path)])
+
+  def test_only_unix_socket_in_instance_dir(self):
+    self.assertEqual(
+        [],
+        [s for s in self.proftpdProcess.connections('unix')
+         if not s.laddr.startswith(self.computer_partition_root_path)])
+

software/proftpd/test/utils.py

@@ -31,6 +31,8 @@ import socket
 from contextlib import closing
 import logging
 
+import xmlrpclib
+import supervisor.xmlrpc
 from erp5.util.testnode.SlapOSControler import SlapOSControler
 from erp5.util.testnode.ProcessManager import ProcessManager
 import slapos
@@ -68,7 +70,14 @@ class SlapOSInstanceTestCase(unittest.TestCase):
 
   @classmethod
   def setUpClass(cls):
+    try:
+      cls._setUpClass()
+    except:
+      cls.stopSlapOSProcesses()
+      raise
 
+  @classmethod
+  def _setUpClass(cls):
     working_directory = os.environ.get(
         'SLAPOS_TEST_WORKING_DIR',
         os.path.join(os.path.dirname(__file__), '.slapos'))
@@ -76,8 +85,11 @@ class SlapOSInstanceTestCase(unittest.TestCase):
     # AF_UNIX path too long This `working_directory` should not be too deep.
     # Socket path is 108 char max on linux
     # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
-    if len(working_directory + '/inst/supervisord.socket') > 108:
-      raise RuntimeError('working directory too deep, try setting SLAPOS_TEST_WORKING_DIR')
+    # Supervisord socket name contains the pid number, which is why we add
+    # .xxxxxxx in this check.
+    if len(working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
+      raise RuntimeError('working directory ( {} ) is too deep, try setting '
+              'SLAPOS_TEST_WORKING_DIR'.format(working_directory))
 
     if not os.path.exists(working_directory):
       os.mkdir(working_directory)
@@ -128,7 +140,7 @@ class SlapOSInstanceTestCase(unittest.TestCase):
     # TODO: log more details in this case
     assert software_status_dict['status_code'] == 0
 
-    instance_parameter_dict = cls.getInstanceParmeterDict()
+    instance_parameter_dict = cls.getInstanceParameterDict()
     instance_status_dict = slapos_controler.runComputerPartition(
         config,
         cluster_configuration=instance_parameter_dict,
@@ -147,7 +159,7 @@ class SlapOSInstanceTestCase(unittest.TestCase):
           partition_parameter_kw=instance_parameter_dict))
 
     # expose some class attributes so that tests can use them:
-    # the ComputerPartition instances, to getInstanceParmeterDict
+    # the ComputerPartition instances, to getInstanceParameterDict
     cls.computer_partition = computer_partition_list[0]
 
     # the path of the instance on the filesystem, for low level inspection
@@ -158,7 +170,25 @@ class SlapOSInstanceTestCase(unittest.TestCase):
 
 
   @classmethod
-  def tearDownClass(cls):
-    # FIXME: if setUpClass fail, this is not called and leaks zombie processes
+  def stopSlapOSProcesses(cls):
+    if hasattr(cls, '_process_manager'):
       cls._process_manager.killPreviousRun()
 
+  @classmethod
+  def tearDownClass(cls):
+    cls.stopSlapOSProcesses()
+
+  # utility methods
+  def getSupervisorRPCServer(self):
+    """Returns a XML-RPC connection to the supervisor used by slapos node
+
+    Refer to http://supervisord.org/api.html for details of available methods.
+    """
+    # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
+    return xmlrpclib.ServerProxy(
+       'http://slapos-supervisor',
+       transport=supervisor.xmlrpc.SupervisorTransport(
+           None,
+           None,
+           # XXX hardcoded socket path
+           serverurl="unix://{working_directory}/inst/supervisord.socket".format(**self.config)))

stack/erp5/buildout.cfg

@@ -24,14 +24,12 @@ extends =
   ../../component/libdmtx/buildout.cfg
   ../../component/libffi/buildout.cfg
   ../../component/libpng/buildout.cfg
-  ../../component/libreoffice-bin/buildout.cfg
   ../../component/matplotlib/buildout.cfg
   ../../component/mesa/buildout.cfg
   ../../component/numpy/buildout.cfg
   ../../component/statsmodels/buildout.cfg
   ../../component/h5py/buildout.cfg
   ../../component/ocropy/buildout.cfg
-  ../../component/onlyoffice-core/buildout.cfg
   ../../component/pandas/buildout.cfg
   ../../component/percona-toolkit/buildout.cfg
   ../../component/patch/buildout.cfg
@@ -46,7 +44,6 @@ extends =
   ../../component/tesseract/buildout.cfg
   ../../component/w3-validator/buildout.cfg
   ../../component/w3m/buildout.cfg
-  ../../component/wkhtmltopdf/buildout.cfg
   ../../component/xorg/buildout.cfg
   ../../component/poppler/buildout.cfg
   ../../component/zabbix/buildout.cfg
@@ -57,7 +54,6 @@ extends =
   ../../component/bash/buildout.cfg
   ../../component/wget/buildout.cfg
   ../../component/aspell/buildout.cfg
-  ../../component/cloudooo/buildout.cfg
   ../../component/jsl/buildout.cfg
   ../../component/6tunnel/buildout.cfg
   ../../component/findutils/buildout.cfg
@@ -94,7 +90,6 @@ parts +=
   libdmtx
   dmtx-utils
   kumo
-  libreoffice-bin
   w3-validator
   tesseract
   scipy
@@ -128,9 +123,6 @@ parts +=
   ocrb-fonts
   android-fonts
 
-# Cloudooo specific part
-  cloudooo-develop
-  cloudooo
 
 # get git repositories
   erp5
@@ -197,9 +189,6 @@ link-binary =
 [template-kumofs]
 <= download-base
 
-[template-cloudooo]
-<= download-base
-
 [template-zope-conf]
 <= download-base
 
@@ -248,6 +237,7 @@ context =
     key dbus_glib_location dbus-glib:location
     key dbus_location dbus:location
     key dcron_location dcron:location
+    key default_cloudooo_url erp5-defaults:cloudooo-connection-url
     key erp5_location erp5:location
     key file_location file:location
     key findutils_location findutils:location
@@ -272,8 +262,6 @@ context =
     key libXrender_location libXrender:location
     key libexpat_location libexpat:location
     key libffi_location libffi:location
-    key libpng12_location libpng12:location
-    key libreoffice_bin_location libreoffice-bin:location
     key librsvg_location librsvg:location
     key libxcb_location libxcb:location
     key local_bt5_repository local-bt5-repository:list
@@ -284,7 +272,6 @@ context =
     key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
     key matplotlibrc_location matplotlibrc:location
     key mesa_location mesa:location
-    key onlyoffice_core_location onlyoffice-core:location
     key parts_directory buildout:parts-directory
     key openssl_location openssl:location
     key percona_toolkit_location percona-toolkit:location
@@ -297,7 +284,6 @@ context =
     key stunnel_location stunnel:location
     key template_apache_conf template-apache-backend-conf:target
     key template_balancer template-balancer:target
-    key template_cloudooo template-cloudooo:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
     key template_jupyter_cfg instance-jupyter-notebook:rendered
@@ -314,11 +300,10 @@ context =
     key template_zope template-zope:target
     key template_zope_conf template-zope-conf:target
     key userhosts_location userhosts:location
+    key unixodbc_location unixodbc:location
     key wget_location wget:location
-    key wkhtmltopdf_location wkhtmltopdf:location
     key xdamage_location xdamage:location
     key xfixes_location xfixes:location
-    key xserver_location xserver:location
     key zlib_location zlib:location
     key extra_path_list eggs:extra-paths
 
@@ -382,6 +367,7 @@ repository_id_list = erp5
 
 # ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
 [erp5-defaults]
+cloudooo-connection-url = https://cloudooo.erp5.net/
 # Jupyter is by default disabled in ERP5
 jupyter-enable-default = false
 

stack/erp5/buildout.hash.cfg

@@ -27,16 +27,12 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = e731fa1d24fc87b26efdbcb40d74c55d
+md5sum = 488b9410da5f23f9af177f81d847c930
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
 md5sum = 091d3c3feb2d543d176b0fadb11c07dc
 
-[template-cloudooo]
-filename = instance-cloudoo.cfg.in
-md5sum = eeb584a393b15582a3be6b591dcdcde6
-
 [template-zope-conf]
 filename = zope.conf.in
 md5sum = adb25a1ab15c8aecf40a3952528f81c2
@@ -71,7 +67,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template]
 filename = instance.cfg.in
-md5sum = eba0b4596484dcf24b1da29ddeac453d
+md5sum = 995b5dd89253b79a9511e8b46e91480c
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -79,7 +75,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 2431422c66082a6fb6a6be236ebacd5c
+md5sum = 9f8851af5faed9d1e346d7dbe0e6db5d
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-cloudoo.cfg.in

-{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
-{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
-[buildout]
-extends =
-  {{ parameter_dict['template-monitor'] }}
-parts +=
-  publish
-  cloudooo-instance
-  resiliency-exclude-file
-  promise
-  promise-openoffice
-  {% if use_ipv6 %}promise-tunnel{% endif %}
-  xvfb-instance
-  wkhtmltopdf-on-xvfb
-
-[publish]
-recipe = slapos.cookbook:publish.serialised
-{% if use_ipv6 -%}
-url = http://[${ipv6toipv4:ipv6}]:${ipv6toipv4:ipv6-port}/
-{% else -%}
-url = http://${cloudooo-instance:ip}:${cloudooo-instance:port}/
-{% endif -%}
-monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
-
-[cloudooo-instance]
-recipe = slapos.cookbook:generic.cloudooo
-
-# Network options
-ip = {{ (ipv4_set | list)[0] }}
-{% set tcpv4_port = slapparameter_dict['tcpv4-port'] -%}
-port = {{ tcpv4_port }}
-openoffice-port = {{ tcpv4_port + 1 }}
-
-# Paths
-configuration-file = ${directory:etc}/cloudooo.cfg
-wrapper = ${directory:service}/cloudooo
-
-# Paths: Data
-data-directory = ${directory:cloudooo-data}
-
-environment =
-  LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
-  FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
-  PATH = ${binary-link:target-directory}
-
-# Binary information
-# cloudooo specific configuration
-ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
-ooo-paster = {{ bin_directory }}/cloudooo_paster
-ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
-
-[fontconfig-instance]
-recipe = slapos.cookbook:fontconfig
-conf-path = ${directory:etc}/font.conf
-font-system-folder = {{ parameter_dict['fonts'] }}
-font-folder = ${directory:font}
-service-folder = ${directory:service}
-
-[resiliency-exclude-file]
-# Generate rdiff exclude file in case of resiliency
-recipe = slapos.recipe.template:jinja2
-mode = 644
-template = {{ 'inline:{{ "**\\n" }}' }}
-rendered = ${directory:srv}/exporter.exclude
-
-[promise]
-recipe = slapos.cookbook:check_port_listening
-path = ${directory:promise}/cloudooo
-hostname = ${cloudooo-instance:ip}
-port = ${cloudooo-instance:port}
-
-[promise-openoffice]
-recipe = slapos.cookbook:check_port_listening
-path = ${directory:promise}/openoffice
-hostname = ${cloudooo-instance:ip}
-port = ${cloudooo-instance:openoffice-port}
-
-{% if use_ipv6 -%}
-[promise-tunnel]
-recipe = slapos.cookbook:check_port_listening
-path = ${directory:promise}/tunnel
-hostname = ${ipv6toipv4:ipv6}
-port = ${ipv6toipv4:ipv6-port}
-
-[ipv6toipv4]
-recipe = slapos.cookbook:ipv6toipv4
-runner-path = ${directory:service}/${:base-name}
-6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel
-shell-path = {{ parameter_dict['dash'] }}/bin/dash
-ipv4 = ${cloudooo-instance:ip}
-ipv6 = {{ (ipv6_set | list)[0] }}
-ipv6-port = ${cloudooo-instance:port}
-ipv4-port = ${cloudooo-instance:port}
-base-name = cloudooo-tunnel
-
-{% endif -%}
-# rest of parts are candidates for some generic stuff
-[directory]
-recipe = slapos.cookbook:mkdirectory
-bin = ${buildout:directory}/bin
-etc = ${buildout:directory}/etc
-srv = ${buildout:directory}/srv
-var = ${buildout:directory}/var
-run = ${:var}/run
-service = ${:etc}/run
-promise = ${:etc}/promise
-cloudooo-data = ${:srv}/cloudooo
-font = ${:srv}/font
-framebuffer = ${:srv}/framebuffer
-
-[monitor-instance-parameter]
-monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
-monitor-httpd-port = {{ tcpv4_port + 2 }}
-monitor-title = {{ slapparameter_dict['name'] }}
-password = {{ slapparameter_dict['monitor-passwd'] }}
-
-[binary-link]
-recipe = slapos.cookbook:symbolic.link
-target-directory = ${directory:bin}
-link-binary =
-  {{ parameter_dict['coreutils'] }}/bin/basename
-  {{ parameter_dict['coreutils'] }}/bin/cat
-  {{ parameter_dict['coreutils'] }}/bin/cp
-  {{ parameter_dict['coreutils'] }}/bin/ls
-  {{ parameter_dict['coreutils'] }}/bin/tr
-  {{ parameter_dict['coreutils'] }}/bin/uname
-  {{ parameter_dict['coreutils'] }}/bin/dirname
-# wrapper recipe needs the head command
-  {{ parameter_dict['coreutils'] }}/bin/head
-  {{ parameter_dict['imagemagick'] }}/bin/convert
-  {{ parameter_dict['imagemagick'] }}/bin/identify
-  {{ parameter_dict['poppler'] }}/bin/pdfinfo
-  {{ parameter_dict['poppler'] }}/bin/pdftotext
-  {{ parameter_dict['poppler'] }}/bin/pdftohtml
-  {{ parameter_dict['onlyoffice-core'] }}/bin/x2t
-
-[xvfb-instance]
-recipe = slapos.cookbook:xvfb
-runner-path = ${directory:service}/xvfb
-xvfb-path = {{ parameter_dict["xserver"] }}/bin/Xvfb
-fbdir-path = ${directory:framebuffer}
-tmp-path = ${directory:run}
-shell-path = {{ parameter_dict["dash"] }}/bin/dash
-
-[wkhtmltopdf-on-xvfb]
-recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:bin}/wkhtmltopdf
-environment =
-  DISPLAY=:0
-  TMPDIR=${directory:run}
-command-line = {{ parameter_dict['wkhtmltopdf'] }}/wkhtmltopdf --use-xserver

stack/erp5/instance-erp5.cfg.in

@@ -44,7 +44,6 @@ config-name = {{ name }}
 
 {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
-{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {% if has_posftix -%}
 {{   request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
@@ -150,7 +149,7 @@ return =
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-caucase-url = ${request-caucase:connection-http-url}
-config-cloudooo-url = ${request-cloudooo:connection-url}
+config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
 config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}

stack/erp5/instance-mariadb.cfg.in

@@ -186,7 +186,12 @@ template = inline:#!{{ dash }}
     --skip-name-resolve \
     --datadir='${my-cnf-parameters:data-directory}' \
     --basedir='{{ parameter_dict['mariadb-location'] }}' \
-  && ODBCSYSINI='${my-cnf-parameters:etc-directory}' exec '{{ parameter_dict['mariadb-location'] }}/bin/mysqld' \
+    && ODBCSYSINI='${my-cnf-parameters:etc-directory}' \
+    LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{{ parameter_dict['unixodbc-location'] }}/lib \
+    {% for variable in slapparameter_dict.get('environment-variables', ()) %}
+    {{ variable }} \
+    {% endfor %}
+    exec '{{ parameter_dict['mariadb-location'] }}/bin/mysqld' \
     --defaults-file='${my-cnf:rendered}' \
     "$@"
 rendered = ${directory:services}/mariadb

stack/erp5/instance.cfg.in

@@ -6,52 +6,6 @@ extends =
 [jinja2-template-base]
 mode = 644
 
-[dynamic-template-cloudooo-parameters]
-cairo = {{ cairo_location }}
-coreutils = {{ dumps(coreutils_location) }}
-cups = {{ cups_location }}
-dbus = {{ dbus_location }}
-dbus-glib = {{ dbus_glib_location }}
-file = {{ file_location }}
-fontconfig = {{ fontconfig_location }}
-freetype = {{ freetype_location }}
-glib = {{ glib_location }}
-glu = {{ glu_location }}
-imagemagick = {{ dumps(imagemagick_location) }}
-libICE = {{ libICE_location }}
-libSM = {{ libSM_location }}
-libX11 = {{ libX11_location }}
-libXau = {{ libXau_location }}
-libXdmcp = {{ libXdmcp_location }}
-libXext = {{ libXext_location }}
-libXrender = {{ libXrender_location }}
-libexpat = {{ libexpat_location }}
-libffi = {{ libffi_location }}
-libpng12 = {{ libpng12_location }}
-libxcb = {{ libxcb_location }}
-mesa = {{ mesa_location }}
-onlyoffice-core = {{ dumps(onlyoffice_core_location) }}
-pixman = {{ pixman_location }}
-poppler = {{ dumps(poppler_location) }}
-wkhtmltopdf = {{ dumps(wkhtmltopdf_location) }}
-xdamage = {{ xdamage_location }}
-xfixes = {{ xfixes_location }}
-xserver = {{ dumps(xserver_location) }}
-zlib = {{ zlib_location }}
-libreoffice-bin = {{ libreoffice_bin_location }}
-fonts = {{ fonts_location }}
-buildout-bin-directory = {{ buildout_bin_directory }}
-6tunnel = {{ sixtunnel_location }}
-dash = {{ dash_location }}
-template-monitor = {{ dumps(template_monitor) }}
-
-[dynamic-template-cloudooo]
-<= jinja2-template-base
-template = {{ template_cloudooo }}
-filename = instance-cloudoo.cfg
-extra-context =
-    section parameter_dict dynamic-template-cloudooo-parameters
-
 [dynamic-template-postfix-parameters]
 bin-directory = {{ bin_directory }}
 buildout-bin-directory = {{ buildout_bin_directory }}
@@ -72,6 +26,7 @@ extra-context =
     import urllib urllib
 
 [dynamic-template-erp5-parameters]
+default-cloudooo-url =  {{ dumps(default_cloudooo_url) }}
 jupyter-enable-default = {{ jupyter_enable_default }}
 local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
 template-monitor = {{ dumps(template_monitor) }}
@@ -81,6 +36,7 @@ template-monitor = {{ dumps(template_monitor) }}
 template = {{ template_erp5 }}
 filename = instance-erp5.cfg
 extra-context =
+    key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key template_monitor dynamic-template-erp5-parameters:template-monitor
@@ -183,6 +139,7 @@ mariadb-start-clone-from-backup = {{ mariadb_start_clone_from_backup }}
 promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result
 percona-tools-location = {{ percona_toolkit_location }}
 template-monitor = {{ template_monitor }}
+unixodbc-location = {{ unixodbc_location }}
 
 [dynamic-template-mariadb]
 <= jinja2-template-base
@@ -207,7 +164,6 @@ RootSoftwareInstance = ${:default}
 # Internal software types
 kumofs = dynamic-template-kumofs:rendered
 caucase = dynamic-template-caucase:rendered
-cloudooo = dynamic-template-cloudooo:rendered
 mariadb = dynamic-template-mariadb:rendered
 balancer = dynamic-template-balancer:rendered
 postfix = dynamic-template-postfix:rendered