Update release candidate

component/glib/buildout.cfg

@@ -6,6 +6,7 @@ extends =
   ../perl/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
+  ../python-2.7/buildout.cfg
 parts =
   glib
 

component/haproxy/buildout.cfg

@@ -12,8 +12,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.5.tar.gz
-md5sum = ed84c80cb97852d2aa3161ed16c48a1c
+url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.9.tar.gz
+md5sum = a2bbbdd45ffe18d99cdcf26aa992f92d
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/libpng/buildout.cfg

@@ -18,15 +18,15 @@ environment =
 
 [libpng12]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.2.57.tar.xz
-md5sum = 307052e5e8af97b82b17b64fb1b3677a
+url = http://download.sourceforge.net/libpng/libpng-1.2.58.tar.xz
+md5sum = 1fe68fa3cdab99dbcfd2a6b4de95645f
 
 [libpng15]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.5.28.tar.xz
-md5sum = 847aa2a1b231c07466d7f4167537424a
+url = http://download.sourceforge.net/libpng/libpng-1.5.29.tar.xz
+md5sum = b9e5452ee9681c313638efedb16c12a6
 
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.6.28.tar.xz
-md5sum = 425354f86c392318d31aedca71019372
+url = http://download.sourceforge.net/libpng/libpng-1.6.32.tar.xz
+md5sum = e01be057a9369183c959b793a685ad15

component/mariadb/buildout.cfg

@@ -20,14 +20,15 @@ extends =
 # The following lines are only for mariarocks.cfg
 # to be extended last without touching 'parts'.
   ../gcc/buildout.cfg
+  ../zstd/buildout.cfg
 
 parts =
   mariadb
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-url = https://downloads.mariadb.org/f/mariadb-10.1.26/source/mariadb-10.1.26.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-md5sum = bb88afb72434c6d567c742896dd50d41
+url = https://downloads.mariadb.org/f/mariadb-10.1.28/source/mariadb-10.1.28.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
+md5sum = 38acd5b44c56791701d80fddf088ef38
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
@@ -61,12 +62,14 @@ configure-options =
   -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
-CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include
+CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
 CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
+extra_cflags =
+extra_include_path =
 extra_library_path =
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
-  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include
+  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
   LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
   PATH=${patch:location}/bin:%(PATH)s

component/mariadb/mariarocks.cfg

 # Do not extend any file that touch buildout:parts.
 
 [mariadb]
-url = https://downloads.mariadb.org/f/mariadb-10.2.8/source/mariadb-10.2.8.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-md5sum = f93cbd5bfde3c0d082994764ff7db580
+url = https://downloads.mariadb.org/f/mariadb-10.2.9/source/mariadb-10.2.9.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
+md5sum = c59999bd182ddeb3db3d55250aecd8f8
 stable-patches =
 configure-options +=
   -DPLUGIN_DAEMON_EXAMPLE=NO
@@ -10,4 +10,6 @@ configure-options +=
   -DPLUGIN_MROONGA=NO
   -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
   -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
-extra_library_path = :${gcc:location}/lib:${gcc:location}/lib64
+extra_cflags = -I${zstd:location}/include
+extra_include_path = :${zstd:location}/include
+extra_library_path = :${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64

component/mosh/buildout.cfg

+[buildout]
+extends =
+  ../pkgconfig/buildout.cfg
+  ../protobuf/buildout.cfg
+  ../ncurses/buildout.cfg
+  ../perl/buildout.cfg
+  ../zlib/buildout.cfg
+  ../openssl/buildout.cfg
+
+[mosh]
+recipe = slapos.recipe.cmmi
+url = https://mosh.org/mosh-1.3.0.tar.gz
+md5sum = d961276995936953bf2d5a794068b076
+configure-options =
+  --with-curses=${ncurses:location}
+environment =
+  PATH=${perl:location}/bin:${pkgconfig:location}/bin:${protobuf-cpp:location}/bin:%(PATH)s
+  CXXFLAGS =-I${zlib:location}/include
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${protobuf-cpp:location}/lib -Wl,-rpath=${protobuf-cpp:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
+  PKG_CONFIG_PATH=${protobuf-cpp:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${ncurses:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig

component/perl-DBD-MySQL/buildout.cfg

@@ -13,8 +13,8 @@ recipe = slapos.recipe.cmmi
 depends =
   ${perl-DBI:location}
   ${perl-Devel-CheckLib:location}
-url = http://www.cpan.org/modules/by-module/DBD/DBD-mysql-4.042.tar.gz
-md5sum = a144bd950b55af68835d44bc4ea6e5aa
+url = http://www.cpan.org/modules/by-module/DBD/DBD-mysql-4.043.tar.gz
+md5sum = 4a00dd7f1c057931147c65dfc4901c36
 patches =
   ${:_profile_base_location_}/DBD-mysql-4.027.rpathsupport.patch#a932982b7725e6621cfce3a3d7917e03
   ${:_profile_base_location_}/DBD-mysql-4.042.mariadb.patch#5864d36d19c4a05034b3a4873f7c659a

component/perl/buildout.cfg

@@ -7,9 +7,9 @@ parts =
 
 [perl]
 recipe = slapos.recipe.cmmi
-version = 5.26.0
+version = 5.26.1
 url = http://www.cpan.org/src/5.0/perl-${:version}.tar.bz2
-md5sum = 8d34cb5a4eccd66e7a6a80e62b7b4aec
+md5sum = 467cd0c43514b9b5e397c8b385581f53
 siteprefix = ${buildout:parts-directory}/site_${:_buildout_section_name_}
 patch-options = -p1
 patches =

component/protobuf/buildout.cfg

@@ -6,3 +6,12 @@ recipe = slapos.recipe.cmmi
 url = https://github.com/google/protobuf/releases/download/v3.1.0/protobuf-python-3.1.0.tar.gz
 md5sum = 7a227a21379a2ea08cc5d7ba1fb1ba5b
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[protobuf-cpp]
+recipe = slapos.recipe.cmmi
+url = https://github.com/google/protobuf/releases/download/v3.1.0/protobuf-cpp-3.1.0.tar.gz
+md5sum = bd5e3eed635a8d32e2b99658633815ef
+configure-command =
+  ./autogen.sh
+  ./configure --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+

component/python-2.7/buildout.cfg

@@ -28,9 +28,9 @@ python = python2.7
 
 [python2.7]
 recipe = slapos.recipe.cmmi
-package_version = 2.7.13
+package_version = 2.7.14
 package_version_suffix =
-md5sum = 53b43534153bb2a0363f08bae8b9d990
+md5sum = 1f6db41ad91d9eb0a6f0c769b8613c5b
 
 # This is actually the default setting for prefix, but we can't use it in
 # other settings in this part if we don't set it explicitly here.

component/python-mysqlclient/buildout.cfg

@@ -2,7 +2,6 @@
 extends =
   ../mariadb/buildout.cfg
   ../openssl/buildout.cfg
-  ../patch/buildout.cfg
   ../zlib/buildout.cfg
 
 parts =
@@ -14,11 +13,6 @@ PATH =${mariadb:location}/bin:%(PATH)s
 [python-mysqlclient]
 recipe = zc.recipe.egg:custom
 egg = mysqlclient
-patches =
-  https://github.com/PyMySQL/mysqlclient-python/commit/1693848c9f6ca863868d94d63499830f7f4f3a1f.diff#a493a91f9263243eb331fcab9901b8b0
-  ${:_profile_base_location_}/mysqlclient-1.3.10-mariadb-10.2.8.patch#807b694fcd7ade4da4bcca321b1a29d2
-patch-options = -p1
-patch-binary = ${patch:location}/bin/patch
 environment = python-mysqlclient-env
 library-dirs =
   ${zlib:location}/lib/

component/python-mysqlclient/mysqlclient-1.3.10-mariadb-10.2.8.patch

-diff -ur mysqlclient-1.3.10.orig/_mysql.c mysqlclient-1.3.10/_mysql.c
---- mysqlclient-1.3.10.orig/_mysql.c	2017-01-04 13:47:08.000000000 +0100
-+++ mysqlclient-1.3.10/_mysql.c	2017-08-21 10:53:21.014929937 +0200
-@@ -1060,7 +1060,7 @@
- 	if (self && PyModule_Check((PyObject*)self))
- 		self = NULL;
- 	if (self && self->open) {
--#if MYSQL_VERSION_ID >= 50707 && !defined(MARIADB_BASE_VERSION)
-+#if MYSQL_VERSION_ID >= 50707 && (!defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID))
- 		len = mysql_real_escape_string_quote(&(self->connection), out, in, size, '\'');
- #else
- 		len = mysql_real_escape_string(&(self->connection), out, in, size);
-@@ -1118,7 +1118,7 @@
- 	out = PyBytes_AS_STRING(str);
- 	check_server_init(NULL);
- 	if (self && self->open) {
--#if MYSQL_VERSION_ID >= 50707 && !defined(MARIADB_BASE_VERSION)
-+#if MYSQL_VERSION_ID >= 50707 && (!defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID))
- 		len = mysql_real_escape_string_quote(&(self->connection), out+1, in, size, '\'');
- #else
- 		len = mysql_real_escape_string(&(self->connection), out+1, in, size);
-diff -ur mysqlclient-1.3.10.orig/setup_posix.py mysqlclient-1.3.10/setup_posix.py
---- mysqlclient-1.3.10.orig/setup_posix.py	2016-07-26 10:12:24.000000000 +0200
-+++ mysqlclient-1.3.10/setup_posix.py	2017-08-21 10:54:59.643071374 +0200
-@@ -63,7 +63,7 @@
-         if extra_compile_args[i] == '-arch':
-             extra_link_args += ['-arch', extra_compile_args[i + 1]]
- 
--    include_dirs = [dequote(i[2:])
-+    include_dirs = [dequote(os.path.isdir(i[2:]+'/server') and i[2:]+'/server' or i[2:])
-                     for i in mysql_config('include') if i.startswith('-I')]
- 
-     if static:

component/vim/buildout.cfg

@@ -4,12 +4,13 @@ parts =
   
 extends = 
   ../ncurses/buildout.cfg
+  ../gettext/buildout.cfg
 
 [vim]
 recipe = slapos.recipe.cmmi
-version = 7.4
-url = http://ftp.vim.org/pub/vim/unix/vim-7.4.tar.bz2
-md5sum = 607e135c559be642f210094ad023dc65
+url = ftp://ftp.vim.org/pub/vim/unix/vim-8.0.586.tar.bz2
+md5sum = b35e794140c196ff59b492b56c1e73db
 environment=
-    CFLAGS=-I${ncurses:location}/include
-    LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/
+    CPPFLAGS=-I${ncurses:location}/include
+    LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
\ No newline at end of file
+    LD_LIBRARY_PATH=${ncurses:location}/lib:${gettext:location}/lib

component/zlib/buildout.cfg

@@ -4,5 +4,5 @@ parts =
 
 [zlib]
 recipe = slapos.recipe.cmmi
-url = http://downloads.sourceforge.net/project/libpng/zlib/1.2.8/zlib-1.2.8.tar.gz
-md5sum = 44d667c142d7cda120332623eab69f40
+url = http://downloads.sourceforge.net/project/libpng/zlib/1.2.11/zlib-1.2.11.tar.gz
+md5sum = 1c9f62f0778697a09d36121ead88e08e

component/zstd/buildout.cfg

+[buildout]
+parts =
+  zstd
+
+[zstd]
+recipe = slapos.recipe.cmmi
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+url = https://github.com/facebook/zstd/archive/v1.3.1.tar.gz
+md5sum = e849ceef2f090240f690c13fba6ca70b
+configure-command = :
+make-options = PREFIX=${:location}

slapos/test/recipe/test_slaposconfiguration.py

@@ -3,6 +3,7 @@ import json
 import mock
 import os
 import unittest
+import tempfile
 from collections import defaultdict
 from slapos.recipe import slapconfiguration
 from slapos import format as slapformat
@@ -12,8 +13,7 @@ class SlapConfigurationTest(unittest.TestCase):
 
   def setUp(self):
     """Prepare files on filesystem."""
-    self.instance_root = "/tmp/instance_test_resourcefile"
-    os.mkdir(self.instance_root)
+    self.instance_root = tempfile.mkdtemp()
     # create testing resource file
     self.resource_file = os.path.join(self.instance_root, slapformat.Partition.resource_file)
     self.resource = {

software/caucase/instance-caucase-input-schema.json

@@ -50,13 +50,13 @@
     "crl-life-period": {
       "title": "CRL life time period",
       "description": "Number of individual certificate validity periods during which the CRL is valid. Default: 1/50.0",
-      "type": "float",
+      "type": "number",
       "default": 0.2
     },
     "ca-life-period": {
       "title": "CA Certificate life period",
       "description": "Number of individual certificate validity periods during which the CA certificate is valid. Default: 10",
-      "type": "float",
+      "type": "number",
       "default": 10
     },
     "crt-keep-time": {

software/dream/software.cfg

@@ -48,7 +48,7 @@ scipy = 0.13.3
 simpy = 3.0.5
 zope.dottedname = 4.1.0
 tablib = 0.10.0
-mysqlclient = 1.3.10+SlapOSPatched002
+mysqlclient = 1.3.12
 
 # indirect dependancies
 cp.recipe.cmd = 0.5

software/erp5testnode/testsuite/deploy-test/deploy-script-controller

@@ -55,10 +55,20 @@ if [[ ! -s "$DEPLOYMENT_SCRIPT" ]] ; then
   echo "exit 1" > $DEPLOYMENT_SCRIPT
 fi
 
+function add_log ()
+{
+  LOG_FILE=$1
+  for f in /opt/slapos/log/slapos-node-{software,instance}.log ; do
+    echo "Tail of '$f':" >> $LOG_FILE
+    tail -n 500 $f >> $LOG_FILE
+  done
+}
+
 function upload ()
 {
   try=$1
   LOG_FILE=$2
+  add_log $LOG_FILE
   t=`date '+%Y%m%d%H%S'`
   mv $LOG_FILE ${LOG_FILE}.$t
   # just to be sure flush all disk operations before uploading

software/erp5testnode/testsuite/deploy-test/software.cfg

@@ -55,7 +55,7 @@ output = ${buildout:directory}/template-original.kvm.cfg
 [deploy-script-controller-script]
 filename = deploy-script-controller
 location = ${:_profile_base_location_}/${:filename}
-md5sum = d2b92f45257a52e5a7ff5c311d46d4ae
+md5sum = 31aadc895acf9fc2fc6e1cbe815339c6
 # configuration
 waittime = 360
 tries = 80

software/kvm/common.cfg

@@ -89,7 +89,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = f40a938400e789361c95d5a9246bf0ef 
+md5sum = bf5ef731c0d8da0267a4939882b4eeee 
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -98,7 +98,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = a849d4a6060fdb4e9e86917fb77ef153
+md5sum = e2b8f86bdc12c86e7d959b55c6d54f6d
 download-only = true
 on-update = true
 
@@ -107,7 +107,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = d9745bc9bd1d22a640a628c005f88ffb
+md5sum = 05b6004e8c7a94de14f247affcef4971
 download-only = true
 on-update = true
 
@@ -143,7 +143,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
 mode = 644
-md5sum = 13387d37bbf430f1d2b827c8f1acc804
+md5sum = fbad91193be6ebde5fc4c05a38a55e7b
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-cluster-input-schema.json

@@ -157,12 +157,6 @@
       "description": "Text content which will be written in a file data of cluster http server. All VM will be able to download that file via the static URL of cluster HTTP server: https://10.0.2.101/FOLDER_HASH/data.",
       "type": "string"
     },
-    "enable-monitor": {
-      "title": "Enable Monitoring on this cluster",
-      "description": "Deploy monitor instance to this kvm instance. It help to check instance status, log and promise results.",
-      "type": "boolean",
-      "default": true
-    },
     "monitor-interface-url": {
         "title": "Monitor Web Interface URL",
         "description": "Give Url of HTML web interface that will be used to render this monitor instance.",
@@ -341,6 +335,20 @@
               "default": "qcow2",
               "enum": ["qcow2", "raw", "vdi", "vmdk", "cloop", "qed"]
             },
+            "wipe-disk-ondestroy": {
+              "title": "Wipe disks when destroy the VM",
+              "description": "Say if disks should be wiped by writing new data over every single bit before delete them. This option is used to securely delete VM disks",
+              "type": "boolean",
+              "default": false
+            },
+            "wipe-disk-iterations": {
+              "title": "Wipe disk iterations",
+              "description": "Number of disk overwrite iterations with random data. Default is 1. WARNING: Increase this value will slow down partition destruction and increase IO.",
+              "type": "integer",
+              "default": 1,
+              "minimum": 1,
+              "maximum": 5
+            },
             "use-tap": {
               "title": "Enable QEMU TAP network interface",
               "description": "Use QEMU TAP network interface, might require a bridge on SlapOS Node.",

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -9,7 +9,13 @@
 {% set kvm_instance_dict = {} -%}
 {% set kvm_hostname_list = [] -%}
 {% set monitor_url_list = [] -%}
-{% set enable_monitoring = slapparameter_dict.get('enable-monitor', True) -%}
+
+{% macro setconfig(name, value) -%}
+{# will set a config-name = value if value is not empty -#}
+{% if value and value != '' -%}
+config-{{ name }} = {{ dumps(value) }}
+{% endif -%}
+{% endmacro -%}
 
 [request-common]
 recipe = slapos.cookbook:request
@@ -37,36 +43,38 @@ state = stopped
 config-frontend-instance-name = {{ instance_name ~ ' VNC Frontend' }}
 config-frontend-software-type = {{ dumps(frontend_dict.get('software-type', 'frontend')) }}
 config-frontend-software-url = {{ dumps(frontend_dict.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg')) }}
-config-frontend-instance-guid = {{ dumps(frontend_dict.get('instance-guid', '')) }}
+{{ setconfig('frontend-instance-guid', kvm_parameter_dict.get('instance-guid', '')) }}
 config-name = {{ instance_name }}
 {% if slapparameter_dict.get('authorized-keys', []) -%}
 config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | join('\n')) }}
 {% endif -%}
 config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
-config-nbd-host = {{ dumps(kvm_parameter_dict.get('nbd-host', '')) }}
 config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
-config-nbd2-host = {{ dumps(kvm_parameter_dict.get('nbd-host2', '')) }}
 config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
 config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
 config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
 config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
-config-cpu-options = {{ dumps(kvm_parameter_dict.get('cpu-options', '')) }}
-config-numa = {{ dumps(kvm_parameter_dict.get('numa', '')) }}
-config-disk-cache = {{ dumps(kvm_parameter_dict.get('disk-cache', '')) }}
-config-disk-aio = {{ dumps(kvm_parameter_dict.get('disk-aio', '')) }}
+{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
+{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
+{{ setconfig('cpu-options', kvm_parameter_dict.get('cpu-options', '')) }}
+{{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
+{{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }}
+
 config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True)) }}
-config-machine-options = {{ dumps(kvm_parameter_dict.get('machine-options', '')) }}
-config-cpu-model = {{ dumps(kvm_parameter_dict.get('cpu-model', '')) }}
+{{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
+{{ setconfig('disk-aio', kvm_parameter_dict.get('disk-aio', '')) }}
+{{ setconfig('cpu-model', kvm_parameter_dict.get('cpu-model', '')) }}
+{{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
 
 {% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
-config-nat-rules = {{ nat_rules_list | join(' ') }}
+{{ setconfig('nat-rules', ' '.join(nat_rules_list)) }}
 config-publish-nat-url = True
 config-use-nat = {{ use_nat }}
 config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
 config-nat-restrict-mode = {{ dumps(kvm_parameter_dict.get('nat-restrict-mode', False)) }}
 config-enable-vhost = {{ dumps(kvm_parameter_dict.get('enable-vhost', False)) }}
-config-virtual-hard-drive-url = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-url', '')) }}
-config-virtual-hard-drive-md5sum = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-md5sum', '')) }}
+{{ setconfig('virtual-hard-drive-url', kvm_parameter_dict.get('virtual-hard-drive-url', '')) }}
+{{ setconfig('virtual-hard-drive-md5sum', kvm_parameter_dict.get('virtual-hard-drive-md5sum', '')) }}
 config-virtual-hard-drive-gzipped = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-gzipped', False)) }}
 config-hard-drive-url-check-certificate = {{ dumps(kvm_parameter_dict.get('hard-drive-url-check-certificate', True)) }}
 config-external-disk-number = {{ dumps(kvm_parameter_dict.get('external-disk-number', 0)) }}
@@ -74,15 +82,18 @@ config-external-disk-size = {{ dumps(kvm_parameter_dict.get('external-disk-size'
 config-external-disk-format = {{ dumps(kvm_parameter_dict.get('external-disk-format', 'qcow2')) }}
 config-enable-http-server = {{ dumps(kvm_parameter_dict.get('enable-http-server', True)) }}
 config-httpd-port = {{ dumps(kvm_parameter_dict.get('httpd-port', 8081)) }}
-{% if kvm_parameter_dict.get('data-to-vm', '') -%}
-config-data-to-vm = {{ dumps(kvm_parameter_dict.get('data-to-vm', '')) }}
-{% endif -%}
+
+{{ setconfig('data-to-vm', kvm_parameter_dict.get('data-to-vm', '')) }}
+
 config-disable-ansible-promise = {{ dumps(kvm_parameter_dict.get('disable-ansible-promise', False)) }}
-config-enable-monitor = {{ enable_monitoring }}
 config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
 config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
-
+# Enable disk wipe options
+{% if kvm_parameter_dict.get('wipe-disk-ondestroy', False) -%}
+config-wipe-disk-ondestroy = True
+config-wipe-disk-iterations = {{ dumps(kvm_parameter_dict.get('wipe-disk-iterations', 1)) }}
+{% endif -%}
 # Enable simple http server on ipv6 so all VMs will access it
 config-document-host = ${apache-conf:ip}
 config-document-port = ${apache-conf:port}
@@ -109,13 +120,11 @@ return =
 {{ '  ' }}nat-rule-url-{{ port }}
 {%   endfor -%}
 {% endif -%}
-{% if enable_monitoring -%}
 {{ '  ' }}monitor-base-url
-{%    do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
-{% endif -%}
 {% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%}
 {{ '  ' }}tap-ipv4
 
+{% do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
 {% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% endif -%}
@@ -151,7 +160,7 @@ name = Frontend {{ name }}
 software-type = {{ slave_frontend_stype }}
 slave = true
 config-url = {{ url }}
-config-custom_domain = {{ dumps(frontend_parameter_dict.get('domain', '')) }}
+{{ setconfig('custom_domain', kvm_parameter_dict.get('domain', '')) }}
 config-enable_cache = {{ dumps(frontend_parameter_dict.get('enable-cache', False)) }}
 config-https-only = {{ dumps(frontend_parameter_dict.get('https-only', False)) }}
 {%  if frontend_parameter_dict.get('type', '') -%}
@@ -245,18 +254,14 @@ recipe = slapos.cookbook:publish
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor %}
-{% if enable_monitoring -%}
-{%   set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
-{% do part_list.append('monitor-base') -%}
+{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
 monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
-{% endif -%}
+{% do part_list.append('monitor-base') -%}
 
 [buildout]
 extends = 
   {{ template_httpd_cfg }}
-{% if enable_monitoring -%}
 {{ '  ' ~ template_monitor }}
-{% endif -%}
 
 parts = 
   httpd

software/kvm/instance-kvm-export.cfg.jinja2

@@ -34,17 +34,17 @@ context =
 # Extends publish section with resilient parameters
 [publish-connection-information]
 <= resilient-publish-connection-parameter
+monitor-base-url = ${monitor-publish:monitor-base-url}
+monitor-setup-url = ${monitor-publish:monitor-setup-url}
 
-{% if str(slapparameter_dict.get('enable-monitor', True)).lower() == 'true' -%}
 [monitor-instance-parameter]
 monitor-httpd-port = 8026
 monitor-title = {{ slapparameter_dict.get('name', 'KVM Standalone') }}
 cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-{%   if slapparameter_dict.get('monitor-username', '') -%}
+{% if slapparameter_dict.get('monitor-username', '') -%}
 username = {{ slapparameter_dict['monitor-username'] }}
-{%   endif -%}
-{%   if slapparameter_dict.get('monitor-password', '') -%}
+{% endif -%}
+{% if slapparameter_dict.get('monitor-password', '') -%}
 password = {{ slapparameter_dict['monitor-password'] }}
-{%   endif -%}
-
 {% endif -%}
+

software/kvm/instance-kvm-input-schema.json

@@ -159,6 +159,21 @@
       "enum": ["qcow2", "raw", "vdi", "vmdk", "cloop", "qed"]
     },
 
+    "wipe-disk-ondestroy": {
+      "title": "Wipe disks when destroy the VM",
+      "description": "Say if disks should be wiped by writing new data over every single bit before delete them. This option is used to securely delete VM disks",
+      "type": "boolean",
+      "default": false
+    },
+    "wipe-disk-iterations": {
+      "title": "Wipe disk iterations",
+      "description": "Number of disk overwrite iterations with random data. Default is 1. WARNING: Increase this value will slow down partition destruction and increase IO.",
+      "type": "integer",
+      "default": 1,
+      "minimum": 1,
+      "maximum": 5
+    },
+
     "use-tap": {
       "title": "Use QEMU TAP network interface",
       "description": "Use QEMU TAP network interface, might require a bridge on SlapOS Node.",
@@ -188,12 +203,6 @@
       "type": "boolean",
       "default": false
     },
-    "enable-monitor": {
-      "title": "Deploy monitoring tools",
-      "description": "Deploy monitor instance to this kvm instance. It help to check instance status, log and promise results.",
-      "type": "boolean",
-      "default": true
-    },
     "monitor-interface-url": {
         "title": "Monitor Web Interface URL",
         "description": "Give Url of HTML web interface that will be used to render this monitor instance.",

software/kvm/instance-kvm.cfg.jinja2

 {% set enable_http = slapparameter_dict.get('enable-http-server', 'False').lower() -%}
 {% set use_tap = slapparameter_dict.get('use-tap', 'False').lower() -%}
 {% set use_nat = slapparameter_dict.get('use-nat', 'True').lower() -%}
+{% set wipe_disk = slapparameter_dict.get('wipe-disk-ondestroy', 'False').lower() -%}
 {% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
 {% set name = slapparameter_dict.get('name', 'localhost') -%}
 {% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
@@ -9,7 +10,6 @@
 {% set frontend_software_type = 'default' -%}
 {% set extends_list = [] -%}
 {% set part_list = [] -%}
-{% set monitor = str(slapparameter_dict.get('enable-monitor', True)).lower() == 'true' -%}
 
 {% set bootstrap_url = '' -%}
 {% set bootstrap_url_md5sum = '' -%}
@@ -26,10 +26,9 @@
 {%   set nat_rule_list = '' %}
 {% endif -%}
 
-{% if monitor -%}
 {% do extends_list.append(template_monitor) -%}
-{% endif -%}
 {% do extends_list.append(logrotate_cfg) -%}
+
 [directory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc
@@ -42,6 +41,7 @@ services = ${:etc}/service
 promises = ${:etc}/promise
 novnc-conf = ${:etc}/novnc
 run = ${:var}/run
+prerm = ${:etc}/prerm
 ca-dir = ${:srv}/ssl
 public = ${:srv}/public/
 cron-entries = ${:etc}/cron.d
@@ -219,6 +219,19 @@ input = inline:#!/bin/sh
 output = ${directory:promises}/kvm-disk-image-corruption
 mode = 700
 
+{% if wipe_disk == 'true' -%}
+{% do part_list.append('wipe-disk-wrapper') -%}
+{% set wipe_file_list = '${kvm-parameter-dict:disk-path}' -%}
+{% if storage_dict -%}
+{% set wipe_file_list = '${kvm-parameter-dict:disk-path}' ~ ' ' ~ '/* '.join(storage_dict.values()) ~ '/*' -%}
+{% endif -%}
+[wipe-disk-wrapper]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:prerm}/slapos_wipe_qemu_disk
+command-line = 
+  {{ wipe_disk_wrapper }} -n {{ slapparameter_dict.get('wipe-disk-iterations', 1) }} -suz --check-pid-file ${kvm-parameter-dict:pid-file-path} --file {{ wipe_file_list }}
+{% endif -%}
+
 [kvm-started-promise]
 recipe = slapos.recipe.template:jinja2
 template = {{ qemu_start_promise_tpl }}
@@ -343,22 +356,20 @@ hostname = ${httpd:host}
 port = ${httpd:port}
 {% endif %}
 
-{% if monitor -%}
-
 [monitor-instance-parameter]
 monitor-httpd-port = 8026
 monitor-title = {{ slapparameter_dict.get('name', 'KVM Standalone') }}
 cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-{%   if slapparameter_dict.get('monitor-username', '') -%}
+{% if slapparameter_dict.get('monitor-username', '') -%}
 username = {{ slapparameter_dict['monitor-username'] }}
-{%   endif -%}
-{%   if slapparameter_dict.get('monitor-password', '') -%}
+{% endif -%}
+{% if slapparameter_dict.get('monitor-password', '') -%}
 password = {{ slapparameter_dict['monitor-password'] }}
-{%   endif -%}
-
 {% endif -%}
+interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
 
 [publish-connection-information]
+<= monitor-publish
 recipe = slapos.cookbook:publish
 ipv6 = ${slap-network-information:global-ipv6}
 backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
@@ -392,13 +403,6 @@ tap-ipv4 = ${slap-network-information:tap-ipv4}
 7_info = Get the publick key file in your VM with the command: wget {{ kvm_http }}/authorized_keys
 {%   endif %}
 {% endif %}
-{% if monitor -%}
-monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
-{%   set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
-{%   if monitor_interface_url -%}
-monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
-{%   endif -%}
-{% endif -%}
 
 {% if use_tap == 'true' and tap_network_dict.has_key('ipv4') -%}
 1_info = Use these configurations below to configure interface {{ iface }} in your VM.
@@ -631,10 +635,8 @@ parts =
   cron
   cron-entry-logrotate
   frontend-promise
-{% if monitor -%}
 # monitor parts
   monitor-base
-{% endif -%}
 # Complete parts with sections
   {{ part_list | join('\n  ') }}
 

software/kvm/instance.cfg.in

@@ -103,6 +103,7 @@ context =
     raw template_kvm_run ${template-kvm-run:location}/${template-kvm-run:filename}
     raw template_monitor ${monitor2-template:rendered}
     raw websockify_executable_location ${buildout:directory}/bin/websockify
+    raw wipe_disk_wrapper ${buildout:directory}/bin/securedelete
 template-parts-destination = ${template-parts:target}
 template-replicated-destination = ${template-replicated:target}
 import-list = file parts :template-parts-destination

software/neoppod/software-common.cfg

@@ -112,7 +112,7 @@ apache-libcloud = 1.5.0
 ecdsa = 0.13
 gitdb2 = 2.0.0
 msgpack-python = 0.4.8
-mysqlclient = 1.3.10+SlapOSPatched002
+mysqlclient = 1.3.12
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0

software/slaprunner/buildout.hash.cfg

@@ -31,7 +31,7 @@ md5sum = 5cfa49bcf20612844e1c50a85740d0b3
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
-md5sum = 94c0eddb2af3290942f64e04f95a707c
+md5sum = 78b2454951e6dbe124612eafebcf5690
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in

software/slaprunner/common.cfg

@@ -153,10 +153,7 @@ eggs =
   plone.recipe.command
   slapos.recipe.build
   slapos.toolbox[flask_auth]
-# gunicorn downgraded because of bug in latter versions
-# setting version here allows to use an older version in tests
-# see more here : https://lab.nexedi.com/nexedi/slapos/commit/391b026e22b05e9a6fba5e063f64f18544a34b92
-  gunicorn==19.4.5
+  gunicorn==19.7.1
   futures
   ${slapos-cookbook:eggs}
 

software/slaprunner/template/runner-export.sh.jinja2

@@ -19,9 +19,21 @@ etc_directory='{{ directory["etc"] }}'
 tmp_directory='{{ directory["tmp"] }}'
 
 rsync () {
+  # Workaround for bug https://bugzilla.samba.org/show_bug.cgi?id=3653
+  IGNOREEXIT=24
+  IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
+  set -o pipefail
+
   set -x
-  '{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@"
+  '{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@" 2>1 | (egrep -v "$IGNOREOUT" || true)
   set +x
+
+  ret=$?
+  if [[ $ret == $IGNOREEXIT ]]; then
+    ret=0
+  fi
+
+  exit $ret
 }
 
 (

stack/boinc/buildout.cfg

@@ -69,7 +69,7 @@ pycrypto = 2.6
 apache-libcloud = 0.12.4
 async = 0.6.1
 gitdb = 0.5.4
-mysqlclient = 1.3.10+SlapOSPatched002
+mysqlclient = 1.3.12
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.4.2
 slapos.toolbox = 0.40.4

stack/erp5/TODO

@@ -8,11 +8,6 @@ General:
 - make postfix log inside partition
 - document postfix parameters (only once it actually works)
 
-Monitoring:
-- daily slow-query digest
-  make percona toolkit available in mysql instance and decide how to send digest
-- daily apachedex
-
 Backups:
 - flush binlogs independently from full backups (in addition to anyway flushing them on full backup creation)
 - rotate tidstorage consistency points

stack/erp5/buildout.cfg

@@ -171,10 +171,6 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 <= download-base
 mode = 755
 
-[mariadb-slowquery-check-script]
-<= download-base
-mode = 755
-
 [mariadb-slow-query-report-script]
 <= download-base
 mode = 755
@@ -233,8 +229,6 @@ context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
     key apache_location apache:location
-    key apdex_result_check_script apdex-result-check-script:target
-    key mariadb_slowquery_check_script mariadb-slowquery-check-script:target 
     key aspell_location aspell:location
     key bin_directory buildout:bin-directory
     key buildout_bin_directory buildout:bin-directory
@@ -353,9 +347,6 @@ link-binary =
 [template-balancer]
 <= download-base
 
-[apdex-result-check-script]
-<= download-base
-
 [template-haproxy-cfg]
 <= download-base
 
@@ -622,6 +613,8 @@ eggs +=
 scripts +=
   is-local-tcp-port-opened
   is-process-older-than-dependency-set
+  check-slow-queries-digest-result
+  check-apachedex-result
   runApacheDex
   zodbpack
 

stack/erp5/buildout.hash.cfg

@@ -21,13 +21,9 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
 filename = mysql-querydigest.sh.in
 md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
 
-[mariadb-slowquery-check-script]
-filename = instance-mariadb-check-slowquery-result.sh.in 
-md5sum = 356e0e2db1da0e8b479908fb739e5cc0
-
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 7ee2e801dda1181d1b42281e6466fc4d
+md5sum = 99450bb6426770bd0c27f9dbec9ca542
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -71,7 +67,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template]
 filename = instance.cfg.in
-md5sum = 258146a9d979b9981d18875fcac4de73
+md5sum = f95174eee2de45d1e3ccb1d5f88355f1
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -91,11 +87,7 @@ md5sum = 27d26c6380883cf3bd7b2f003f7888d8
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = d14ee7f13e2bd815cc96e28101e59670
-
-[apdex-result-check-script]
-filename = instance-balancer-check-apachedex-result.sh.in 
-md5sum = 421c68c97cadc49911382cd3185288a1
+md5sum = c80648d5918ae865bc02cb8fccfc56e3
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer-check-apachedex-result.sh.in

-#!{{ bash }}
-
-set -e
-
-APACHEDEX_FILE='{{ apdex_file }}/ApacheDex-'$(date +%Y-%m-%d)'.html'
-APACHEDEX_REPORT_JSON_FILE={{ apdex_status_file }} 
-DESIRED_THRESHOLD={{ user_threshold }}
-
-# Check if the file is there
-if [ ! -s "$APACHEDEX_FILE" ]; then 
-  # If file doesn't exists create one
-  # If it is empty check for modification time
-  if [ ! -f "$APACHEDEX_FILE" ]; then
-    touch $APACHEDEX_FILE
-  else
-    MODIFIED_DATE=`stat -c '%Z' $APACHEDEX_FILE`
-    CURRENT_DATE=`date +%s`
-    if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
-    then 
-      echo "File modification date is greater than 30 hours"
-      JSON_CONTENT=`cat $APACHEDEX_REPORT_STATUS_FILE`
-      MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
-      echo $MESSAGE
-      exit 2
-    else
-      echo "File is empty for now"
-    fi
-  fi
-else
-  # Check if the result exists
-  {
-    REGEX="Overall<\/h2><table .*><tr>[[:space:]]<th>apdex<\/th><th>.*?<tr>[[:space:]]<td [^<]*>(.*?)%<\/td>"
-    FILE_CONTENT=`cat $APACHEDEX_FILE`
-    if [[ $FILE_CONTENT =~ $REGEX ]]
-    then
-      RESULT=${BASH_REMATCH[1]}
-      RESULT=${RESULT:-0}
-      if [[ `echo "$RESULT > $DESIRED_THRESHOLD" | bc` -eq 1 ]]
-      then
-        echo "Your score is $RESULT %, Thanks for keeping it all clean"
-      else
-        echo "Threshold is lower than exptected: Expected was $DESIRED_THRESHOLD % and we current is $RESULT %"
-        exit 2
-      fi
-    else
-      echo "No threshold found in the result"
-    fi
-  } || {
-    echo "Cannot parse the apdex result"
-  }
-fi

stack/erp5/instance-balancer.cfg.in

@@ -298,15 +298,10 @@ configuration = {{ slapparameter_dict['apachedex-configuration'] }}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]
-recipe = slapos.recipe.template:jinja2
-template = {{ parameter_dict['apdex-result-check-script'] }}
-rendered = ${monitor-directory:promises}/check-apachedex-result
-status-file = ${monitor-directory:private}/apachedex.report.json 
-context = 
-  raw bash {{ parameter_dict['bash'] }}/bin/bash
-  raw user_threshold ${apachedex-parameters:promise-threshold}
-  key apdex_file directory:apachedex
-  key apdex_status_file :status-file
+recipe = slapos.cookbook:wrapper
+recipe =
+wrapper-path = ${directory:promise}/check-apachedex-result
+command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_file "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}

stack/erp5/instance-mariadb-check-slowquery-result.sh.in

-#!{{ bash }}
-set -e
-
-DIGEST_FILE='{{ slow_query_digest }}/slowquery_digest.txt'
-SLOW_QUERY_STATUS_FILE='{{ slow_query_status }}'
-DESIRED_MAX_QUERY_THRESHOLD={{ max_queries_threshold }}
-DESIRED_SLOW_QUERY_THRESHOLD={{ slowest_queries_threshold  }}
-
-# Check if the file is there
-if [ ! -s "$DIGEST_FILE" ]; then 
-  # If file doesn't exists create one
-  # If it is empty check for modification time
-  if [ ! -f "$DIGEST_FILE" ]; then
-    touch $DIGEST_FILE
-  else
-    MODIFIED_DATE=`stat -c '%Z' $DIGEST_FILE`
-    CURRENT_DATE=`date +%s`
-    if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
-    then 
-      echo "File modification date is greater than 30 hours"
-      JSON_CONTENT=`cat $SLOW_QUERY_STATUS_FILE`
-      MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
-      echo $MESSAGE 
-      exit 2
-    else
-      echo "File is empty for now"
-    fi
-  fi
-else
-  # Check if the result exists
-  {
-    # get the total number of queries ran and the max time
-    # TODO: improve regex
-    # TODO: improve the parameters (currently we are using threshold on queries and max execute time)
-    # # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
-    REGEX="# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
-    FILE_CONTENT=`cat $DIGEST_FILE`
-    if [[ $FILE_CONTENT =~ $REGEX ]]
-    then
-      TOTAL_QUERIES_EXEC=${BASH_REMATCH[1]}
-      SLOWEST_QUERY_TIME=${BASH_REMATCH[2]}
-      HAS_K="${TOTAL_QUERIES_EXEC: -1}"
-      if [[ "$HAS_K" == "k" ]] 
-      then
-        PRE="${TOTAL_QUERIES_EXEC::-1}"
-        TOTAL_QUERIES_EXEC=$(echo "scale=4; ${PRE:-0}*1000" | bc)
-      else
-        TOTAL_QUERIES_EXEC=${TOTAL_QUERIES_EXEC:-0}
-      fi
-      # TODO: support ms
-      SLOWEST_QUERY_TIME="${SLOWEST_QUERY_TIME:-0}"
-      if [[ `echo "$TOTAL_QUERIES_EXEC < $DESIRED_MAX_QUERY_THRESHOLD" | bc` -eq 1 && `echo "$SLOWEST_QUERY_TIME < $DESIRED_SLOW_QUERY_THRESHOLD" | bc` -eq 1 ]]
-      then
-        echo "Total number of slow queries are: $TOTAL_QUERIES_EXEC"
-        echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME"
-        echo "Thanks for keeping it all clean"
-      else
-        echo "Ops! One of the two expected parameters did not meet"
-        echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME s and required maximum is $DESIRED_SLOW_QUERY_THRESHOLD s"
-        echo "Total slow queries are $TOTAL_QUERIES_EXEC and expected maximum value is $DESIRED_MAX_QUERY_THRESHOLD"
-        exit 2
-      fi
-    else
-      echo "No threshold found in the result"
-    fi
-  } || {
-    echo "Cannot parse the result"
-  }
-fi

stack/erp5/instance-mariadb.cfg.in

@@ -295,17 +295,11 @@ max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
 slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
 
 [{{ section('monitor-promise-slowquery-result') }}]
-recipe = slapos.recipe.template:jinja2
-template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
-rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
-status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
-context =
-  raw default_threshold 4000
-  raw bash {{ parameter_dict['bash'] }}/bin/bash
-  key slow_query_digest monitor-directory:private
-  key slow_query_status :status-file
-  key max_queries_threshold slow-query-digest-parameters:max_queries_threshold
-  key slowest_queries_threshold slow-query-digest-parameters:slowest_queries_threshold
+recipe = slapos.cookbook:wrapper
+recipe =
+wrapper-path = ${directory:promise}/check-slow-query-pt-digest-result
+command-line = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_file "${monitor-directory:private}/slowquery_digest.txt" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}"
+
 
 [{{ section('promise') }}]
 recipe = slapos.cookbook:wrapper

stack/erp5/instance.cfg.in

@@ -93,12 +93,12 @@ openssl-location = {{ openssl_location }}
 
 [dynamic-template-balancer-parameters]
 apache = {{ apache_location }}
-apdex-result-check-script = {{ apdex_result_check_script }}
 openssl = {{ openssl_location }}
 haproxy = {{ haproxy_location }}
 bin-directory = {{ bin_directory }}
 apachedex-location = {{ bin_directory }}/apachedex
 run-apachedex-location = {{ bin_directory }}/runApacheDex
+promise-check-apachedex-result = {{ bin_directory }}/check-apachedex-result
 6tunnel = {{ sixtunnel_location }}
 curl-location = {{ curl_location }}
 dash = {{ dash_location }}
@@ -183,7 +183,7 @@ link-binary = {{ dumps(mariadb_link_binary) }}
 bin-directory = {{ bin_directory }}
 mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
 mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
-mariadb-slowquery-check-script = {{ mariadb_slowquery_check_script}}
+promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result
 percona-tools-location = {{ percona_toolkit_location }}
 template-monitor = {{ template_monitor }}
 

stack/lamp.cfg

@@ -42,7 +42,7 @@ keep-compile-dir = false
 apache-libcloud = 0.9.1
 async = 0.6.1
 gitdb = 0.5.4
-mysqlclient = 1.3.10+SlapOSPatched002
+mysqlclient = 1.3.12
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.3
 slapos.toolbox = 0.40.4

stack/lamp/buildout.cfg

@@ -185,7 +185,7 @@ eggs =
 apache-libcloud = 0.12.3
 async = 0.6.1
 gitdb = 0.5.4
-mysqlclient = 1.3.10+SlapOSPatched002
+mysqlclient = 1.3.12
 pycrypto = 2.6
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 2.4.2

stack/slapos.cfg

@@ -100,7 +100,7 @@ eggs =
 
 [versions]
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.5.2+slapos009
+zc.buildout = 2.5.2+slapos010
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -136,7 +136,7 @@ requests = 2.13.0
 setuptools = 33.1.1
 six = 1.10.0
 slapos.cookbook = 1.0.53
-slapos.core = 1.4.1
+slapos.core = 1.4.2
 slapos.extension.strip = 0.4
 slapos.libnetworkcache = 0.15
 slapos.rebootstrap = 4.1
@@ -148,7 +148,7 @@ xml-marshaller = 0.9.7
 paramiko = 2.1.3
 
 # Required by:
-# slapos.core==1.4.0
+# slapos.core==1.4.2
 Flask = 0.12
 
 # Required by:
@@ -168,7 +168,7 @@ ipaddress = 1.0.18
 jsonschema = 2.6.0
 
 # Required by:
-# slapos.core==1.4.0
+# slapos.core==1.4.2
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
@@ -181,15 +181,15 @@ packaging = 16.8
 pycparser = 2.17
 
 # Required by:
-# slapos.core==1.4.0
+# slapos.core==1.4.2
 supervisor = 3.3.3
 
 # Required by:
-# slapos.core==1.4.0
+# slapos.core==1.4.2
 uritemplate = 3.0.0
 
 # Required by:
-# slapos.core==1.4.0
+# slapos.core==1.4.2
 zope.interface = 4.3.3
 
 [networkcache]