Update Release Candidate

component/geoip2/buildout.cfg

@@ -16,6 +16,6 @@ rpath = ${:library-dirs}
 
 [geolite2-country]
 recipe = slapos.recipe.build:download-unpacked
-url = http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz
-md5sum = 17ee7db01dc5ac73a09e2009f9e5ed04
+url = http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz#${:md5sum}
+md5sum = a06d162867715573a919f10f6bf36e48
 strip-top-level-dir = true

component/golang/buildout.cfg

@@ -33,21 +33,21 @@ environment-extra =
 make-targets= cd src && ./make.bash && cp -alf .. ${:location}
 
 
-[golang18]
+[golang19]
 <= golang-common
-url = https://dl.google.com/go/go1.8.7.src.tar.gz
-md5sum = c61cfe9c85e7d42f903d3fe146d7cde6
+url = https://dl.google.com/go/go1.9.4.src.tar.gz
+md5sum = 6816441fd6680c63865cdd5cb8bc1960
 
-# go1.8 needs go1.4 to bootstrap
+# go1.9 needs go1.4 to bootstrap
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
-[golang19]
+[golang1.10]
 <= golang-common
-url = https://dl.google.com/go/go1.9.4.src.tar.gz
-md5sum = 6816441fd6680c63865cdd5cb8bc1960
+url = https://dl.google.com/go/go1.10.3.src.tar.gz
+md5sum = d15dfb264105c5e84fbe33f4a4aa5021
 
-# go1.9 needs go1.4 to bootstrap
+# go1.10 needs go1.4 to bootstrap
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
@@ -84,7 +84,7 @@ bin	= ${:directory}/bin
 depends = ${gowork.goinstall:recipe}
 
 # go version used for the workspace (possible to override in applications)
-golang  = ${golang19:location}
+golang  = ${golang1.10:location}
 
 # no special build flags by default
 buildflags =

component/groonga/buildout.cfg

@@ -14,8 +14,8 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-8.0.0.tar.gz
-md5sum = f4641e68a5301e641399f2d112a494dd
+url = https://packages.groonga.org/source/groonga/groonga-8.0.5.tar.gz
+md5sum = 72ce790646ce561c246ef4e2ea961f79
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/mariadb/buildout.cfg

@@ -17,8 +17,6 @@ extends =
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
   ../unixodbc/buildout.cfg
-# The following lines are only for mariarocks.cfg
-# to be extended last without touching 'parts'.
   ../gcc/buildout.cfg
   ../zstd/buildout.cfg
 
@@ -28,11 +26,11 @@ parts =
 [mariadb]
 recipe = slapos.recipe.cmmi
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.1.34
-md5sum = 2fc6c7809be74e9e7718e8863df26b2b
+version = 10.2.16
+md5sum = 234156a94f8ea6cff92c4751880b8316
 patch-options = -p0
 patches =
-  ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
+  ${:_profile_base_location_}/mariadb_10.2.16_create_system_tables__no_test.patch#3fd5f9febabdb42d4b6653969a0194f9
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
@@ -63,14 +61,13 @@ configure-options =
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
   -DCMAKE_INCLUDE_PATH=${unixodbc:location}/include
   -DCMAKE_LIBRARY_PATH=${unixodbc:location}/lib
-CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include ${:extra_cflags}
-CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${:extra_library_path}
-extra_cflags =
-extra_include_path =
-extra_library_path =
+  -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
+  -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
+CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include -I${zstd:location}/include
+CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
-  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${:extra_include_path}
+  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${zstd:location}/include
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
   LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib
   PATH=${patch:location}/bin:%(PATH)s
@@ -82,8 +79,8 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-8.00.tar.gz
-md5sum = 8b906705be5fc14785207ca6752b96e9
+url = https://packages.groonga.org/source/mroonga/mroonga-8.03.tar.gz
+md5sum = b394f3dcbff5668ced5d7a82dc648a7e
 pre-configure = set -e
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source

component/mariadb/mariadb_10.1.21_create_system_tables__no_test.patch

-# 33_scripts__mysql_create_system_tables__no_test.dpatch by  <ch@debian.org>
-
-A user with no password prevents a normal user from login under certain
-circumstances as it is checked first.
-See http://bugs.debian.org/301741
-and http://bugs.mysql.com/bug.php?id=6901
-
---- scripts/mysql_system_tables_data.sql.orig	2017-01-17 20:38:26.000000000 +0100
-+++ scripts/mysql_system_tables_data.sql	2017-01-18 17:03:23.795082252 +0100
-@@ -30,8 +30,6 @@
- -- Fill "db" table with default grants for anyone to
- -- access database 'test' and 'test_%' if "db" table didn't exist
- CREATE TEMPORARY TABLE tmp_db LIKE db;
--INSERT INTO tmp_db VALUES ('%','test','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');
--INSERT INTO tmp_db VALUES ('%','test\_%','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');
- INSERT INTO db SELECT * FROM tmp_db WHERE @had_db_table=0;
- DROP TABLE tmp_db;
- 
-@@ -48,9 +46,6 @@
- REPLACE INTO tmp_user_nopasswd VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0);
- -- More secure root account using unix sucket auth.
- INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),'','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'unix_socket','','N', 'N','', 0);
---- Anonymous user with no privileges.
--INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost','');
--INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost';
- 
- INSERT INTO user SELECT * FROM tmp_user_nopasswd WHERE @had_user_table=0 AND @skip_auth_root_nopasswd IS NULL;
- INSERT INTO user SELECT * FROM tmp_user_socket WHERE @had_user_table=0 AND @auth_root_socket IS NOT NULL;

component/mariadb/mariarocks.cfg

-# Do not extend any file that touch buildout:parts.
-
-[mariadb]
-version = 10.2.16
-md5sum = 234156a94f8ea6cff92c4751880b8316
-patches =
-  ${:_profile_base_location_}/mariadb_10.2.16_create_system_tables__no_test.patch#3fd5f9febabdb42d4b6653969a0194f9
-configure-options +=
-  -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-  -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
-extra_cflags = -I${zstd:location}/include
-extra_include_path = :${zstd:location}/include
-extra_library_path = :${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64

component/perl-DBD-MySQL/DBD-mysql-4.042.mariadb.patch

-diff -u DBD-mysql-4.042/dbdimp.c DBD-mysql-4.042/dbdimp.c
---- DBD-mysql-4.042/dbdimp.c	2017-02-28 14:36:40.000000000 +0100
-+++ DBD-mysql-4.042/dbdimp.c	2017-06-08 12:43:39.704228470 +0200
-@@ -2139,6 +2139,7 @@
- 
-     if (result)
-     {
-+      my_bool reconnect= 0;
- #if MYSQL_VERSION_ID >=SERVER_PREPARE_VERSION
-       /* connection succeeded. */
-       /* imp_dbh == NULL when mysql_dr_connect() is called from mysql.xs
-@@ -2155,7 +2156,7 @@
-         we turn off Mysql's auto reconnect and handle re-connecting ourselves
-         so that we can keep track of when this happens.
-       */
--      result->reconnect=0;
-+      mysql_options(result, MYSQL_OPT_RECONNECT, &reconnect);
-     }
-     else {
-       /* 
-diff -u DBD-mysql-4.042/dbdimp.h DBD-mysql-4.042/dbdimp.h
---- DBD-mysql-4.042/dbdimp.h	2017-02-28 14:36:40.000000000 +0100
-+++ DBD-mysql-4.042/dbdimp.h	2017-06-08 12:44:21.233948264 +0200
-@@ -67,6 +67,12 @@
- #define SSL_LAST_VERIFY_VERSION 50799
- #define MYSQL_VERSION_5_0 50001
- /* This is to avoid the ugly #ifdef mess in dbdimp.c */
-+
-+/* MariaDB Connector/C has MARIADB_VERSION_ID instead of MYSQL_VERSION_ID */
-+#if !defined(MYSQL_VERSION_ID) && defined(MARIADB_VERSION_ID)
-+#define MYSQL_VERSION_ID MARIADB_VERSION_ID
-+#endif
-+
- #if MYSQL_VERSION_ID < SQL_STATE_VERSION
- #define mysql_sqlstate(svsock) (NULL)
- #endif

component/perl-DBD-MySQL/buildout.cfg

@@ -13,11 +13,10 @@ recipe = slapos.recipe.cmmi
 depends =
   ${perl-DBI:location}
   ${perl-Devel-CheckLib:location}
-url = http://www.cpan.org/modules/by-module/DBD/DBD-mysql-4.043.tar.gz
-md5sum = 4a00dd7f1c057931147c65dfc4901c36
+url = https://github.com/perl5-dbi/DBD-mysql/archive/4.046_01.tar.gz
+md5sum = 90f87aec4b7ab95a9543f2dd10969a07
 patches =
   ${:_profile_base_location_}/DBD-mysql-4.027.rpathsupport.patch#a932982b7725e6621cfce3a3d7917e03
-  ${:_profile_base_location_}/DBD-mysql-4.042.mariadb.patch#5864d36d19c4a05034b3a4873f7c659a
 patch-options = -p1
 configure-command =
   ${perl:location}/bin/perl Makefile.PL --libs="-L${zlib:location}/lib -L${openssl:location}/lib $(mysql_config --libs)"

component/rdiff-backup/buildout.cfg

@@ -25,7 +25,7 @@ find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1
 [rdiff-backup-build-1.3.4]
 <= rdiff-backup-build
 # use our own version
-find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
+find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd4.tar.gz
 patches =
   ${:_profile_base_location_}/rdiff-backup-1.3.4-librsync-1.0.0.patch#31fafc8bc4a00f002f52008a9f3b671f
 

software/agent/software.cfg

@@ -51,7 +51,7 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 smmap = 0.9.0
 
 # Required by:

software/apache-frontend/software.cfg

@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3

software/backupserver/software.cfg

@@ -121,9 +121,9 @@ eggs =
   ${rdiff-backup-build-1.3.4:egg}
 
 [versions]
-# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
+# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd4+SlapOSPatched001'
 # but '1.3.4nxd2-SlapOSPatched001'.
-rdiff-backup = 1.3.4nxd2-SlapOSPatched001
+rdiff-backup = 1.3.4nxd4-SlapOSPatched001
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.4.2

software/caddy-frontend/README.caddy_frontend.rst

@@ -449,6 +449,16 @@ the proxy::
     }
   )
 
+QUIC Protocol
+=============
+
+Experimental QUIC available in Caddy is not configurable, thus it is required to open port ``udp:11443`` on the machine, like::
+
+  iptables -I INPUT -p udp --dport 11443 --destination ${ip} -j ACCEPT
+
+where ``${ip}`` is the IP of the partition with running caddy process.
+
+
 Notes
 =====
 

software/caddy-frontend/TODO.rst

@@ -63,7 +63,6 @@ Generally things to be done with ``caddy-frontend``:
  * reduce the time of configuration validation (in ``instance-apache-frontend.cfg.in`` sections ``[configtest]``, ``[caddy-configuration]``, ``[nginx-configuration]``), as it is not scalable on frontend with 2000+ slaves (takes few minutes instead of few, < 5, seconds), issue posted `upstream <https://github.com/mholt/caddy/issues/2220>`_
  * drop ``6tunnel`` and use ``bind`` in Caddy configuration, as soon as multiple binds will be possible, tracked in upstream `bind: support multiple values <https://github.com/mholt/caddy/pull/2128>`_ and `ipv6: does not bind on ipv4 and ipv6 for sites that resolve to both <https://github.com/mholt/caddy/issues/864>`_
  * use caddy-frontend in `standalone style playbooks <https://lab.nexedi.com/nexedi/slapos.package/tree/master/playbook/roles/standalone-shared>`_
- * ensure `QUIC <https://en.wikipedia.org/wiki/QUIC>`_ is used by caddy
 
 Things which can't be implemented:
 

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = d649e128d36cf76f870c189c53985569
+md5sum = 2c2d051825c76384732faad9c407c538
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 64fb8005a62f0a3a9987de2e336b68e1
+md5sum = 5cfc4dd69c2800906d6648c626023cf0
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in

software/caddy-frontend/instance-apache-frontend.cfg.in

+{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
 [buildout]
 extends =
   {{ parameter_dict['common_profile'] }}
@@ -224,6 +225,9 @@ command-line = {{ parameter_dict['caddy'] }}
   -conf ${dynamic-caddy-frontend-template:rendered}
   -log ${caddy-configuration:error-log}
   -http2=true
+{% if instance_parameter['configuration.enable-quic'].lower() in TRUE_VALUES %}
+  -quic
+{% endif %}
   -grace {{ instance_parameter['configuration.mpm-graceful-shutdown-timeout'] }}s
   -disable-http-challenge
   -disable-tls-sni-challenge

software/caddy-frontend/instance-caddy-input-schema.json

@@ -66,6 +66,16 @@
             "description": "Url to verify if the internet and/or re6stnet is working.",
             "title": "Test Verification URL",
             "type": "string"
+        },
+        "enable-quic": {
+            "default": "false",
+            "description": "Enables QUIC - Quick UDP Internet Connections. Note that this is experimental feature, thus can result in undefined behaviour.",
+            "enum": [
+                "true",
+                "false"
+            ],
+            "title": "Enable QUIC",
+            "type": "string"
         }
     },
     "title": "Input Parameters",

software/caddy-frontend/instance.cfg.in

@@ -88,6 +88,7 @@ configuration.trafficserver-autoconf-port = 8083
 configuration.trafficserver-mgmt-port = 8084
 configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
 configuration.enable-http2-by-default = true
+configuration.enable-quic = false
 configuration.mpm-graceful-shutdown-timeout = 5
 configuration.monitor-httpd-port = 8072
 configuration.frontend-name =
\ No newline at end of file

software/caddy-frontend/software.cfg

@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3

software/caddy-frontend/test/test.py

@@ -256,6 +256,9 @@ class TestDataMixin(object):
       'trafficserver/diags.log',
       'trafficserver/squid.blog',
       'trafficserver/.squid.blog.meta',
+      # not important, appears sometimes
+      'trafficserver/.error.log.meta',
+      'trafficserver/error.log',
     ])
 
   def test_file_list_run(self):
@@ -2830,3 +2833,110 @@ class TestDefaultMonitorHttpdPort(SlaveHttpFrontendTestCase, TestDataMixin):
       'Listen [%s]:8196' % (utils.GLOBAL_IPV6,) in master_monitor_conf)
     self.assertTrue(
       'Listen [%s]:8072' % (utils.GLOBAL_IPV6,) in slave_monitor_conf)
+
+
+class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'domain': 'example.com',
+      'nginx-domain': 'nginx.example.com',
+      'public-ipv4': utils.LOCAL_IPV4,
+      'enable-quic': 'true',
+      'apache-certificate': open('wildcard.example.com.crt').read(),
+      'apache-key': open('wildcard.example.com.key').read(),
+      '-frontend-authorized-slave-string':
+      '_apache_custom_http_s-accepted _caddy_custom_http_s-accepted',
+      'port': HTTPS_PORT,
+      'plain_http_port': HTTP_PORT,
+      'nginx_port': NGINX_HTTPS_PORT,
+      'plain_nginx_port': NGINX_HTTP_PORT,
+      'monitor-httpd-port': MONITOR_HTTPD_PORT,
+      '-frontend-config-1-monitor-httpd-port': MONITOR_F1_HTTPD_PORT,
+      'mpm-graceful-shutdown-timeout': 2,
+    }
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'url': {
+        'url': cls.backend_url,
+      },
+    }
+
+  def getMasterPartitionPath(self):
+    # partition w/o etc/trafficserver, but with buildout.cfg
+    return [
+      q for q in glob.glob(os.path.join(self.instance_path, '*',))
+      if not os.path.exists(os.path.join(q, 'etc', 'trafficserver')) and
+      os.path.exists(os.path.join(q, 'buildout.cfg'))][0]
+
+  def getSlavePartitionPath(self):
+    # partition w/ etc/trafficserver
+    return [
+      q for q in glob.glob(os.path.join(self.instance_path, '*',))
+      if os.path.exists(os.path.join(q, 'etc', 'trafficserver'))][0]
+
+  def test_url(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'url'].copy()
+    self.assertLogAccessUrlWithPop(parameter_dict, 'url')
+    self.assertEqual(
+      parameter_dict,
+      {
+        'domain': 'url.example.com',
+        'replication_number': '1',
+        'url': 'http://url.example.com',
+        'site_url': 'http://url.example.com',
+        'secure_access': 'https://url.example.com',
+        'public-ipv4': utils.LOCAL_IPV4,
+      }
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      utils.der2pem(result.peercert),
+      open('wildcard.example.com.crt').read())
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertFalse('remote_user' in j['Incoming Headers'].keys())
+
+    self.assertKeyWithPop('Date', result.headers)
+    self.assertKeyWithPop('Content-Length', result.headers)
+
+    self.assertEqual(
+      result.headers,
+      {'Content-Encoding': 'gzip',
+       'Alt-Svc': 'quic=":11443"; ma=2592000; v="39"',  # QUIC advertises
+       'Set-Cookie': 'secured=value;secure, nonsecured=value',
+       'Vary': 'Accept-Encoding',
+       'Server': 'Caddy, BaseHTTP/0.3 Python/2.7.14',
+       'Content-Type': 'application/json'}
+    )
+
+    result_http = self.fakeHTTPResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+    self.assertEqualResultJson(result_http, 'Path', '/test-path')
+
+    try:
+      j = result_http.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertFalse('remote_user' in j['Incoming Headers'].keys())
+
+    self.assertEqual(
+      result_http.headers['Content-Encoding'],
+      'gzip'
+    )
+
+    self.assertEqual(
+      result_http.headers['Set-Cookie'],
+      'secured=value;secure, nonsecured=value'
+    )

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_file_list_log-CADDY.txt

+TestQuicEnabled-0/var/log/monitor-httpd-error.log
+TestQuicEnabled-1/var/log/frontend-access.log
+TestQuicEnabled-1/var/log/frontend-error.log
+TestQuicEnabled-1/var/log/httpd/_url_access_log
+TestQuicEnabled-1/var/log/httpd/_url_error_log
+TestQuicEnabled-1/var/log/monitor-httpd-error.log
+TestQuicEnabled-1/var/log/nginx-access.log
+TestQuicEnabled-1/var/log/nginx-error.log
+TestQuicEnabled-1/var/log/trafficserver/manager.log
+TestQuicEnabled-1/var/log/trafficserver/traffic.out
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_file_list_run-CADDY.txt

+TestQuicEnabled-0/var/run/monitor-httpd.pid
+TestQuicEnabled-0/var/run/monitor/monitor-bootstrap.pid
+TestQuicEnabled-1/var/run/caddy_configuration.signature
+TestQuicEnabled-1/var/run/httpd.pid
+TestQuicEnabled-1/var/run/monitor-httpd.pid
+TestQuicEnabled-1/var/run/monitor/monitor-bootstrap.pid
+TestQuicEnabled-1/var/run/ncaddy_configuration.signature
+TestQuicEnabled-1/var/run/nginx.pid
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_monitor_promise_list-CADDY.txt

+TestQuicEnabled-1/etc/monitor-promise/check-_url-error-log-last-day
+TestQuicEnabled-1/etc/monitor-promise/check-_url-error-log-last-hour
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_promise_list-CADDY.txt

+TestQuicEnabled-0/etc/promise/check-free-disk-space
+TestQuicEnabled-0/etc/promise/monitor-http-frontend
+TestQuicEnabled-0/etc/promise/monitor-httpd-listening-on-tcp
+TestQuicEnabled-0/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
+TestQuicEnabled-1/etc/promise/caddy-frontend-is-running-actual-software-release
+TestQuicEnabled-1/etc/promise/caddy_cached
+TestQuicEnabled-1/etc/promise/caddy_frontend_ipv4_http
+TestQuicEnabled-1/etc/promise/caddy_frontend_ipv4_https
+TestQuicEnabled-1/etc/promise/caddy_frontend_ipv6_http
+TestQuicEnabled-1/etc/promise/caddy_frontend_ipv6_https
+TestQuicEnabled-1/etc/promise/caddy_ssl_cached
+TestQuicEnabled-1/etc/promise/check-free-disk-space
+TestQuicEnabled-1/etc/promise/frontend-caddy-configuration-promise
+TestQuicEnabled-1/etc/promise/monitor-http-frontend
+TestQuicEnabled-1/etc/promise/monitor-httpd-listening-on-tcp
+TestQuicEnabled-1/etc/promise/nginx-configuration-promise
+TestQuicEnabled-1/etc/promise/nginx_frontend_ipv4_http
+TestQuicEnabled-1/etc/promise/nginx_frontend_ipv4_https
+TestQuicEnabled-1/etc/promise/nginx_frontend_ipv6_http
+TestQuicEnabled-1/etc/promise/nginx_frontend_ipv6_https
+TestQuicEnabled-1/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
+TestQuicEnabled-1/etc/promise/promise-nginx-is-process-older-than-dependency-set
+TestQuicEnabled-1/etc/promise/re6st-connectivity
+TestQuicEnabled-1/etc/promise/trafficserver-cache-availability
+TestQuicEnabled-1/etc/promise/trafficserver-port-listening
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_supervisor_state-CADDY.txt

+TestQuicEnabled-0:bootstrap-monitor EXITED
+TestQuicEnabled-0:certificate_authority-on-watch RUNNING
+TestQuicEnabled-0:crond RUNNING
+TestQuicEnabled-0:monitor-httpd-graceful EXITED
+TestQuicEnabled-0:monitor-httpd-on-watch RUNNING
+TestQuicEnabled-1:6tunnel-11080-on-watch RUNNING
+TestQuicEnabled-1:6tunnel-11443-on-watch RUNNING
+TestQuicEnabled-1:6tunnel-12080-on-watch RUNNING
+TestQuicEnabled-1:6tunnel-12443-on-watch RUNNING
+TestQuicEnabled-1:6tunnel-26011-on-watch RUNNING
+TestQuicEnabled-1:6tunnel-26012-on-watch RUNNING
+TestQuicEnabled-1:bootstrap-monitor EXITED
+TestQuicEnabled-1:certificate_authority-on-watch RUNNING
+TestQuicEnabled-1:crond-on-watch RUNNING
+TestQuicEnabled-1:frontend-caddy-safe-graceful EXITED
+TestQuicEnabled-1:frontend-nginx-safe-graceful EXITED
+TestQuicEnabled-1:frontend_caddy-on-watch RUNNING
+TestQuicEnabled-1:frontend_nginx-on-watch RUNNING
+TestQuicEnabled-1:monitor-httpd-graceful EXITED
+TestQuicEnabled-1:monitor-httpd-on-watch RUNNING
+TestQuicEnabled-1:trafficserver-on-watch RUNNING
+TestQuicEnabled-1:trafficserver-reload EXITED
+watchdog:watchdog RUNNING
\ No newline at end of file

software/erp5/software-erp5-component.cfg

-# This file is for building ERP5 software release using erp5-component
-# branch.
-
-[buildout]
-extends =
-  software.cfg
-  ../../component/mariadb/mariarocks.cfg
-
-[slapos.cookbook-repository]
-branch = erp5-component
-
-[erp5]
-branch = erp5-component
-
-[cloudooo-repository]
-branch = master
-revision =
-
-[versions]
-cloudooo =

software/jstestnode/runTestSuite.in

@@ -120,10 +120,15 @@ def main():
       appium_url = "http://%s@ondemand.saucelabs.com/wd/hub" % (args.appium_server_auth)
       browser = webdriver.Remote(appium_url, capabilities)
 
-      # adjust make path to testnode's frontend
-      full_path = '$${runTestSuite-instance:buildout-directory}/software_release/parts/%s' % parsed_parameters['test-url']
-      full_path = full_path.split('srv')[-1]
-      url = "%s%s" % (args.frontend_url, full_path)
+      # adjust path for remote test url
+      remote_access_url = parsed_parameters.get('remote-access-url', None)
+      if remote_access_url:
+        if ('jio' in test_suite):
+          url = os.path.join(remote_access_url, 'jio/test/tests.html')
+        else:
+          url = os.path.join(remote_access_url, 'renderjs/test/')
+      else:
+        raise ValueError('remote-access-url is not defined in instance parameter')
 
     is_browser_running = True
     agent = browser.execute_script("return navigator.userAgent")
@@ -131,7 +136,7 @@ def main():
     print url
 
     browser.get(url)
-    WebDriverWait(browser, 60).until(EC.presence_of_element_located((
+    WebDriverWait(browser, 300).until(EC.presence_of_element_located((
       By.XPATH, '//p[@id="qunit-testresult" and contains(text(), "completed")]')
     ))
 
@@ -141,10 +146,15 @@ def main():
     print ' '.join(body.xpath('//*[@id="qunit-testresult"]//text()'))
 
     for elt in body.xpath('.//ol[@id="qunit-tests"]/li'):
+      if (len(elt.xpath('.//span[@class="module-name"]'))):
         test_name = '%s: %s' % (
           elt.xpath('.//span[@class="module-name"]')[0].text,
           elt.xpath('.//span[@class="test-name"]')[0].text
         )
+      #global failure, like Uncaught ReferenceError: RSVP is not defined
+      else:
+        test_name = elt.xpath('.//span[@class="test-name"]')[0].text
+         
       print elt.get('class'), ''.join(elt.xpath('.//strong')[0].itertext())
       # print elt.find_element_by_tag_name('ol').get_attribute('innerHTML')
 

software/jstestnode/software.cfg

@@ -107,7 +107,7 @@ mode = 0644
 [template-runTestSuite]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/runTestSuite.in
-md5sum = ff66d13f73982e8257eb5535cdb541c7
+md5sum = 8528d90cce5d034195e3e71bef855c48
 output = ${buildout:directory}/runTestSuite.in
 mode = 0644
 

software/kvm/software.cfg

@@ -255,7 +255,7 @@ context =
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 erp5.util = 0.4.51
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0

software/neoppod/software-common.cfg

@@ -134,7 +134,7 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 setproctitle = 1.1.10
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0

software/neoppod/software-myrocks-dev.cfg

-[buildout]
-extends =
-    software.cfg
-    ../../component/mariadb/mariarocks.cfg

software/nginx-push-stream/software.cfg

@@ -46,7 +46,7 @@ mode = 0644
 
 [versions]
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 dnspython = 1.15.0
 PyRSS2Gen = 1.1
 erp5.util = 0.4.51

software/re6stnet/software.cfg

@@ -118,7 +118,7 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 smmap = 0.9.0
 dnspython = 1.15.0
 erp5.util = 0.4.51

software/slapos-testing/software.cfg

@@ -175,7 +175,7 @@ Pygments = 2.1.3
 slapos.recipe.build = 0.36
 slapos.recipe.cmmi = 0.7
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 
 # All depencies should be pinned.
 apache-libcloud = 2.3.0

software/slaprunner/software.cfg

@@ -171,7 +171,7 @@ prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 4.3
 collective.recipe.environment = 0.2.0
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 smmap = 0.9.0
 lockfile = 0.12.2
 

software/wendelin-scalability/test-common.cfg

 [buildout]
 extends =
   ../wendelin/software.cfg
-  ../../component/mariadb/mariarocks.cfg
 
 [local-bt5-repository]
 list += ${slapos.cookbook-repository:location}/software/wendelin-scalability

stack/caucase/buildout.cfg

@@ -95,7 +95,7 @@ futures = 3.1.1
 gitdb2 = 2.0.2
 gunicorn = 19.7.1
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 smmap2 = 2.0.3
 
 # Required by:

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 488b9410da5f23f9af177f81d847c930
+md5sum = 93f4f4c27d9302977752240acefd4530
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -86,7 +86,7 @@ md5sum = a4dda17c1671ead45ac92b7decc32228
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 55fce4e96d844cc418c586d8d9a67c29
+md5sum = 89872075cd2f31d824369447aaf0532f
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -285,10 +285,19 @@ crl = ${:ca-dir}/crl
 apachedex = ${monitor-directory:private}/apachedex
 
 [{{ section('monitor-generate-apachedex-report') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = generate-apachedex-report
+# The goal is to be executed before logrotate log rotation.
+# Here, logrotate-entry-base:frequency = daily, so we run at 23 o'clock every day.
+frequency = 0 23 * * *
+command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}
+
+[monitor-generate-apachedex-report-wrapper]
 recipe = slapos.cookbook:wrapper
-wrapper-path = ${monitor-directory:reports}/${:command}
+wrapper-path = ${directory:bin}/${:command}
 command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
-command = apachedex_every_23_hour
+command = generate-apachedex-report
 
 [apachedex-parameters]
 # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz

stack/erp5/instance-mariadb.cfg.in

@@ -308,10 +308,19 @@ context =
   key update_executable update-mysql:output
 
 [{{ section('monitor-generate-mariadb-slow-query-report') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = generate-mariadb-slow-query-report
+# The goal is to be executed before logrotate log rotation.
+# Here, logrotate-entry-base:frequency = daily, so we run at 23 o'clock every day.
+frequency = 0 23 * * *
+command = ${monitor-generate-mariadb-slow-query-report-wrapper:rendered}
+
+[monitor-generate-mariadb-slow-query-report-wrapper]
 recipe = slapos.recipe.template:jinja2
 template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
-rendered = ${monitor-directory:reports}/${:filename}
-filename = mariadb_slow_query_every_23_hour
+rendered = ${directory:bin}/${:filename}
+filename = generate-mariadb-slow-query-report
 mode = 755
 context = 
   raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log

stack/monitor/buildout.cfg

@@ -121,6 +121,6 @@ depends =
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
-slapos.toolbox = 0.77
+slapos.toolbox = 0.78
 pyasn1 = 0.3.7
 

stack/resilient/buildout.cfg

@@ -125,13 +125,11 @@ mode = 0644
 # Time in minutes to wait for the backup to finish
 backup_wait_time = 10
 
-[rdiff-backup-build]
-# use our own version
-find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
-patches =
-  ${:_profile_base_location_}/rdiff-backup-1.3.4-librsync-1.0.0.patch#31fafc8bc4a00f002f52008a9f3b671f
+[rdiff-backup]
+eggs =
+  ${rdiff-backup-build-1.3.4:egg}
 
 [versions]
-# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
-# but '1.3.4nxd2-SlapOSPatched001'.
-rdiff-backup = 1.3.4nxd2-SlapOSPatched001
+# 1.3.4nxd4 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
+# but '1.3.4nxd4-SlapOSPatched001'.
+rdiff-backup = 1.3.4nxd4-SlapOSPatched001