Merge branch 'master' of https://lab.nexedi.com/nexedi/slapos

component/haproxy/buildout.cfg

@@ -12,8 +12,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.9.tar.gz
-md5sum = a2bbbdd45ffe18d99cdcf26aa992f92d
+url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.1.tar.gz
+md5sum = e42892d4b6ee33200fccaa1d81837e49
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/jemalloc/buildout.cfg

@@ -5,7 +5,8 @@ parts =
 
 [jemalloc]
 recipe = slapos.recipe.cmmi
-url = http://www.canonware.com/download/jemalloc/jemalloc-3.6.0.tar.bz2
+url = https://github.com/jemalloc/jemalloc/releases/download/${:version}/jemalloc-${:version}.tar.bz2
+version = 3.6.0
 md5sum = e76665b63a8fddf4c9f26d2fa67afdf2
 configure-options =
   --disable-static

component/mariadb/buildout.cfg

@@ -12,7 +12,6 @@ extends =
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
   ../patch/buildout.cfg
-  ../pcre/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../readline/buildout.cfg
   ../xz-utils/buildout.cfg
@@ -27,8 +26,9 @@ parts =
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-url = https://downloads.mariadb.org/f/mariadb-10.1.28/source/mariadb-10.1.28.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-md5sum = 38acd5b44c56791701d80fddf088ef38
+url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
+version = 10.1.29
+md5sum = 6d2cc6b0f8c5c7525e08aaddad8dca96
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
@@ -45,7 +45,7 @@ configure-options =
   -DWITH_ZLIB=system
   -DWITH_READLINE=0
   -DWITH_PIC=1
-  -DWITH_PCRE=system
+  -DWITH_PCRE=bundled
   -DENABLE_DTRACE=0
   -DWITH_EXTRA_CHARSETS=complex
   -DWITH_EMBEDDED_SERVER=0
@@ -56,22 +56,22 @@ configure-options =
   -DWITH_INNODB_LZO=OFF
   -DWITH_INNODB_SNAPPY=OFF
   -DWITH_SAFEMALLOC=OFF
-  -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-  -DWITHOUT_MROONGA_STORAGE_ENGINE=1
-  -DWITHOUT_DAEMON_EXAMPLE=1
+  -DPLUGIN_DAEMON_EXAMPLE=NO
+  -DPLUGIN_EXAMPLE=NO
+  -DPLUGIN_MROONGA=NO
   -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
-CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
-CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
+CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
+CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
 extra_cflags =
 extra_include_path =
 extra_library_path =
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
-  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
+  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
-  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
   PATH=${patch:location}/bin:%(PATH)s
 post-install =
   mkdir -p ${:location}/include/wsrep &&
@@ -108,6 +108,6 @@ pre-build =
   sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
 environment =
   PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
+  CPPFLAGS=-I${groonga:location}/include/groonga
   LDFLAGS=-L${groonga:location}/lib
   PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig

component/mariadb/mariarocks.cfg

 # Do not extend any file that touch buildout:parts.
 
 [mariadb]
-url = https://downloads.mariadb.org/f/mariadb-10.2.9/source/mariadb-10.2.9.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-md5sum = c59999bd182ddeb3db3d55250aecd8f8
+version = 10.2.11
+md5sum = 954088299fe5f11b4fda3b540558adbd
 stable-patches =
 configure-options +=
-  -DPLUGIN_DAEMON_EXAMPLE=NO
-  -DPLUGIN_EXAMPLE=NO
-  -DPLUGIN_MROONGA=NO
+# force build of TokuDB due to a regression in 10.2.11
+  -DTOKUDB_OK=1
   -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
   -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
 extra_cflags = -I${zstd:location}/include

component/openssl/buildout.cfg

@@ -16,8 +16,8 @@ parts =
 
 [openssl]
 recipe = slapos.recipe.cmmi
-url = https://www.openssl.org/source/openssl-1.0.2m.tar.gz
-md5sum = 10e9e37f492094b9ef296f68f24a7666
+url = https://www.openssl.org/source/openssl-1.0.2n.tar.gz
+md5sum = 13bdc1b1d1ff39b6fd42a255e74676a4
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs

software/agent/software.cfg

@@ -50,7 +50,7 @@ gitdb = 0.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap = 0.9.0
 

software/apache-frontend/software.cfg

@@ -10,7 +10,7 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap = 0.9.0
 numpy = 1.11.2

software/backupserver/software.cfg

@@ -88,7 +88,7 @@ mode = 0644
 [status2rss]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/status2rss.py
-md5sum = 0cd1cf97b199dd18fc0168c7281890ea
+md5sum = a023694817975e73998fb9187a6015d6
 output = ${buildout:directory}/status2rss.py
 mode = 0644
 

software/backupserver/status2rss.py

@@ -29,9 +29,8 @@ while 1:
     title = desc,
     description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc,
       open(statistic).read())).split("\n")),
-    link = LINK,
     pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
-    guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
+    guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)), isPermaLink=0)
     )
   items.append(rss_item)
 

software/buildout-testing/runTestSuite.in

@@ -77,7 +77,7 @@ def main():
 
   test_title = args.test_suite_title or args.test_suite
   if args.master_url:
-    tool = taskdistribution.TaskDistributionTool(args.master_url)
+    tool = taskdistribution.TaskDistributor(args.master_url)
     test_result = tool.createTestResult(args.revision,
                                         list(test_dict),
                                         args.test_node_title,

software/cdn-me/software.cfg

@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 erp5.util = 0.4.49
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/dream/software.cfg

@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
 # indirect dependancies
 cp.recipe.cmd = 0.5
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 zope.exceptions = 4.0.7
 zope.testing = 4.1.3
 zc.recipe.testrunner = 2.0.0

software/erp5testnode/software.cfg

@@ -58,8 +58,8 @@ mode = 0644
 
 [versions]
 PyXML = 0.8.5
-erp5.util = 0.4.49
-slapos.recipe.template = 4.1
+erp5.util = 0.4.50
+slapos.recipe.template = 4.2
 ipython = 5.3.0
 apache-libcloud = 2.1.0
 gitdb2 = 2.0.2

software/erp5testnode/testsuite/deploy-test/runTestSuite.py

@@ -158,10 +158,10 @@ def main():
     access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
     os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
     os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
-  tool = taskdistribution.TaskDistributionTool(
+  distributor = taskdistribution.TaskDistributor(
       args.master_url,
       logger=logger)
-  test_result = tool.createTestResult(
+  test_result = distributor.createTestResult(
       revision, suite.getTestList(), args.test_node_title,
       suite.allow_restart, test_suite_title, args.project_title)
   if test_result is None:

software/erp5testnode/testsuite/deploy-test/software.cfg

@@ -71,4 +71,4 @@ output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [versions]
-erp5.util = 0.4.49
+erp5.util = 0.4.50

software/erp5testnode/testsuite/dummy/software.cfg

@@ -5,6 +5,7 @@ extends =
 
 parts =
   instance-template
+  slapos-cookbook
 
 [instance-template]
 recipe = slapos.recipe.template
@@ -14,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/gitlab/software.cfg

@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
 rubygemsrecipe  = 0.2.2+slapos001
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 z3c.recipe.scripts = 1.0.1

software/html5as/software.cfg

@@ -72,7 +72,7 @@ async = 0.6.1
 gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.40.4
 smmap = 0.8.2
 plone.recipe.command = 1.1

software/html5ide/software.cfg

@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/jstestnode/runTestSuite.in

@@ -91,8 +91,8 @@ def main():
     if args.target == 'firefox':
       firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX
       firefox_capabilities['marionette'] = True
-      firefox_capabilities['binary'] = '${firefox:location}/firefox-slapos'
       browser = webdriver.Firefox(capabilities=firefox_capabilities,
+                                  firefox_binary='${firefox:location}/firefox-slapos',
                                   executable_path='${firefox:location}/geckodriver')
     elif args.target in ['iOS', 'Android']:
       # parameters for mobile emulators have different names then parameters for
@@ -169,7 +169,7 @@ def main():
     browser.quit()
     is_browser_running = False
 
-    tool = taskdistribution.TaskDistributionTool(portal_url=args.master_url)
+    tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
     test_result = tool.createTestResult(revision = revision,
                                         test_name_list = test_line_dict.keys(),
                                         node_title = args.test_node_title,

software/jstestnode/software.cfg

@@ -107,11 +107,11 @@ mode = 0644
 [template-runTestSuite]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/runTestSuite.in
-md5sum = fcf15b2a90340e0afe8f8b9921a4ffae
+md5sum = ff66d13f73982e8257eb5535cdb541c7
 output = ${buildout:directory}/runTestSuite.in
 mode = 0644
 
 [versions]
-erp5.util = 0.4.49
-slapos.recipe.template = 4.1
-selenium = 2.53.1
+erp5.util = 0.4.50
+slapos.recipe.template = 4.2
+selenium = 3.8.0

software/jupyter/software.cfg

@@ -85,7 +85,7 @@ pyzmq = 16.0.2
 scikit-learn = 0.18.1
 seaborn = 0.7.1
 simplegeneric = 0.8.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 statsmodels = 0.8.0
 terminado = 0.6
 tornado = 4.4.2

software/kvm/software.cfg

@@ -11,7 +11,7 @@ apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4
 pycurl = 7.43.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 smmap = 0.9.0
 # websockify = 0.8.0
 

software/nayuos/software.cfg

@@ -17,7 +17,7 @@ parts +=
 versions = versions
 
 [versions]
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 
 
 [template-instance]

software/neoppod/software-common.cfg

@@ -116,7 +116,7 @@ mysqlclient = 1.3.12
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap2 = 2.0.1
 transaction = 1.7.0

software/nginx-push-stream/software.cfg

@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73

software/powerdns/software.cfg

@@ -65,4 +65,4 @@ mode = 0644
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/re6stnet/software.cfg

@@ -110,7 +110,7 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap = 0.9.0
 

software/seleniumrunner/software.cfg

@@ -47,4 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/slapos-in-partition/software.cfg

@@ -11,4 +11,4 @@ extends = common.cfg
 Pygments = 1.6
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.10
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/slapos-testing/software.cfg

@@ -97,4 +97,4 @@ mode = 640
 Pygments = 2.1.3
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/slaprunner/software.cfg

@@ -15,7 +15,7 @@ gitdb = 0.6.4
 gunicorn = 19.7.1
 prettytable = 0.7.2
 pycurl = 7.43.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap = 0.9.0
 

software/test-slave-instance-deployment/software.cfg

@@ -59,7 +59,7 @@ eggs = collective.recipe.template
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
 slapos.recipe.build = 0.28
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
 [buildout]

software/varnish/software.cfg

@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
 mode = 640
 
 [versions]
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/wendelin-scalability/test-common.cfg

+[buildout]
+extends =
+  ../wendelin/software.cfg
+  ../../component/mariadb/mariarocks.cfg
+
+[local-bt5-repository]
+list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
+
+[patch-template]
+recipe = slapos.recipe.build
+location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
+script =
+  with open(self.options['location'], 'w') as dst, \
+       open(self.options['base']) as src:
+    src = src.read()
+    i = src.index('[buildout]')
+    dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])

software/wendelin-scalability/test-fluentd-1pb.cfg

@@ -33,7 +33,7 @@ tags =
 
 [feeder]
 feeder =
-  #
+  # Same algorithm as ERP5Site_simulateFluentdIngestion
     import os, struct
     from random import lognormvariate
     pack = struct.Struct('!d').pack

software/wendelin-scalability/test-fluentd-common.cfg

 [buildout]
 extends =
-  ../../component/gnupg/buildout.cfg
   ../fluentd/software.cfg
-  ../wendelin/software.cfg
-  ../../component/mariadb/mariarocks.cfg
-
-[local-bt5-repository]
-list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
-
-[patch-template]
-recipe = slapos.recipe.build
-location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
-script =
-  with open(self.options['location'], 'w') as dst, \
-       open(self.options['base']) as src:
-    src = src.read()
-    i = src.index('[buildout]')
-    dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
+  test-common.cfg
 
 [template-erp5]
 recipe =

software/wendelin-scalability/test-much-deduplication-high-compressibility.cfg

 # NEO: data deduplication must be enabled
 
 [buildout]
-extends = test-fluentd-common.cfg
+extends =
+  ../../component/gnupg/buildout.cfg
+  test-fluentd-common.cfg
 
 [template-erp5-patched]
 extra =

software/wendelin-scalability/test-neo.cfg

+# The 'start_ingest' command causes a zope to fill NEO as fast as possible,
+# as if fluentd pushed data.
+#
+# Use sigma > 0 to have oids of variable size inside NEO.
+# To know average compression ratio:
+#   x=test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py
+#   $x 10 1
+#   0.434851958247
+#   2155 - 65536 (99th percentile)
+#   $x 8.787 0
+#   0.100036621094
+#   6556
+
+[buildout]
+extends = test-common.cfg
+parts += start_ingest
+
+[start_ingest]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode = 0755
+template =
+  inline:#!${buildout:executable}
+  import argparse, base64, httplib, sys
+  parser = argparse.ArgumentParser()
+  _ = parser.add_argument
+  _('--site-id', default='erp5')
+  _('hostport', metavar='host[:port]', help='Zope address')
+  _('password', help="'zope' user password")
+  _('reference', help='Data Stream reference')
+  _('mu', type=float)
+  _('sigma', type=float)
+  _('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
+  args = parser.parse_args()
+  qs = []
+  for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
+    v = getattr(args, k)
+    if v is not None:
+      t = type(v)
+      qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
+  c = httplib.HTTPConnection(args.hostport)
+  c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
+                      % (args.site_id, '&'.join(qs)))
+  c.putheader('Authorization',
+    'Basic ' + base64.b64encode('zope:'+args.password))
+  c.endheaders()

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py

+#!/usr/bin/python
+from __future__ import division, print_function
+import os, struct
+from random import lognormvariate
+
+bigfile_chunk_size = 65536
+
+def simulateFluentdIngestion(self, reference, mu, sigma,
+                             chunks_per_transaction=128):
+  from time import time
+  import transaction
+  note = (self['portal_ingestion_policies']['scalability_test_unpack'].getPath()
+          + '/ingest')
+  module = self['data_stream_module']
+  try:
+    data_stream = module[reference]
+  except KeyError:
+    data_stream = module.newContent(reference, 'Data Stream')
+    transaction.commit()
+
+  pack = struct.Struct('!d').pack
+  data = os.urandom(bigfile_chunk_size - 8)
+
+  while 1:
+    txn = transaction.begin()
+    data_stream.appendData(''.join(
+      (pack(time()) + data[:int(lognormvariate(mu, sigma))]
+      ).ljust(bigfile_chunk_size, '\0')
+      for _ in xrange(chunks_per_transaction)))
+    txn.note(note)
+    txn.commit()
+
+if __name__ == '__main__':
+  import sys
+  mu, sigma = map(float, sys.argv[1:3])
+  if sigma:
+    try:
+      n = int(sys.argv[3])
+    except IndexError:
+      n = 1000000
+  else:
+    n = 1
+  x = sorted(min(int(lognormvariate(mu, sigma)), bigfile_chunk_size - 8)
+             for _ in xrange(n))
+  print((8 * n + sum(x)) / (bigfile_chunk_size * n))
+  if n == 1:
+    print(x[0] + 8)
+  else:
+    n //= 100
+    if n:
+      print(8 + x[n], '-', 8 + x[-n-1], '(99th percentile)')
+    else:
+      print(8 + x[0], '-', 8 + x[-1])

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Extension Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>ScalabilityFluentd</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>extension.erp5.ScalabilityFluentd</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Extension Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple>
+                <string>W:  8, 46: Redefining name \'mu\' from outer scope (line 35) (redefined-outer-name)</string>
+                <string>W:  8, 50: Redefining name \'sigma\' from outer scope (line 35) (redefined-outer-name)</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_simulateFluentdIngestion.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>simulateFluentdIngestion</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>ScalabilityFluentd</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_simulateFluentdIngestion</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/bt/template_extension_id_list

+extension.erp5.ScalabilityFluentd
\ No newline at end of file

software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list

 portal_ingestion_policies/scalability_test_*
 portal_skins/custom/DataStreamModule_getTotalSize
-portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
\ No newline at end of file
+portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
+portal_skins/custom/ERP5Site_simulateFluentdIngestion
\ No newline at end of file

stack/caucase/buildout.cfg

@@ -94,7 +94,7 @@ caucase = 0.1.4
 futures = 3.1.1
 gitdb2 = 2.0.2
 gunicorn = 19.7.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap2 = 2.0.3
 

stack/cloudooo.cfg

@@ -89,7 +89,7 @@ PasteScript = 2.0.2
 WSGIUtils = 0.7
 python-magic = 0.4.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 
 # Required by:
 # PasteScript==2.0

stack/erp5/buildout.cfg

@@ -736,7 +736,7 @@ uuid = 1.30
 validictory = 1.1.0
 xfw = 0.10
 xupdate-processor = 0.4
-selenium = 2.53.1
+selenium = 3.8.0
 
 # Required by:
 # Products.CMFCore==2.2.10

stack/erp5/buildout.hash.cfg

@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 8ea5033142f450a2e90431817771cb44
+md5sum = 244e185ed26831fe0b5abf49c67753d6
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in

stack/erp5/instance-mariadb.cfg.in

@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }}
 #   can be fully restored.
 # master-data: use value "2" as we are not in a replication case
 #}
-command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
+command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
 {# KEEP GLOB PATTERN IN SYNC with generated filenames above
 #           YYYYmmddHHMMSS -#}
 file-glob = ??????????????.sql.gz

stack/monitor/buildout.cfg

@@ -132,4 +132,5 @@ PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
 slapos.toolbox = 0.73
+pyasn1 = 0.3.7
 

stack/slapos.cfg

@@ -91,6 +91,7 @@ eggs =
   slapos.libnetworkcache
 
 [versions]
+setuptools = 33.1.1
 # Use SlapOS patched zc.buildout
 zc.buildout = 2.5.2+slapos011
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
@@ -125,7 +126,6 @@ pyOpenSSL = 17.2.0
 pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
-setuptools = 33.1.1
 six = 1.10.0
 slapos.cookbook = 1.0.53
 slapos.core = 1.4.3