Commit a5582a6a authored by Aurélien Vermylen's avatar Aurélien Vermylen

Merge branch 'master' of https://lab.nexedi.com/nexedi/slapos

parents 77f0b402 6db1245b
...@@ -12,8 +12,8 @@ parts = haproxy ...@@ -12,8 +12,8 @@ parts = haproxy
[haproxy] [haproxy]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.9.tar.gz url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.1.tar.gz
md5sum = a2bbbdd45ffe18d99cdcf26aa992f92d md5sum = e42892d4b6ee33200fccaa1d81837e49
configure-command = true configure-command = true
# If the system is running on Linux 2.6, we use "linux26" as the TARGET, # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
# otherwise use "generic". # otherwise use "generic".
......
...@@ -5,7 +5,8 @@ parts = ...@@ -5,7 +5,8 @@ parts =
[jemalloc] [jemalloc]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.canonware.com/download/jemalloc/jemalloc-3.6.0.tar.bz2 url = https://github.com/jemalloc/jemalloc/releases/download/${:version}/jemalloc-${:version}.tar.bz2
version = 3.6.0
md5sum = e76665b63a8fddf4c9f26d2fa67afdf2 md5sum = e76665b63a8fddf4c9f26d2fa67afdf2
configure-options = configure-options =
--disable-static --disable-static
......
...@@ -12,7 +12,6 @@ extends = ...@@ -12,7 +12,6 @@ extends =
../ncurses/buildout.cfg ../ncurses/buildout.cfg
../openssl/buildout.cfg ../openssl/buildout.cfg
../patch/buildout.cfg ../patch/buildout.cfg
../pcre/buildout.cfg
../pkgconfig/buildout.cfg ../pkgconfig/buildout.cfg
../readline/buildout.cfg ../readline/buildout.cfg
../xz-utils/buildout.cfg ../xz-utils/buildout.cfg
...@@ -27,8 +26,9 @@ parts = ...@@ -27,8 +26,9 @@ parts =
[mariadb] [mariadb]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-10.1.28/source/mariadb-10.1.28.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = 38acd5b44c56791701d80fddf088ef38 version = 10.1.29
md5sum = 6d2cc6b0f8c5c7525e08aaddad8dca96
patch-options = -p0 patch-options = -p0
patches = patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3 ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
...@@ -45,7 +45,7 @@ configure-options = ...@@ -45,7 +45,7 @@ configure-options =
-DWITH_ZLIB=system -DWITH_ZLIB=system
-DWITH_READLINE=0 -DWITH_READLINE=0
-DWITH_PIC=1 -DWITH_PIC=1
-DWITH_PCRE=system -DWITH_PCRE=bundled
-DENABLE_DTRACE=0 -DENABLE_DTRACE=0
-DWITH_EXTRA_CHARSETS=complex -DWITH_EXTRA_CHARSETS=complex
-DWITH_EMBEDDED_SERVER=0 -DWITH_EMBEDDED_SERVER=0
...@@ -56,22 +56,22 @@ configure-options = ...@@ -56,22 +56,22 @@ configure-options =
-DWITH_INNODB_LZO=OFF -DWITH_INNODB_LZO=OFF
-DWITH_INNODB_SNAPPY=OFF -DWITH_INNODB_SNAPPY=OFF
-DWITH_SAFEMALLOC=OFF -DWITH_SAFEMALLOC=OFF
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1 -DPLUGIN_DAEMON_EXAMPLE=NO
-DWITHOUT_MROONGA_STORAGE_ENGINE=1 -DPLUGIN_EXAMPLE=NO
-DWITHOUT_DAEMON_EXAMPLE=1 -DPLUGIN_MROONGA=NO
-DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}" -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}" -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH} -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags} CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path} CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
extra_cflags = extra_cflags =
extra_include_path = extra_include_path =
extra_library_path = extra_library_path =
environment = environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin CMAKE_PROGRAM_PATH=${cmake:location}/bin
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path} CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH} CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
PATH=${patch:location}/bin:%(PATH)s PATH=${patch:location}/bin:%(PATH)s
post-install = post-install =
mkdir -p ${:location}/include/wsrep && mkdir -p ${:location}/include/wsrep &&
...@@ -108,6 +108,6 @@ pre-build = ...@@ -108,6 +108,6 @@ pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment = environment =
PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include CPPFLAGS=-I${groonga:location}/include/groonga
LDFLAGS=-L${groonga:location}/lib LDFLAGS=-L${groonga:location}/lib
PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig
# Do not extend any file that touch buildout:parts. # Do not extend any file that touch buildout:parts.
[mariadb] [mariadb]
url = https://downloads.mariadb.org/f/mariadb-10.2.9/source/mariadb-10.2.9.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve version = 10.2.11
md5sum = c59999bd182ddeb3db3d55250aecd8f8 md5sum = 954088299fe5f11b4fda3b540558adbd
stable-patches = stable-patches =
configure-options += configure-options +=
-DPLUGIN_DAEMON_EXAMPLE=NO # force build of TokuDB due to a regression in 10.2.11
-DPLUGIN_EXAMPLE=NO -DTOKUDB_OK=1
-DPLUGIN_MROONGA=NO
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++ -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
extra_cflags = -I${zstd:location}/include extra_cflags = -I${zstd:location}/include
......
...@@ -16,8 +16,8 @@ parts = ...@@ -16,8 +16,8 @@ parts =
[openssl] [openssl]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://www.openssl.org/source/openssl-1.0.2m.tar.gz url = https://www.openssl.org/source/openssl-1.0.2n.tar.gz
md5sum = 10e9e37f492094b9ef296f68f24a7666 md5sum = 13bdc1b1d1ff39b6fd42a255e74676a4
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
# 'prefix' option to override --openssldir/--prefix (which is useful # 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
......
...@@ -50,7 +50,7 @@ gitdb = 0.6.4 ...@@ -50,7 +50,7 @@ gitdb = 0.6.4
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.download = 1.0 slapos.recipe.download = 1.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
......
...@@ -10,7 +10,7 @@ gitdb = 0.6.4 ...@@ -10,7 +10,7 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
numpy = 1.11.2 numpy = 1.11.2
......
...@@ -88,7 +88,7 @@ mode = 0644 ...@@ -88,7 +88,7 @@ mode = 0644
[status2rss] [status2rss]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/status2rss.py url = ${:_profile_base_location_}/status2rss.py
md5sum = 0cd1cf97b199dd18fc0168c7281890ea md5sum = a023694817975e73998fb9187a6015d6
output = ${buildout:directory}/status2rss.py output = ${buildout:directory}/status2rss.py
mode = 0644 mode = 0644
......
...@@ -29,9 +29,8 @@ while 1: ...@@ -29,9 +29,8 @@ while 1:
title = desc, title = desc,
description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc, description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc,
open(statistic).read())).split("\n")), open(statistic).read())).split("\n")),
link = LINK,
pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))), pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc))) guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)), isPermaLink=0)
) )
items.append(rss_item) items.append(rss_item)
......
...@@ -77,7 +77,7 @@ def main(): ...@@ -77,7 +77,7 @@ def main():
test_title = args.test_suite_title or args.test_suite test_title = args.test_suite_title or args.test_suite
if args.master_url: if args.master_url:
tool = taskdistribution.TaskDistributionTool(args.master_url) tool = taskdistribution.TaskDistributor(args.master_url)
test_result = tool.createTestResult(args.revision, test_result = tool.createTestResult(args.revision,
list(test_dict), list(test_dict),
args.test_node_title, args.test_node_title,
......
...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3 ...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
erp5.util = 0.4.49 erp5.util = 0.4.49
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12 ...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
# indirect dependancies # indirect dependancies
cp.recipe.cmd = 0.5 cp.recipe.cmd = 0.5
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
zope.exceptions = 4.0.7 zope.exceptions = 4.0.7
zope.testing = 4.1.3 zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0 zc.recipe.testrunner = 2.0.0
......
...@@ -58,8 +58,8 @@ mode = 0644 ...@@ -58,8 +58,8 @@ mode = 0644
[versions] [versions]
PyXML = 0.8.5 PyXML = 0.8.5
erp5.util = 0.4.49 erp5.util = 0.4.50
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
ipython = 5.3.0 ipython = 5.3.0
apache-libcloud = 2.1.0 apache-libcloud = 2.1.0
gitdb2 = 2.0.2 gitdb2 = 2.0.2
......
...@@ -158,10 +158,10 @@ def main(): ...@@ -158,10 +158,10 @@ def main():
access_url_https = 'https://%s:10443' % (args.partition_ipv4,) access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
tool = taskdistribution.TaskDistributionTool( distributor = taskdistribution.TaskDistributor(
args.master_url, args.master_url,
logger=logger) logger=logger)
test_result = tool.createTestResult( test_result = distributor.createTestResult(
revision, suite.getTestList(), args.test_node_title, revision, suite.getTestList(), args.test_node_title,
suite.allow_restart, test_suite_title, args.project_title) suite.allow_restart, test_suite_title, args.project_title)
if test_result is None: if test_result is None:
......
...@@ -71,4 +71,4 @@ output = ${buildout:directory}/template.cfg ...@@ -71,4 +71,4 @@ output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
[versions] [versions]
erp5.util = 0.4.49 erp5.util = 0.4.50
...@@ -5,6 +5,7 @@ extends = ...@@ -5,6 +5,7 @@ extends =
parts = parts =
instance-template instance-template
slapos-cookbook
[instance-template] [instance-template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
...@@ -14,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf ...@@ -14,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3 ...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
docutils = 0.12 docutils = 0.12
plone.recipe.command = 1.1 plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos001 rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
...@@ -72,7 +72,7 @@ async = 0.6.1 ...@@ -72,7 +72,7 @@ async = 0.6.1
gitdb = 0.5.4 gitdb = 0.5.4
pycrypto = 2.6 pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.40.4 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
plone.recipe.command = 1.1 plone.recipe.command = 1.1
......
...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 ...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -91,8 +91,8 @@ def main(): ...@@ -91,8 +91,8 @@ def main():
if args.target == 'firefox': if args.target == 'firefox':
firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True firefox_capabilities['marionette'] = True
firefox_capabilities['binary'] = '${firefox:location}/firefox-slapos'
browser = webdriver.Firefox(capabilities=firefox_capabilities, browser = webdriver.Firefox(capabilities=firefox_capabilities,
firefox_binary='${firefox:location}/firefox-slapos',
executable_path='${firefox:location}/geckodriver') executable_path='${firefox:location}/geckodriver')
elif args.target in ['iOS', 'Android']: elif args.target in ['iOS', 'Android']:
# parameters for mobile emulators have different names then parameters for # parameters for mobile emulators have different names then parameters for
...@@ -169,7 +169,7 @@ def main(): ...@@ -169,7 +169,7 @@ def main():
browser.quit() browser.quit()
is_browser_running = False is_browser_running = False
tool = taskdistribution.TaskDistributionTool(portal_url=args.master_url) tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
test_result = tool.createTestResult(revision = revision, test_result = tool.createTestResult(revision = revision,
test_name_list = test_line_dict.keys(), test_name_list = test_line_dict.keys(),
node_title = args.test_node_title, node_title = args.test_node_title,
......
...@@ -107,11 +107,11 @@ mode = 0644 ...@@ -107,11 +107,11 @@ mode = 0644
[template-runTestSuite] [template-runTestSuite]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/runTestSuite.in url = ${:_profile_base_location_}/runTestSuite.in
md5sum = fcf15b2a90340e0afe8f8b9921a4ffae md5sum = ff66d13f73982e8257eb5535cdb541c7
output = ${buildout:directory}/runTestSuite.in output = ${buildout:directory}/runTestSuite.in
mode = 0644 mode = 0644
[versions] [versions]
erp5.util = 0.4.49 erp5.util = 0.4.50
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
selenium = 2.53.1 selenium = 3.8.0
...@@ -85,7 +85,7 @@ pyzmq = 16.0.2 ...@@ -85,7 +85,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1 scikit-learn = 0.18.1
seaborn = 0.7.1 seaborn = 0.7.1
simplegeneric = 0.8.1 simplegeneric = 0.8.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
statsmodels = 0.8.0 statsmodels = 0.8.0
terminado = 0.6 terminado = 0.6
tornado = 4.4.2 tornado = 4.4.2
......
...@@ -11,7 +11,7 @@ apache-libcloud = 1.1.0 ...@@ -11,7 +11,7 @@ apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
gitdb = 0.6.4 gitdb = 0.6.4
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
smmap = 0.9.0 smmap = 0.9.0
# websockify = 0.8.0 # websockify = 0.8.0
......
...@@ -17,7 +17,7 @@ parts += ...@@ -17,7 +17,7 @@ parts +=
versions = versions versions = versions
[versions] [versions]
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
[template-instance] [template-instance]
......
...@@ -116,7 +116,7 @@ mysqlclient = 1.3.12 ...@@ -116,7 +116,7 @@ mysqlclient = 1.3.12
persistent = 4.2.3 persistent = 4.2.3
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap2 = 2.0.1 smmap2 = 2.0.1
transaction = 1.7.0 transaction = 1.7.0
......
...@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in ...@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
...@@ -65,4 +65,4 @@ mode = 0644 ...@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -110,7 +110,7 @@ gitdb = 0.6.4 ...@@ -110,7 +110,7 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
......
...@@ -47,4 +47,4 @@ mode = 0644 ...@@ -47,4 +47,4 @@ mode = 0644
[versions] [versions]
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -11,4 +11,4 @@ extends = common.cfg ...@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6 Pygments = 1.6
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10 collective.recipe.template = 1.10
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -97,4 +97,4 @@ mode = 640 ...@@ -97,4 +97,4 @@ mode = 640
Pygments = 2.1.3 Pygments = 2.1.3
collective.recipe.template = 1.10 collective.recipe.template = 1.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -15,7 +15,7 @@ gitdb = 0.6.4 ...@@ -15,7 +15,7 @@ gitdb = 0.6.4
gunicorn = 19.7.1 gunicorn = 19.7.1
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
......
...@@ -59,7 +59,7 @@ eggs = collective.recipe.template ...@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11 collective.recipe.template = 1.11
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.28 slapos.recipe.build = 0.28
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test. # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout] [buildout]
......
...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb ...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640 mode = 640
[versions] [versions]
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
[buildout]
extends =
../wendelin/software.cfg
../../component/mariadb/mariarocks.cfg
[local-bt5-repository]
list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
[patch-template]
recipe = slapos.recipe.build
location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
script =
with open(self.options['location'], 'w') as dst, \
open(self.options['base']) as src:
src = src.read()
i = src.index('[buildout]')
dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
...@@ -33,7 +33,7 @@ tags = ...@@ -33,7 +33,7 @@ tags =
[feeder] [feeder]
feeder = feeder =
# # Same algorithm as ERP5Site_simulateFluentdIngestion
import os, struct import os, struct
from random import lognormvariate from random import lognormvariate
pack = struct.Struct('!d').pack pack = struct.Struct('!d').pack
......
[buildout] [buildout]
extends = extends =
../../component/gnupg/buildout.cfg
../fluentd/software.cfg ../fluentd/software.cfg
../wendelin/software.cfg test-common.cfg
../../component/mariadb/mariarocks.cfg
[local-bt5-repository]
list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
[patch-template]
recipe = slapos.recipe.build
location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
script =
with open(self.options['location'], 'w') as dst, \
open(self.options['base']) as src:
src = src.read()
i = src.index('[buildout]')
dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
[template-erp5] [template-erp5]
recipe = recipe =
......
# NEO: data deduplication must be enabled # NEO: data deduplication must be enabled
[buildout] [buildout]
extends = test-fluentd-common.cfg extends =
../../component/gnupg/buildout.cfg
test-fluentd-common.cfg
[template-erp5-patched] [template-erp5-patched]
extra = extra =
......
# The 'start_ingest' command causes a zope to fill NEO as fast as possible,
# as if fluentd pushed data.
#
# Use sigma > 0 to have oids of variable size inside NEO.
# To know average compression ratio:
# x=test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py
# $x 10 1
# 0.434851958247
# 2155 - 65536 (99th percentile)
# $x 8.787 0
# 0.100036621094
# 6556
[buildout]
extends = test-common.cfg
parts += start_ingest
[start_ingest]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
template =
inline:#!${buildout:executable}
import argparse, base64, httplib, sys
parser = argparse.ArgumentParser()
_ = parser.add_argument
_('--site-id', default='erp5')
_('hostport', metavar='host[:port]', help='Zope address')
_('password', help="'zope' user password")
_('reference', help='Data Stream reference')
_('mu', type=float)
_('sigma', type=float)
_('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
args = parser.parse_args()
qs = []
for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
v = getattr(args, k)
if v is not None:
t = type(v)
qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
c = httplib.HTTPConnection(args.hostport)
c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
% (args.site_id, '&'.join(qs)))
c.putheader('Authorization',
'Basic ' + base64.b64encode('zope:'+args.password))
c.endheaders()
#!/usr/bin/python
from __future__ import division, print_function
import os, struct
from random import lognormvariate
bigfile_chunk_size = 65536
def simulateFluentdIngestion(self, reference, mu, sigma,
chunks_per_transaction=128):
from time import time
import transaction
note = (self['portal_ingestion_policies']['scalability_test_unpack'].getPath()
+ '/ingest')
module = self['data_stream_module']
try:
data_stream = module[reference]
except KeyError:
data_stream = module.newContent(reference, 'Data Stream')
transaction.commit()
pack = struct.Struct('!d').pack
data = os.urandom(bigfile_chunk_size - 8)
while 1:
txn = transaction.begin()
data_stream.appendData(''.join(
(pack(time()) + data[:int(lognormvariate(mu, sigma))]
).ljust(bigfile_chunk_size, '\0')
for _ in xrange(chunks_per_transaction)))
txn.note(note)
txn.commit()
if __name__ == '__main__':
import sys
mu, sigma = map(float, sys.argv[1:3])
if sigma:
try:
n = int(sys.argv[3])
except IndexError:
n = 1000000
else:
n = 1
x = sorted(min(int(lognormvariate(mu, sigma)), bigfile_chunk_size - 8)
for _ in xrange(n))
print((8 * n + sum(x)) / (bigfile_chunk_size * n))
if n == 1:
print(x[0] + 8)
else:
n //= 100
if n:
print(8 + x[n], '-', 8 + x[-n-1], '(99th percentile)')
else:
print(8 + x[0], '-', 8 + x[-1])
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 8, 46: Redefining name \'mu\' from outer scope (line 35) (redefined-outer-name)</string>
<string>W: 8, 50: Redefining name \'sigma\' from outer scope (line 35) (redefined-outer-name)</string>
</tuple>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>simulateFluentdIngestion</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_simulateFluentdIngestion</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
extension.erp5.ScalabilityFluentd
\ No newline at end of file
portal_ingestion_policies/scalability_test_* portal_ingestion_policies/scalability_test_*
portal_skins/custom/DataStreamModule_getTotalSize portal_skins/custom/DataStreamModule_getTotalSize
portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
\ No newline at end of file portal_skins/custom/ERP5Site_simulateFluentdIngestion
\ No newline at end of file
...@@ -94,7 +94,7 @@ caucase = 0.1.4 ...@@ -94,7 +94,7 @@ caucase = 0.1.4
futures = 3.1.1 futures = 3.1.1
gitdb2 = 2.0.2 gitdb2 = 2.0.2
gunicorn = 19.7.1 gunicorn = 19.7.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap2 = 2.0.3 smmap2 = 2.0.3
......
...@@ -89,7 +89,7 @@ PasteScript = 2.0.2 ...@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7 WSGIUtils = 0.7
python-magic = 0.4.6 python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
# Required by: # Required by:
# PasteScript==2.0 # PasteScript==2.0
......
...@@ -736,7 +736,7 @@ uuid = 1.30 ...@@ -736,7 +736,7 @@ uuid = 1.30
validictory = 1.1.0 validictory = 1.1.0
xfw = 0.10 xfw = 0.10
xupdate-processor = 0.4 xupdate-processor = 0.4
selenium = 2.53.1 selenium = 3.8.0
# Required by: # Required by:
# Products.CMFCore==2.2.10 # Products.CMFCore==2.2.10
......
...@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d ...@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
[template-mariadb] [template-mariadb]
filename = instance-mariadb.cfg.in filename = instance-mariadb.cfg.in
md5sum = 8ea5033142f450a2e90431817771cb44 md5sum = 244e185ed26831fe0b5abf49c67753d6
[template-kumofs] [template-kumofs]
filename = instance-kumofs.cfg.in filename = instance-kumofs.cfg.in
......
...@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }} ...@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }}
# can be fully restored. # can be fully restored.
# master-data: use value "2" as we are not in a replication case # master-data: use value "2" as we are not in a replication case
#} #}
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz" command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
{# KEEP GLOB PATTERN IN SYNC with generated filenames above {# KEEP GLOB PATTERN IN SYNC with generated filenames above
# YYYYmmddHHMMSS -#} # YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz file-glob = ??????????????.sql.gz
......
...@@ -132,4 +132,5 @@ PyRSS2Gen = 1.1 ...@@ -132,4 +132,5 @@ PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
pycurl = 7.43.0 pycurl = 7.43.0
slapos.toolbox = 0.73 slapos.toolbox = 0.73
pyasn1 = 0.3.7
...@@ -91,6 +91,7 @@ eggs = ...@@ -91,6 +91,7 @@ eggs =
slapos.libnetworkcache slapos.libnetworkcache
[versions] [versions]
setuptools = 33.1.1
# Use SlapOS patched zc.buildout # Use SlapOS patched zc.buildout
zc.buildout = 2.5.2+slapos011 zc.buildout = 2.5.2+slapos011
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2) # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
...@@ -125,7 +126,6 @@ pyOpenSSL = 17.2.0 ...@@ -125,7 +126,6 @@ pyOpenSSL = 17.2.0
pyparsing = 2.2.0 pyparsing = 2.2.0
pytz = 2016.10 pytz = 2016.10
requests = 2.13.0 requests = 2.13.0
setuptools = 33.1.1
six = 1.10.0 six = 1.10.0
slapos.cookbook = 1.0.53 slapos.cookbook = 1.0.53
slapos.core = 1.4.3 slapos.core = 1.4.3
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment