Commit bb23fc27 authored by Benjamin Blanc's avatar Benjamin Blanc Committed by Julien Muchembled

Add scalability test support

parent 145a2a94
...@@ -17,7 +17,6 @@ command = grep parts ${buildout:develop-eggs-directory}/cloudooo.egg-link ...@@ -17,7 +17,6 @@ command = grep parts ${buildout:develop-eggs-directory}/cloudooo.egg-link
[cloudooo-repository] [cloudooo-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/cloudooo.git repository = http://git.erp5.org/repos/cloudooo.git
branch = master
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[cloudooo] [cloudooo]
......
...@@ -108,6 +108,7 @@ setup(name=name, ...@@ -108,6 +108,7 @@ setup(name=name,
'erp5.update = slapos.recipe.erp5_update:Recipe', 'erp5.update = slapos.recipe.erp5_update:Recipe',
'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe', 'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
'erp5testnode = slapos.recipe.erp5testnode:Recipe', 'erp5testnode = slapos.recipe.erp5testnode:Recipe',
'scalability = slapos.recipe.scalability:Recipe',
'firefox = slapos.recipe.firefox:Recipe', 'firefox = slapos.recipe.firefox:Recipe',
'fontconfig = slapos.recipe.fontconfig:Recipe', 'fontconfig = slapos.recipe.fontconfig:Recipe',
'generate.mac = slapos.recipe.generatemac:Recipe', 'generate.mac = slapos.recipe.generatemac:Recipe',
......
...@@ -33,10 +33,61 @@ import urlparse ...@@ -33,10 +33,61 @@ import urlparse
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
""" """
Instanciate ERP5 in Zope Instanciate ERP5 in Zope
Input:
mysql-url
mysql url, must contain connexion informations.
zope-url (optional)
Url of zope, this url will be parsed to get these values:
protocol://username@password:hostname:port/site_id
If specified following inputs have priority on parsed zope-url values:
zope-hostname (optional)
Zope hostname.
Default value: "localhost", if not contained in zope-url.
zope-port (optional)
Zope port.
Default value: "80", if not contained in zope-url.
zope-protocol (optional)
Protocol used.
Default "http" if not contained in zope-url.
zope-password (optional)
Zope user password.
Default value: "insecure", if not contained in zope-url.
zope-username (optional)
Zope username.
Default value: "zope", if not contained in zope-url.
site-id (optional)
Site id
ex: erp5
Default value: "erp5", if not contained in zope-url.
runner-path
The path to create the runner.
scalability (optional)
Boolean, default value: "False"
If true erp5 site will fix the site consistency and
will use configurator to automaticlly install a
small and medium buisiness.
""" """
def install(self): def install(self):
parsed = urlparse.urlparse(self.options['mysql-url'])
# Get mysql url
mysql_url = None
if self.options.get('mysql-url'):
mysql_url = self.options['mysql-url']
else:
# Get url of the first database of the list
mysql_url_list = self.options['mysql-url-list']
# Parse string to list
if not isinstance(mysql_url_list, list):
mysql_url_list = mysql_url_list[1:len(mysql_url_list)-1].split(',')
mysql_url = mysql_url_list[0]
parsed = urlparse.urlparse(mysql_url)
mysql_connection_string = "%(database)s@%(hostname)s:%(port)s "\ mysql_connection_string = "%(database)s@%(hostname)s:%(port)s "\
"%(username)s %(password)s" % dict( "%(username)s %(password)s" % dict(
database=parsed.path.split('/')[1], database=parsed.path.split('/')[1],
...@@ -46,15 +97,70 @@ class Recipe(GenericBaseRecipe): ...@@ -46,15 +97,70 @@ class Recipe(GenericBaseRecipe):
password=parsed.password password=parsed.password
) )
# Init zope configuration
zope_username = None
zope_password = None
zope_hostname = None
zope_port = None
zope_protocol = None
# Get informations from zope url
if self.options.get('zope-url'):
zope_parsed = urlparse.urlparse(self.options['zope-url']) zope_parsed = urlparse.urlparse(self.options['zope-url'])
# Zope hostname
if self.options.get('zope-hostname'):
zope_hostname = self.options['zope-hostname']
elif self.options.get('zope-url'):
zope_hostname = zope_parsed.hostname
else:
zope_hostname = 'localhost'
# Zope port
if self.options.get('zope-port'):
zope_port = self.options['zope-port']
elif self.options.get('zope-url'):
zope_port = str(zope_parsed.port)
else:
zope_port = "8080"
# Zope username and password
if self.options.get('zope-username') and self.options.get('zope-password'):
zope_username = self.options['zope-username']
zope_password = self.options['zope-password']
elif self.options.get('zope-url'):
zope_username = zope_parsed.username
zope_password = zope_parsed.password
else:
zope_username = 'zope'
zope_password = 'insecure'
# Zope protocol
if self.options.get('zope-protocol'):
zope_protocol = self.options['zope-protocol']
elif self.options.get('zope-url'):
zope_protocol = zope_parsed.scheme
else:
zope_protocol = 'http'
# Zope site-id
if self.options.get('zope-site-id'):
zope_site_id = self.options['zope-site-id']
elif self.options.get('zope-url'):
zope_site_id = zope_parsed.path.split('/')[1],
else:
zope_site_id = 'erp5'
# Scalability
if self.options.get('scalability'):
scalability = self.options['scalability']
else:
scalability = "False"
config = dict( config = dict(
python_path=sys.executable, python_path=sys.executable,
user=zope_parsed.username, user=zope_username,
password=zope_parsed.password, password=zope_password,
site_id=zope_parsed.path.split('/')[1], site_id=zope_site_id,
host="%s:%s" % (zope_parsed.hostname, zope_parsed.port), host="%s:%s" % (zope_hostname, str(zope_port)),
protocol=zope_protocol,
sql_connection_string=mysql_connection_string, sql_connection_string=mysql_connection_string,
scalability="True",
) )
# Runners # Runners
......
...@@ -3,28 +3,108 @@ ...@@ -3,28 +3,108 @@
import httplib import httplib
import urllib import urllib
import base64 import base64
import time
MAX_INSTALLATION_TIME = 60*30
MAX_TESTING_TIME = 60*2
MAX_GETTING_CONNECTION_TIME = 60*5
user = "%(user)s" user = "%(user)s"
password = "%(password)s" password = "%(password)s"
host = "%(host)s" host = "%(host)s"
site_id = "%(site_id)s" site_id = "%(site_id)s"
erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
mysql_url = "%(sql_connection_string)s" mysql_url = "%(sql_connection_string)s"
protocol = "%(protocol)s"
scalability = "%(scalability)s" == "True"
erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
header_dict = {'Authorization': 'Basic %%s' %% \ header_dict = {'Authorization': 'Basic %%s' %% \
base64.encodestring('%%s:%%s' %% (user, password)).strip()} base64.encodestring('%%s:%%s' %% (user, password)).strip()}
zope_connection = httplib.HTTPConnection(host)
# Check if an ERP5 site is already created, as ERP5 does support having def getConnection():
# 2 instances in the same zope, and this script should not destroy user data print "Getting new connection"
zope_connection.request('GET', '/isERP5SitePresent', headers=header_dict) start_time = time.time()
result = zope_connection.getresponse() count = 0
while MAX_GETTING_CONNECTION_TIME > time.time()-start_time:
try:
count = count + 1
if protocol == 'https':
return httplib.HTTPSConnection(host)
elif protocol == 'http':
return httplib.HTTPConnection(host)
else:
raise ValueError("Protocol not implemented")
except:
print "Getting new connection failed, retry"
time.sleep(10)
raise ValueError("Cannot get new connection after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
if result.status == 204: # and (result.read() == "False"): def testIfExist(page, unexcepted_content="Site Error"):
print "Test if %%s exists" %%(page)
start_time = time.time()
count = 0
while MAX_TESTING_TIME > time.time()-start_time:
try:
count = count + 1
zope_connection = getConnection()
zope_connection.request('GET', '/%%s/%%s' %%(site_id, page))
result = zope_connection.getresponse()
return not unexcepted_content in result.read()
except:
print "Test if exists failed, retry..."
time.sleep(10)
raise ValueError("Cannot testIfExist after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
# Use a new connection
zope_connection = httplib.HTTPConnection(host)
def waitFor0PendingActivities():
# TODO: tolerate 1 pending activities ? (mail server...)
print "waitFor0PendingActivities.."
start_time = time.time()
count = 0
ok = False
while MAX_INSTALLATION_TIME > time.time()-start_time and not ok:
try:
count = count + 1
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/portal_activities/getMessageList' %%(site_id),
headers=header_dict
)
result = zope_connection.getresponse()
message_list_text = result.read()
message_list = [s.strip() for s in message_list_text[1:-1].split(',')]
if len(message_list)==0:
print "There is no pending activities."
ok = True
#Hack to do not take into account persistent Alarm_installMailServer acitivities
if len(message_list)==1:
print "1 pending activity for 'Alarm_installMailServer'."
print "ok."
ok = True
print "There is %%d pending activities" %%len(message_list)
time.sleep(5)
except:
time.sleep(5)
print "Getting activities failed, retry."
if not ok:
raise ValueError("Pending activities always here after %%d check/try (for %%s s)" %%(count, str(time.time()-start_time)))
def createERP5Site():
# Check if an ERP5 site is already created, as ERP5 does support having
# 2 instances in the same zope, and this script should not destroy user data
zope_connection = getConnection()
zope_connection.request('GET', '/isERP5SitePresent', headers=header_dict)
result = zope_connection.getresponse()
if result.status == 204: # and (result.read() == "False"):
# Use a new connection
zope_connection = getConnection()
# Create the expected ERP5 instance # Create the expected ERP5 instance
zope_connection.request( zope_connection.request(
'POST', '/manage_addProduct/ERP5/manage_addERP5Site', 'POST', '/manage_addProduct/ERP5/manage_addERP5Site',
...@@ -40,6 +120,125 @@ if result.status == 204: # and (result.read() == "False"): ...@@ -40,6 +120,125 @@ if result.status == 204: # and (result.read() == "False"):
result = zope_connection.getresponse() result = zope_connection.getresponse()
# Read result make sure the site really finished to # Read result make sure the site really finished to
#created the ERP5 site. # created the ERP5 site.
result.read() print result.read()
print "ERP5 site created." print "ERP5 site created."
createERP5Site()
# Scalability case
if scalability:
while True:
count = 0
createERP5Site()
# Fix site consistency
if not testIfExist("/%%s/portal_configurator/" %%site_id):
count = count + 1
print "Going to fix site consistency..."
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/ERP5Site_launchFixConfigurationConsistency' %%(site_id),
headers=header_dict
)
result = zope_connection.getresponse()
print "Check consistency..."
print result.read()
# Wait activities
waitFor0PendingActivities()
print "Site consistency checking: done."
# Install testing scalability business configuration if not exists
if testIfExist("/%%s/portal_configurator/" %%site_id) \
and not testIfExist("/%%s/sale_order_module/" %%site_id):
count = count + 1
print "Going to install testing scalability business configuration.."
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/business_configuration_module/1/build' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
print "Build scalability business configuration..."
print result.read()
# Wait activities
waitFor0PendingActivities()
print "Scalability business configuration building: done."
# Force to use random id generator
print "Force to use random id.."
try:
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/use_random_id_scalability' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
assert(result.read()=="1")
# Wait activities
waitFor0PendingActivities()
print "use_random_id_scalability: done."
except:
print "use_random_id_scalability: fail."
# Create scalability users
if testIfExist("/%%s/person_module/scalability_user/getTitle" %%site_id) \
and not testIfExist("/%%s/person_module/scalability_user_0/getTitle" %%site_id):
count = count + 1
print "Going to create scalability users.."
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/init_scalability' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
print "Creating scalaiblity users..."
assert(result.read()=="1")
# Wait activities
waitFor0PendingActivities()
print "Scalability users creation: done."
# Try to Updates roles and sleep
if count == 0:
print "Trying to update roles.."
try:
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/update_roles_scalability' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
assert(result.read()=="1")
# Wait activities
waitFor0PendingActivities()
print "Update roles: done."
except:
print "Update roles: fail. (but may be already done before)"
time.sleep(30)
# XXX: Hack to perform load balacing
# TODO: Use an other way to do the load balancing (for example
# using zope.conf, and using family-zope group names).
# Apply load balacing
if testIfExist("/%%s/portal_activities/manageLoadBalancing" %%site_id) \
and testIfExist("/%%s/person_module/scalability_user_0/getTitle" %%site_id):
print "Load balacing.."
try:
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/apply_load_balancing_scalability' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
assert("Processing Nodes" in result.read())
# Wait activities
waitFor0PendingActivities()
print "Load balancing init: done."
except:
print "Load balancing init: fail."
# sleep
time.sleep(30)
\ No newline at end of file
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
path_list = []
configuration_file = self.createFile(
self.options['configuration-file'],
'',
)
path_list.append(configuration_file)
path_list.append(
self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
[ # Executable
[ self.options['exec-path'],
'--erp5-url', self.options['erp5-url'],
'--test-result-path', self.options['test-result-path'],
'--test-suite-master-url', self.options['test-suite-master-url'],
'--node-title', self.options['node-title'],
'--test-suite', self.options['test-suite'],
'--revision', self.options['revision'],
'--log-path', self.options['log-path'],
'--runner-path', self.options['runner-path'],
'--erp5-location', self.options['erp5-location'],
],
# Environment
{
'GIT_SSL_NO_VERIFY': '1',
}
],
)
)
return path_list
[buildout] [buildout]
parts = parts =
scalability-runner
test-runner test-runner
sh-environment sh-environment
...@@ -24,6 +25,18 @@ git-executable = ${git:location}/bin/git ...@@ -24,6 +25,18 @@ git-executable = ${git:location}/bin/git
<= download-source <= download-source
repository = ${erp5-util-repository:location} repository = ${erp5-util-repository:location}
[scalability-runner]
recipe = slapos.cookbook:egg_test
run-scalability-test-suite = $${create-directory:bin}/runScalabilityTestSuite
run-scalability-test-suite-binary = $${buildout:bin-directory}/runScalabilityTestSuite
# The list of executables should be defined here and a combination
# of tests should dynamically generated.
#python-list = $${}
test-list =
$${erp5-util:location}
prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
environment = environment
[test-runner] [test-runner]
recipe = slapos.cookbook:egg_test recipe = slapos.cookbook:egg_test
run-test-suite = $${create-directory:bin}/runTestSuite run-test-suite = $${create-directory:bin}/runTestSuite
......
...@@ -45,7 +45,7 @@ branch = master ...@@ -45,7 +45,7 @@ branch = master
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
md5sum = 6b919dd280d6972afe0f50d389ba5fe6 md5sum = 4449ebd6037fd248dd70a0ff7a6d6805
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 640 mode = 640
......
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
develop = develop =
${:parts-directory}/slapos.cookbook-repository ${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/cloudooo-repository ${:parts-directory}/cloudooo-repository
${:parts-directory}/erp5
extensions = extensions =
slapos.zcbworkarounds slapos.zcbworkarounds
...@@ -14,6 +15,8 @@ find-links = ...@@ -14,6 +15,8 @@ find-links =
http://dist.repoze.org http://dist.repoze.org
http://www.nexedi.org/static/packages/source/ http://www.nexedi.org/static/packages/source/
http://www.owlfish.com/software/wsgiutils/download.html http://www.owlfish.com/software/wsgiutils/download.html
https://pypi.python.org/packages/source/z/z3c.etestbrowser/
https://pypi.python.org/packages/source/z/zope.testbrowser/
allow-hosts += pybrary.net allow-hosts += pybrary.net
...@@ -104,6 +107,8 @@ parts = ...@@ -104,6 +107,8 @@ parts =
eggs eggs
testrunner testrunner
test_suite_runner test_suite_runner
scalability_test_suite_runner
performance_tester
# basic Xorg # basic Xorg
libXdmcp libXdmcp
...@@ -133,6 +138,9 @@ parts = ...@@ -133,6 +138,9 @@ parts =
# Local development # Local development
slapos.cookbook-repository slapos.cookbook-repository
erp5.util
slapos.cookbook
erp5.util-check
check-recipe check-recipe
# Create instance template # Create instance template
template template
...@@ -142,12 +150,33 @@ recipe = slapos.recipe.build:download ...@@ -142,12 +150,33 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
mode = 644 mode = 644
[erp5.util-check]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
[erp5.util]
recipe = zc.recipe.egg
eggs = erp5.util
scripts =
ugly-depend-on = ${erp5:repository}
# Local development # Local development
[slapos.cookbook-repository] [slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git repository = http://git.erp5.org/repos/slapos.git
branch = erp5-cluster #branch = erp5-cluster
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
#repository = http://192.168.242.77:1236/slapos.git
branch = erp5-cluster-scalability
[slapos.cookbook]
recipe = zc.recipe.egg
eggs = slapos.cookbook
scripts =
ugly-depend-on = ${slapos.cookbook-repository:repository} ${slapos.cookbook-repository:branch}
[check-recipe] [check-recipe]
recipe = plone.recipe.command recipe = plone.recipe.command
...@@ -166,6 +195,8 @@ context = ...@@ -166,6 +195,8 @@ context =
key bin_directory buildout:bin-directory key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key erp5_location erp5:location
key erp5_dev_location erp5:location
${:extra-context} ${:extra-context}
[template-mariadb] [template-mariadb]
...@@ -213,7 +244,7 @@ md5sum = 564006953b7d7a12d40a14b6648b32f0 ...@@ -213,7 +244,7 @@ md5sum = 564006953b7d7a12d40a14b6648b32f0
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/instance.cfg.in
md5sum = f71e14621d90903eba1b943401548b8a md5sum = 3d5a16f831a3b73f9c0e2aae7abbd811
extra-context = extra-context =
key mariadb_link_binary template-mariadb:link-binary key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary key zope_link_binary template-zope:link-binary
...@@ -224,7 +255,6 @@ extra-context = ...@@ -224,7 +255,6 @@ extra-context =
key curl_location curl:location key curl_location curl:location
key dash_location dash:location key dash_location dash:location
key dcron_location dcron:location key dcron_location dcron:location
key erp5_location erp5:location
key file_location file:location key file_location file:location
key findutils_location findutils:location key findutils_location findutils:location
key fontconfig_location fontconfig:location key fontconfig_location fontconfig:location
...@@ -264,13 +294,14 @@ extra-context = ...@@ -264,13 +294,14 @@ extra-context =
key template_zeo template-zeo:target key template_zeo template-zeo:target
key template_zope template-zope:target key template_zope template-zope:target
key template_zope_conf template-zope-conf:target key template_zope_conf template-zope-conf:target
key template_scalability template-scalability:target
key wget_location wget:location key wget_location wget:location
key zlib_location zlib:location key zlib_location zlib:location
[template-erp5] [template-erp5]
< = download-base < = download-base
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 5b11875e6beba48db7d45c5d462a6d2d md5sum = 0620af0820d202dedcf73334365a44bb
[template-neo] [template-neo]
< = download-base < = download-base
...@@ -328,6 +359,11 @@ extra-context = ...@@ -328,6 +359,11 @@ extra-context =
key gzip_location gzip:location key gzip_location gzip:location
key logrotate_location logrotate:location key logrotate_location logrotate:location
[template-scalability]
< = download-base
filename = instance-scalability.cfg.in
md5sum = d45b0b8e2fdb7fc11a5f5e4720a66879
[bt5-repository] [bt5-repository]
# Format: # Format:
# <url or path> [...] # <url or path> [...]
...@@ -355,7 +391,8 @@ repository_id_list = erp5 ...@@ -355,7 +391,8 @@ repository_id_list = erp5
[erp5] [erp5]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/erp5.git repository = http://git.erp5.org/repos/erp5.git
branch = master #repository = http://192.168.242.77:1235/erp5.git
#branch = master
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[fix-products-paths] [fix-products-paths]
...@@ -404,16 +441,7 @@ initialization = ...@@ -404,16 +441,7 @@ initialization =
os.environ['PATH'] = '${jsl:location}/bin:%s' % os.environ['PATH'] os.environ['PATH'] = '${jsl:location}/bin:%s' % os.environ['PATH']
os.environ['CGI_PATH'] = '${w3-validator:location}/httpd/cgi-bin' os.environ['CGI_PATH'] = '${w3-validator:location}/httpd/cgi-bin'
[test_suite_runner] [erp5-bin-base]
# XXX: Workaround for fact ERP5Type is not an distribution and does not
# expose entry point for test runner
recipe = zc.recipe.egg
python = python2.7
eggs = ${eggs:eggs}
extra-paths = ${eggs:extra-paths}
entry-points =
runTestSuite=Products.ERP5Type.tests.runTestSuite:main
scripts = runTestSuite
initialization = initialization =
import os import os
import sys import sys
...@@ -425,6 +453,36 @@ initialization = ...@@ -425,6 +453,36 @@ initialization =
repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split())) repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list] sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
[test_suite_runner]
< = erp5-bin-base
# XXX: Workaround for fact ERP5Type is not an distribution and does not
# expose entry point for test runner
recipe = zc.recipe.egg
python = python2.7
eggs = ${eggs:eggs}
extra-paths = ${eggs:extra-paths}
entry-points =
runTestSuite=Products.ERP5Type.tests.runTestSuite:main
scripts = runTestSuite
[scalability_test_suite_runner]
< = erp5-bin-base
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
extra-paths = ${eggs:extra-paths}
entry-points =
runScalabilityTestSuite=erp5.util.scalability.runScalabilityTestSuite:main
scripts = runScalabilityTestSuite
[performance_tester]
< = erp5-bin-base
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
extra-paths = ${eggs:extra-paths}
entry-points =
performance_tester=erp5.util.benchmark.performance_tester:main
scripts = performance_tester
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
python = python2.7 python = python2.7
...@@ -465,7 +523,8 @@ eggs = ...@@ -465,7 +523,8 @@ eggs =
feedparser feedparser
argparse argparse
validictory validictory
erp5.util # erp5.util doesn't have to be here because is it developed (at part/erp5)
# erp5.util
huBarcode huBarcode
qrcode qrcode
spyne spyne
...@@ -509,7 +568,12 @@ eggs = ...@@ -509,7 +568,12 @@ eggs =
# Needed for parsing .po files from our Localizer subset # Needed for parsing .po files from our Localizer subset
polib polib
# performance_tester dependencies
z3c.etestbrowser
zope.testbrowser
# runScalabilityTestSuite dependencie
slapos.core
psutil
# parameterizing the version of the generated python interpreter name by the # parameterizing the version of the generated python interpreter name by the
# python section version causes dependency between this egg section and the # python section version causes dependency between this egg section and the
# installation of python, which we don't want on an instance # installation of python, which we don't want on an instance
...@@ -536,7 +600,6 @@ entry-points = ...@@ -536,7 +600,6 @@ entry-points =
scripts = zodbanalyze scripts = zodbanalyze
[cloudooo-repository] [cloudooo-repository]
branch = master
revision = 5c67568c403239bd8e25993602d03c553236fcec revision = 5c67568c403239bd8e25993602d03c553236fcec
[mysql-python] [mysql-python]
...@@ -568,6 +631,10 @@ scripts = ...@@ -568,6 +631,10 @@ scripts =
zodbpack zodbpack
[versions] [versions]
z3c.etestbrowser = 2.0.0
zope.testbrowser =
slapos.core = 0.32.3
psutil =
# pin Acquisition and Products.DCWorkflow to Nexedi flavour of eggs # pin Acquisition and Products.DCWorkflow to Nexedi flavour of eggs
Acquisition = 2.13.7nxd001 Acquisition = 2.13.7nxd001
Products.DCWorkflow = 2.2.3nxd002 Products.DCWorkflow = 2.2.3nxd002
...@@ -658,7 +725,7 @@ coverage = 3.6 ...@@ -658,7 +725,7 @@ coverage = 3.6
csp-eventlet = 0.7.0 csp-eventlet = 0.7.0
elementtree = 1.2.7-20070827-preview elementtree = 1.2.7-20070827-preview
erp5.recipe.cmmiforcei686 = 0.1.3 erp5.recipe.cmmiforcei686 = 0.1.3
erp5.util = 0.4.34 erp5.util =
erp5diff = 0.8.1.5 erp5diff = 0.8.1.5
eventlet = 0.12.1 eventlet = 0.12.1
feedparser = 5.1.3 feedparser = 5.1.3
......
...@@ -3,6 +3,14 @@ ...@@ -3,6 +3,14 @@
{% set frontend_dict = slapparameter_dict.get('frontend', {}) %} {% set frontend_dict = slapparameter_dict.get('frontend', {}) %}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%} {% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%} {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[request-common] [request-common]
recipe = slapos.cookbook:request.serialised recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
...@@ -17,6 +25,7 @@ config = ...@@ -17,6 +25,7 @@ config =
${:extra-config} ${:extra-config}
extra-config = extra-config =
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
state = ${slap-configuration:instance-state}
{% macro request(name, software_type, config_key, config={}, ret={'url': True}) -%} {% macro request(name, software_type, config_key, config={}, ret={'url': True}) -%}
{% do config.update(slapparameter_dict.get(config_key, {})) -%} {% do config.update(slapparameter_dict.get(config_key, {})) -%}
......
{% if slap_software_type == software_type -%}
{% set site_id = slapparameter_dict.get('cluster').get('site-id', 'erp5') -%}
{% set promise_path = slapparameter_dict.get('promise-path', 'erp5') -%}
[buildout]
extends =
parts =
erp5-cluster
binary-wrap-launcher
binary-wrap-performance-tester
erp5-bootstrap
erp5-promise
promise-erp5-site
promise-scalability-user-data
scalability-launcher-instance
publish
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
# Request erp5-cluster instance
[erp5-cluster]
recipe = slapos.cookbook:request.serialised
sla = computer_guid
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
name = ERP5 Cluster
software-url = ${slap-connection:software-release-url}
software-type = default
sla-computer_guid = {{ slapparameter_dict.get('launcher-computer-guid', computer_id) }}
{# Add a zope scalability family to the erp5 cluster configuration, -#}
{# this zope family is used by erp5-bootstrap and may serve to scalability monitoring tools. -#}
{% set additional_scalability_zope = {'scalability':{'family':'scalability', 'computer-guid':computer_id}} -%}
{% do slapparameter_dict.get('cluster').get('zope-partition-dict').update( additional_scalability_zope ) -%}
{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
config = {{ cluster_parameter_dict.keys() | join(' ') }}
{% for key, value in cluster_parameter_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
return = mariadb-database-list family-scalability family-user memcached-persistent-url memcached-volatile-url cloudooo-url
state = ${slap-configuration:instance-state}
# Create wrapper
[binary-wrap-launcher]
recipe = slapos.cookbook:wrapper
log-path = ${basedirectory:log}
binary-path = {{ bin_directory }}/runScalabilityTestSuite
wrapper-path = ${rootdirectory:bin}/runScalabilityTestSuite
output = ${binary-wrap-launcher:binary-path}
site-id = "/{{ site_id }}"
# Suppose that there is a user zope family in the configuration
erp5-url = ${erp5-cluster:connection-family-user}${:site-id}
parameters-extra = $*
command-line = ${binary-wrap-launcher:binary-path}
return = url
# Create wrapper
[binary-wrap-performance-tester]
recipe = slapos.cookbook:wrapper
binary-path = {{ bin_directory }}/performance_tester
wrapper-path = ${rootdirectory:bin}/performance_tester
output = ${binary-wrap-performance-tester:binary-path}
parameters-extra = $*
command-line = ${binary-wrap-performance-tester:binary-path}
return = url
[scalability-launcher-instance]
recipe = slapos.cookbook:scalability
configuration-file = ${rootdirectory:etc}/scalability_runner.conf
wrapper = ${basedirectory:services}/scalability_runner
exec-path = ${binary-wrap-launcher:wrapper-path}
erp5-url = ${binary-wrap-launcher:erp5-url}
test-result-path = {{ slapparameter_dict.get('test-result-path') }}
revision = {{ slapparameter_dict.get('test-suite-revision') }}
node-title = {{ slapparameter_dict.get('node-title', 'Unknown') }}
test-suite = {{ slapparameter_dict.get('test-suite') }}
test-suite-master-url = {{ slapparameter_dict.get('test-suite-master-url') }}
log-path = ${binary-wrap-launcher:log-path}
runner-path = ${binary-wrap-performance-tester:wrapper-path}
erp5-location = {{ erp5_location }}
# Create partition's directories
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = ${rootdirectory:var}/log
services = ${rootdirectory:etc}/run
promises = ${rootdirectory:etc}/promise/
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
bin = ${buildout:directory}/bin
[erp5-bootstrap]
recipe = slapos.cookbook:erp5.bootstrap
runner-path = ${basedirectory:services}/erp5-bootstrap
#mysql-url = ${erp5-cluster:mariadb-url}
mysql-url-list = ${erp5-cluster:connection-mariadb-database-list}
zope-username = zope
zope-password = insecure
zope-site-id = {{ site_id }}
zope-url = ${erp5-cluster:connection-family-scalability}
scalability = True
[erp5-promise]
recipe = slapos.cookbook:erp5.promise
promise-path = {{ promise_path }}
kumofs-url = ${erp5-cluster:connection-memcached-persistent-url}
memcached-url = ${erp5-cluster:connection-memcached-volatile-url}
cloudooo-url = ${erp5-cluster:connection-cloudooo-url}
smtp-url =
bt5 =
bt5-repository-url =
[promise-erp5-site]
recipe = slapos.cookbook:check_url_available
path = ${basedirectory:promises}/erp5site
site-id = "/{{ site_id }}"
url = ${erp5-cluster:connection-family-scalability}${:site-id}
dash_path = {{ dash_location }}/bin/dash
curl_path = {{ curl_location }}/bin/curl
[promise-scalability-user-data]
recipe = slapos.cookbook:check_url_available
path = ${basedirectory:promises}/scalabilityData
site-id = "/{{ site_id }}"
page = "/person_module/scalability_user_0/getTitle"
url = ${erp5-cluster:connection-family-scalability}${:site-id}${:page}
dash_path = {{ dash_location }}/bin/dash
curl_path = {{ curl_location }}/bin/curl
[publish]
recipe = slapos.cookbook:publish.serialised
erp5-frontend-url = ${scalability-launcher-instance:erp5-url}
erp5-scalability-url = ${erp5-cluster:connection-family-scalability}
{% endif %}
...@@ -187,6 +187,29 @@ extra-context = ...@@ -187,6 +187,29 @@ extra-context =
# Must match the key id in [switch-softwaretype] which uses this section. # Must match the key id in [switch-softwaretype] which uses this section.
raw software_type mariadb raw software_type mariadb
[dynamic-template-scalability-parameters]
erp5 = {{ erp5_location }}
erp5-dev = {{ erp5_dev_location }}
local-bt5-repository = {{ local_bt5_repository }}
curl-location = {{ curl_location }}
dash-location = {{ dash_location }}
bin-directory = {{ bin_directory }}
[dynamic-template-scalability]
< = jinja2-template-base
template = {{ template_scalability }}
filename = instance-scalability.cfg
extensions = jinja2.ext.do
extra-context =
key bin_directory dynamic-template-scalability-parameters:bin-directory
key curl_location dynamic-template-scalability-parameters:curl-location
key dash_location dynamic-template-scalability-parameters:dash-location
key erp5_location dynamic-template-scalability-parameters:erp5-dev
key local_bt5_repository dynamic-template-scalability-parameters:local-bt5-repository
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type test
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.recipe.build recipe = slapos.recipe.build
script = script =
...@@ -214,6 +237,7 @@ slapos_update_promise = ${:slapos_promise} ...@@ -214,6 +237,7 @@ slapos_update_promise = ${:slapos_promise}
override = {{ dumps(override_switch_softwaretype |default) }} override = {{ dumps(override_switch_softwaretype |default) }}
default = ${dynamic-template-erp5:rendered} default = ${dynamic-template-erp5:rendered}
test = ${dynamic-template-scalability:rendered}
kumofs = ${dynamic-template-kumofs:rendered} kumofs = ${dynamic-template-kumofs:rendered}
cloudooo = ${dynamic-template-cloudooo:rendered} cloudooo = ${dynamic-template-cloudooo:rendered}
mariadb = ${dynamic-template-mariadb:rendered} mariadb = ${dynamic-template-mariadb:rendered}
...@@ -221,3 +245,4 @@ balancer = ${dynamic-template-balancer:rendered} ...@@ -221,3 +245,4 @@ balancer = ${dynamic-template-balancer:rendered}
zodb-neo = ${dynamic-template-neo:rendered} zodb-neo = ${dynamic-template-neo:rendered}
zodb-zeo = ${dynamic-template-zeo:rendered} zodb-zeo = ${dynamic-template-zeo:rendered}
zope = ${dynamic-template-zope:rendered} zope = ${dynamic-template-zope:rendered}
RootSoftwareInstance = ${dynamic-template-erp5-RootSoftwareInstance:rendered}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment