Commit 50506c28 authored by Roque's avatar Roque

erp5-util-benchmark/scalability

First commit with all my changes in webrunner-testnode in order to run the benchmark suites using performance tester and runScalabilityTestSuite.
Full of logs, comments and hard-codes.
Cleanup code needed.
parent f9aaa614
# Specify user login/password used to run the tests. Note that there must be
# the same number of users specified here *and* on the script command-line.
user_tuple = tuple([('scalability_user_%i' % x, 'insecure') for x in range(0, 1000)])
#user_tuple = tuple([('scalability_user_%i' % x, 'insecure') for x in range(0, 1000)])
user_tuple = tuple([('zope', 'yiujrsvp') for x in range(0, 1)])
\ No newline at end of file
......@@ -82,7 +82,8 @@ def fillRelatedObjects(browser, result, name, maximum=1, actionName="", TMIN_SLE
for i in range(0, iteration):
line_number = random.randint(1,num_line) + 2
# Check the box corresponding to line_number if not already checked
if browser.mainForm.getListboxControl(line_number=line_number, column_number=1).selected == False:
#if browser.mainForm.getListboxControl(line_number=line_number, column_number=1).selected == False:
if browser.mainForm.getListboxControl(line_number=line_number, column_number=1).value == False:
browser.mainForm.getListboxControl(line_number=line_number, column_number=1).click()
result('Submit '+actionName+' Relations',
browser.mainForm.submit(name='Base_callDialogMethod:method',
......
......@@ -45,8 +45,10 @@ class PerformanceTester(object):
if not namespace:
self._argument_namespace = self._parse_arguments(argparse.ArgumentParser(
description='Run ERP5 benchmarking suites.'))
#print "INIT method. Arguments: " + str(self._argument_namespace)
else:
self._argument_namespace = namespace
#print "INIT method. Arguments: " + str(self._argument_namespace)
@staticmethod
def _add_parser_arguments(parser):
......@@ -166,15 +168,14 @@ class PerformanceTester(object):
namespace.user_tuple = ArgumentType.objectFromModule(namespace.user_info_filename,
object_name='user_tuple',
searchable_path_list=users_file_path_list)
print "USER TUPLE: " + str(namespace.user_tuple)
namespace.benchmark_suite_list = namespace.benchmark_suite_list[0].split(" ")
object_benchmark_suite_list = []
for benchmark_suite in namespace.benchmark_suite_list:
object_benchmark_suite_list.append(ArgumentType.objectFromModule(benchmark_suite,
callable_object=True,
searchable_path_list=namespace.benchmark_path_list))
if namespace.repeat > 0:
namespace.max_error_number = \
min(len(namespace.benchmark_suite_list) * namespace.repeat,
......@@ -238,6 +239,7 @@ class PerformanceTester(object):
result_class = self.getResultClass()
for user_index in range(nb_users):
print "[PERFORMANCE TESTER] Creating benchmark process for user: " + str(user_index)
process = BenchmarkProcess(exit_msg_queue, result_class,
self._argument_namespace, nb_users,
user_index,
......@@ -289,6 +291,7 @@ class PerformanceTester(object):
return (error_message_set, exit_status)
def run(self):
print "[PERFORMANCE TESTER] run method"
error_message_set, exit_status = set(), 0
self.preRun()
......
......@@ -79,6 +79,7 @@ class BenchmarkProcess(multiprocessing.Process):
def runBenchmarkSuiteList(self, result):
for target_idx, target in enumerate(self._argument_namespace.benchmark_suite_list):
self._logger.debug("EXECUTE: %s" % target)
print "[PROCESS] EXECUTE: %s" % target
result.enterSuite(target.__name__)
with_error = False
......@@ -115,7 +116,9 @@ class BenchmarkProcess(multiprocessing.Process):
# Clear the Browser history (which keeps (request, response))
# otherwise it will consume a lot of memory after some time. Also it
# does make sense to keep it as suites are independent of each other
self._browser.mech_browser.clear_history()
#self._browser.mech_browser.clear_history()
self._browser._history.clear()
print "Browser memory cleaned"
result.exitSuite(with_error)
......@@ -127,6 +130,7 @@ class BenchmarkProcess(multiprocessing.Process):
result.iterationFinished()
def run(self):
print "[PROCESS] run method"
result_instance = self._result_klass(self._argument_namespace,
self._nb_users,
self._user_index,
......@@ -147,10 +151,13 @@ class BenchmarkProcess(multiprocessing.Process):
socket.socket = _patched_socket
print "[PROCESS] (signal.SIGTERM, self.stopGracefully)"
# Ensure the data are flushed before exiting, handled by Result class
# __exit__ block
signal.signal(signal.SIGTERM, self.stopGracefully)
print "[PROCESS] Ignore KeyboardInterrupt"
# Ignore KeyboardInterrupt as it is handled by the parent process
signal.signal(signal.SIGINT, signal.SIG_IGN)
......
File mode changed from 100755 to 100644
......@@ -80,10 +80,14 @@ class BenchmarkResultStatistic(object):
@property
def mean(self):
if self.n == 0:
self.n = 1
return self._value_sum / self.n
@property
def standard_deviation(self):
if self.n == 0:
self.n = 1
return math.sqrt(self._variance_sum / self.n)
class NothingFlushedException(Exception):
......
......@@ -124,12 +124,16 @@ def getCreatedDocumentNumberFromERP5(erp5_url, log):
# XXX: This import is required, just to populate sys.modules['test_suite'].
# Even if it's not used in this file. Yuck.
import product.ERP5Type.tests.ERP5TypeTestSuite
# ROQUE: can't find this module. If I add an egg 'product', it also fails
#import product.ERP5Type.tests.ERP5TypeTestSuite
# XXX: dirty hack until product is property imported
import sys
sys.path.append('/opt/slapgrid/7031c818b335cf4caf0052ae845689d0/parts/erp5/product/ERP5Type/tests/')
import ERP5TypeTestSuite
from subprocess import call
LOG_FILE_PREFIX = "performance_tester_erp5"
LOG_FILE_PREFIX = "scalability_tester_erp5"
# Duration of a test case
TEST_CASE_DURATION = 60
# Maximum limit of documents to create during a test case
......@@ -145,6 +149,14 @@ def doNothing(**kwargs):
pass
def makeSuite(test_suite=None, log=doNothing, **kwargs):
### ROQUE hardcoded test suite for debug
import imp
module = imp.load_source('tests', '/opt/slapgrid/7031c818b335cf4caf0052ae845689d0/parts/erp5/tests/__init__.py')
suite = module.ERP5_scalability(max_instance_count=1, **kwargs)
return suite
# BBB tests (plural form) is only checked for backward compatibility
for k in sys.modules.keys():
if k in ('tests', 'test',) or k.startswith('tests.') or k.startswith('test.'):
......@@ -153,6 +165,9 @@ def makeSuite(test_suite=None, log=doNothing, **kwargs):
while True:
module_name, class_name = ('%s.%s' % (singular_succeed and 'test' or 'tests',
test_suite)).rsplit('.', 1)
print "module name and class name:"
print module_name
print class_name
try:
suite_class = getattr(__import__(module_name, None, None, [class_name]),
class_name)
......@@ -165,7 +180,6 @@ def makeSuite(test_suite=None, log=doNothing, **kwargs):
suite = suite_class(max_instance_count=1, **kwargs)
return suite
class ScalabilityLauncher(object):
def __init__(self):
# Parse arguments
......@@ -187,9 +201,16 @@ class ScalabilityLauncher(object):
logger.addHandler(file_handler)
self.log = logger.info
# ROQUE testing if this proxy works
proxy = taskdistribution.ServerProxy(
self.__argumentNamespace.test_suite_master_url,
allow_none=True
).portal_task_distribution
# Proxy to with erp5 master test_result
self.test_result = taskdistribution.TestResultProxy(
self.__argumentNamespace.test_suite_master_url,
proxy, #self.__argumentNamespace.test_suite_master_url,
1.0, DummyLogger(self.log),
self.__argumentNamespace.test_result_path,
self.__argumentNamespace.node_title,
......@@ -265,7 +286,10 @@ class ScalabilityLauncher(object):
Return a ScalabilityTest with current running test case informations,
or None if no test_case ready
"""
data = self.test_result.getRunningTestCase()
# ROQUE hardcoded due to error: test_result is not referencing master test_result module
#data = self.test_result.getRunningTestCase() # this runs in master
data = json.dumps({"relative_path": "test_result_module/9667/2",
"title": "0", "count" : 1, })
if not data:
return None
decoded_data = Utils.deunicodeData(json.loads(data))
......@@ -285,6 +309,7 @@ class ScalabilityLauncher(object):
# Get suite informations
suite = makeSuite(self.__argumentNamespace.test_suite, self.log)
test_suite_list = suite.getTestList()
test_suite_list = ['createPerson']
# Main loop
while True:
......@@ -297,6 +322,7 @@ class ScalabilityLauncher(object):
self.log("Test Case %s going to be run." %(current_test.title))
# Prepare configuration
### ROQUE hardcoded test suite for debug
current_test_number = int(current_test.title)
test_duration = suite.getTestDuration(current_test_number)
benchmarks_path = os.path.join(self.__argumentNamespace.erp5_location, suite.getTestPath())
......@@ -304,8 +330,8 @@ class ScalabilityLauncher(object):
user_file_path = os.path.split(user_file_full_path)[0]
user_file = os.path.split(user_file_full_path)[1]
tester_path = self.__argumentNamespace.runner_path
user_number = suite.getUserNumber(current_test_number)
repetition = suite.getTestRepetition(current_test_number)
user_number = 1 #suite.getUserNumber(current_test_number)
repetition = 1 #suite.getTestRepetition(current_test_number)
self.log("user_number: %s" %str(user_number))
self.log("test_duration: %s seconds" %str(test_duration))
......@@ -319,9 +345,10 @@ class ScalabilityLauncher(object):
# Get the number of documents present before running the test.
waitFor0PendingActivities(self.__argumentNamespace.erp5_url, self.log)
previous_document_number = getCreatedDocumentNumberFromERP5(self.__argumentNamespace.erp5_url, self.log)
self.log("previous_document_number: %d" %previous_document_number)
# ROQUE hardcoded. Don't know what is this method. /erp5/count_docs_scalability ????
#previous_document_number = getCreatedDocumentNumberFromERP5(self.__argumentNamespace.erp5_url, self.log)
#self.log("previous_document_number: %d" %previous_document_number)
self.log("previous_document_number:---")
# Generate commands to run
command_list = []
user_index = 0
......@@ -333,11 +360,11 @@ class ScalabilityLauncher(object):
'--benchmark-path-list', benchmarks_path,
'--users-file-path', user_file_path,
'--users-file', user_file,
'--filename-prefix', "%s_%s_repetition%d" %(LOG_FILE_PREFIX, current_test.title, i),
#'--filename-prefix', "%s_%s_repetition%d" %(LOG_FILE_PREFIX, current_test.title, i),
'--report-directory', self.__argumentNamespace.log_path,
'--repeat', "%s" %str(MAX_DOCUMENTS),
'--repeat', "%s" %str(1), #(MAX_DOCUMENTS),
'--max-errors', str(1000000),
'--user-index', str(user_index),
#'--user-index', str(user_index),
])
user_index += user_number/len(test_suite_list)
......@@ -348,6 +375,7 @@ class ScalabilityLauncher(object):
tester_process_list.append(subprocess.Popen(command))
# Sleep
#test_duration = 5
time.sleep(test_duration)
# Stop
......@@ -358,14 +386,16 @@ class ScalabilityLauncher(object):
# Count created documents
# Wait for 0 pending activities before counting
waitFor0PendingActivities(self.__argumentNamespace.erp5_url, self.log)
current_document_number = getCreatedDocumentNumberFromERP5(self.__argumentNamespace.erp5_url, self.log)
created_document_number = current_document_number - previous_document_number
self.log("previous_document_number: %d" %previous_document_number)
self.log("current_document_number: %d" %current_document_number)
self.log("created_document_number: %d" %created_document_number)
document_number.append(created_document_number)
# ROQUE hardcoded. Dont know what is this method. /erp5/count_docs_scalability ????
#current_document_number = getCreatedDocumentNumberFromERP5(self.__argumentNamespace.erp5_url, self.log)
#created_document_number = current_document_number - previous_document_number
#self.log("previous_document_number: %d" %previous_document_number)
#self.log("current_document_number: %d" %current_document_number)
#self.log("created_document_number: %d" %created_document_number)
self.log("commented lines of document number 'count_docs_scalability'")
#document_number.append(created_document_number)
# Move csv/logs
self.moveLogs(current_test.title)
#self.moveLogs(current_test.title)
self.log("Test Case %s is finish" %(current_test.title))
......@@ -399,10 +429,13 @@ class ScalabilityLauncher(object):
'_'.join(test_suite_list)
)
self.log("Results: %s" %results)
test_result_line_test.stop(stdout=results,
test_count=len(test_suite_list),
duration=test_duration)
#test_result_line_test.stop(stdout=results,
# test_count=len(test_suite_list),
# duration=test_duration)
# ROQUE commented because it tries to do "_logger.warning" and self.log is already a function
self.log("Test Case Stopped")
print "SLEEPING FOR 100 (in while TRUE)"
time.sleep(100)
#
error_message_set = None
......
......@@ -9,7 +9,7 @@
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# consequences resulting from its eventual inadequacies and bugsc
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
......@@ -36,7 +36,7 @@ import urllib
from urlparse import urljoin
from z3c.etestbrowser.browser import ExtendedTestBrowser
from zope.testbrowser.browser import onlyOne, fix_exception_name
from zope.testbrowser.browser import onlyOne #, fix_exception_name
def measurementMetaClass(prefix):
"""
......@@ -234,7 +234,7 @@ class Browser(ExtendedTestBrowser):
try:
response = self.mech_browser.open_novisit(url_or_path, data)
except Exception, e:
fix_exception_name(e)
#fix_exception_name(e)
raise
except mechanize.HTTPError, e:
if e.code >= 200 and e.code <= 299:
......@@ -313,21 +313,22 @@ class Browser(ExtendedTestBrowser):
@todo: Patch zope.testbrowser to allow the class to be given
rather than duplicating the code
"""
print "MAINFORM PROPERTY ASKED"
# If the page has not changed, no need to re-create a class, so
# just return the main_form instance
if self._main_form and self._counter == self._main_form._browser_counter:
print "return last MAINFORM"
return self._main_form
main_form = None
for form in self.mech_browser.forms():
if form.attrs.get('id') == 'main_form':
main_form = form
#for form in self.mech_browser.forms():
# if form.attrs.get('id') == 'main_form':
# main_form = form
main_form = self.getForm(id='main_form')._form
if not main_form:
raise LookupError("Could not get 'main_form'")
self.mech_browser.form = form
self._main_form = ContextMainForm(self, form)
#self.mech_browser.form = form
self._main_form = ContextMainForm(self, main_form)
print "return new ContextMainForm"
return self._main_form
def getLink(self, text=None, url=None, id=None, index=0,
......@@ -682,6 +683,8 @@ class MainForm(Form):
class attribute name, if class_attribute
parameter is given.
"""
print "SUBMIT METHOD"
print "Submitting (name='%s', label='%s', class='%s')" % (name, label, class_attribute)
self._logger.debug(
"Submitting (name='%s', label='%s', class='%s')" % (name, label,
class_attribute))
......@@ -700,14 +703,34 @@ class MainForm(Form):
class_attribute)
if label is None and name is None:
#self._form.submit(label=label, name=name, *args, **kwargs)
super(MainForm, self).submit(label=label, name=name, *args, **kwargs)
else:
if index is None:
index = 0
super(MainForm, self).submit(label=label, name=name, index=index,
*args, **kwargs)
print "Browser.url"
print str(self.browser.url)
print "Browser.title"
print str(self.browser.title)
#print "Browser.contents"
#print str(self.browser.contents)
print "Browser.headers"
print str(self.browser.headers)
print "Browser.cookies"
print str(self.browser.cookies)
print "len(browser.cookies)"
print len(self.browser.cookies)
print "Browser.cookies.url"
print str(self.browser.cookies.url)
print "Browser..cookies.keys()"
print str(self.browser.cookies.keys())
def submitSelect(self, select_name, submit_name, label=None, value=None,
select_index=None, control_index=None):
"""
......@@ -747,8 +770,15 @@ class MainForm(Form):
@raise LookupError: The select, option or submit control could not
be found
"""
print "SUBMIT SELECT ASKED"
print "select_name: %s " % select_name
print "submit_name: %s " % submit_name
print "label: %s " % label
print "value: %s " % value
#form = self.mainForm._form
#form = self._form
#select_control = form.getControl(name=select_name, index=select_index)
select_control = self.getControl(name=select_name, index=select_index)
# zope.testbrowser checks for a whole word but it is also useful
# to match the end of the option control value string because in
# ERP5, the value could be URL (such as 'http://foo:81/erp5/logout')
......@@ -760,13 +790,15 @@ class MainForm(Form):
if item.endswith(value):
value = item
break
print "select_id='%s', label='%s', value='%s'" % \
(select_name, label, value)
self._logger.debug("select_id='%s', label='%s', value='%s'" % \
(select_name, label, value))
control = select_control.getControl(label=label, value=value,
index=control_index)
select_control.getControl(label=label, value=value,
index=control_index).selected = True
print "CONTROL %s FOUND AND SELECTED" % (control.value)
self.submit(name=submit_name)
def submitLogin(self):
......@@ -783,13 +815,18 @@ class MainForm(Form):
@todo: Use information sent back as headers rather than looking
into the page content?
"""
print "SUBMIT LOGIN ASKED"
check_logged_in_xpath = '//div[@id="logged_in_as"]/*'
if self.etree.xpath(check_logged_in_xpath):
print "LOGIN: Already logged in"
self._logger.debug("Already logged in")
# TODO: Perhaps zope.testbrowser should be patched instead?
self.browser.timer.start_time = self.browser.timer.end_time = 0
return
print "Logging in: username='%s', password='%s'" % \
(self.browser._username, self.browser._password)
self._logger.debug("Logging in: username='%s', password='%s'" % \
(self.browser._username, self.browser._password))
......@@ -797,6 +834,7 @@ class MainForm(Form):
form.getControl(name='__ac_name').value = self.browser._username
form.getControl(name='__ac_password').value = self.browser._password
form.submit()
print "LOGGED IN "
try:
login(self)
......@@ -810,6 +848,15 @@ class MainForm(Form):
self.browser._username,
self.browser._password))
print "SETTING COOKIES AFTER LOGIN"
import Cookie
cookie = Cookie.SimpleCookie()
cookie.load(self.browser.headers['set-cookie'])
ac_value = cookie['__ac'].value
self.browser.cookies["__ac"] = ac_value
print "BROWSER COOKIES:"
print self.browser.cookies
def submitSelectFavourite(self, label=None, value=None, **kw):
"""
Select and submit a favourite, given either by its label (such as
......@@ -949,6 +996,9 @@ class ContextMainForm(MainForm):
"""
Create a new object.
"""
print "SUBMIT NEW ASKED"
#self._form.submit(name='Folder_create:method')
#self.mainForm._form.submit(name='Folder_create:method')
self.submit(name='Folder_create:method')
def submitClone(self):
......@@ -1187,7 +1237,8 @@ class ContextMainForm(MainForm):
# control), then get the item from its value
if isinstance(control, ListControl):
control = control.getControl(value=input_element.get('value'))
print "CONTROL TYPE: "
print str(type(control))
return control
from zope.testbrowser.browser import SubmitControl
......
......@@ -85,7 +85,7 @@ class ScalabilityTestRunner():
self.log("SlapOS Master hateoas url is: %s" %self.slapos_api_rest_url)
self.key_path, self.cert_path, config_path = self.slapos_controler.createSlaposConfigurationFileAccount(
key, certificate, self.slapos_url, self.testnode.config, self.slapos_api_rest_url)
key, certificate, self.slapos_url, self.testnode.config)
self.slapos_communicator = None
# Dict containing used to store which SR is not yet correctly installed.
# looks like: {'comp_id1':'SR_urlA', 'comp_id2':'SR_urlA',..}
......@@ -150,7 +150,6 @@ class ScalabilityTestRunner():
config = self._generateInstanceXML(software_configuration,
test_result, test_suite)
request_kw = {"partition_parameter_kw": {"_" : json.dumps(config)} }
#self.log("request_kw: " + str(request_kw)) # kept for DEBUG
self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STARTED, instance_title, request_kw)
self.authorize_request = False
return {'status_code' : 0}
......@@ -235,13 +234,13 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
raise ValueError(error_message)
self.log("Instance correctly '%s' after %s seconds." %(state, str(time.time()-start_time)))
def _waitInstanceCreation(self, instance_title, hateoas, max_time=MAX_CREATION_INSTANCE_TIME):
def _waitInstanceCreation(self, instance_title, max_time=MAX_CREATION_INSTANCE_TIME):
"""
Wait for 'max_time' the instance creation
"""
self.log("Waiting for instance creation...")
start_time = time.time()
while (not instance_title in hateoas.getHostingSubscriptionDict() \
while (not self.slapos_communicator.isInstanceRequested(instance_title) \
and (max_time > (time.time()-start_time)) ):
self.log("Instance not ready yet. Sleeping 5 sec.")
time.sleep(5)
......@@ -390,7 +389,7 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
except:
self.log("Unable to launch instance")
raise ValueError("Unable to launch instance")
self._waitInstanceCreation(self.instance_title, hateoas)
self._waitInstanceCreation(self.instance_title)
return {'status_code' : 0}
return {'status_code' : 1}
......
......@@ -78,7 +78,7 @@ class SlapOSControler(object):
#TODO: implement a method to get all instance related the slapOS account
# and deleting all old instances (based on creation date or name etc...)
def createSlaposConfigurationFileAccount(self, key, certificate, slapos_url, config, slapos_rest_url):
def createSlaposConfigurationFileAccount(self, key, certificate, slapos_url, config):
# Create "slapos_account" directory in the "slapos_directory"
slapos_account_directory = os.path.join(config['slapos_directory'], "slapos_account")
createFolder(slapos_account_directory)
......@@ -86,10 +86,9 @@ class SlapOSControler(object):
slapos_account_key_path = os.path.join(slapos_account_directory, "key")
slapos_account_certificate_path = os.path.join(slapos_account_directory, "certificate")
configuration_file_path = os.path.join(slapos_account_directory, "slapos.cfg")
configuration_file_value = "[slapos]\nmaster_url = %s\nmaster_rest_url = %s\n\
configuration_file_value = "[slapos]\nmaster_url = %s\n\
[slapconsole]\ncert_file = %s\nkey_file = %s" %(
slapos_url,
slapos_rest_url,
slapos_account_certificate_path,
slapos_account_key_path)
createFile(slapos_account_key_path, "w", key)
......
......@@ -114,6 +114,10 @@ class SlapOSMasterCommunicator(object):
state=state,
**self.request_kw)
def isInstanceRequested(self, instance_title):
hateoas = getattr(self.slap, '_hateoas_navigator', None)
return instance_title in hateoas.getHostingSubscriptionDict()
@retryOnNetworkFailure
def _hateoas_getComputer(self, reference):
......@@ -513,4 +517,3 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator):
self.deadline = now + delay
stepfunc(self)
return self.deadline
\ No newline at end of file
......@@ -404,14 +404,11 @@ shared = true
node_test_suite.edit(test_result=test_result)
# get cluster configuration for this test suite, this is needed to
# know slapos parameters to user for creating instances
log("Getting configuration from test suite " + str(node_test_suite.test_suite_title))
generated_config = self.test_suite_portal.generateConfiguration(node_test_suite.test_suite_title)
#log("Generated configuration: " + str(generated_config)) # kept for debug
jsonData = json.loads(generated_config)
cluster_configuration = Utils.deunicodeData(jsonData['configuration_list'][0])
node_test_suite.edit(cluster_configuration=cluster_configuration)
# Now prepare the installation of SlapOS and create instance
status_dict = runner.prepareSlapOSForTestSuite(node_test_suite)
# Give some time so computer partitions may start
......
File mode changed from 100755 to 100644
File mode changed from 100755 to 100644
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment