Commit de75a33d authored by Julien Muchembled's avatar Julien Muchembled

Import erp5.utils.{benchmark,test_browser} from svn.erp5.org:public/erp5/trunk/utils

parents 65d6f5fc 91247aec
......@@ -4,7 +4,12 @@ Changes
0.2 (unreleased)
----------------
* No changes yet.
* Imported from https://svn.erp5.org/repos/public/erp5/trunk/utils/
- erp5.util.test_browser:
Programmable browser for functional and performance tests for ERP5
- erp5.util.benchmark:
Performance benchmarks for ERP5 with erp5.utils.test_browser
0.1 (2011-08-08)
----------------
......
API Documentation
-----------------
You can generate the API documentation using ``epydoc'':
$ epydoc src/erp5
##############################################################################
#
# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
# Arnaud Fontaine <arnaud.fontaine@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import argparse
import functools
class ArgumentType(object):
@classmethod
def directoryType(cls, path):
if not (os.path.isdir(path) and os.access(path, os.W_OK)):
raise argparse.ArgumentTypeError("'%s' is not a valid directory or is "\
"not writable" % path)
return path
@classmethod
def objectFromModule(cls, module_name, object_name=None,
callable_object=False):
if module_name.endswith('.py'):
module_name = module_name[:-3]
if not object_name:
object_name = module_name
import sys
sys.path.append(os.getcwd())
try:
module = __import__(module_name, globals(), locals(), [object_name], -1)
except Exception, e:
raise argparse.ArgumentTypeError("Cannot import '%s.%s': %s" % \
(module_name, object_name, str(e)))
try:
obj = getattr(module, object_name)
except AttributeError:
raise argparse.ArgumentTypeError("Could not get '%s' in '%s'" % \
(object_name, module_name))
if callable_object and not callable(obj):
raise argparse.ArgumentTypeError(
"'%s.%s' is not callable" % (module_name, object_name))
return obj
@classmethod
def strictlyPositiveIntType(cls, value):
try:
converted_value = int(value)
except ValueError:
pass
else:
if converted_value > 0:
return converted_value
raise argparse.ArgumentTypeError('expects a strictly positive integer')
@classmethod
def strictlyPositiveIntOrRangeType(cls, value):
try:
return cls.strictlyPositiveIntType(value)
except argparse.ArgumentTypeError:
try:
min_max_list = value.split(',')
except ValueError:
pass
else:
if len(min_max_list) == 2:
minimum, maximum = cls.strictlyPositiveIntType(min_max_list[0]), \
cls.strictlyPositiveIntType(min_max_list[1])
if minimum >= maximum:
raise argparse.ArgumentTypeError('%d >= %d' % (minimum, maximum))
return (minimum, maximum)
raise argparse.ArgumentTypeError(
'expects either a strictly positive integer or a range of strictly '
'positive integer separated by a comma')
@classmethod
def ERP5UrlType(cls, url):
if url[-1] == '/':
url_list = url.rsplit('/', 2)[:-1]
else:
url_list = url.rsplit('/', 1)
url_list[0] = url_list[0] + '/'
if len(url_list) != 2:
raise argparse.ArgumentTypeError("Invalid URL given")
return url_list
# -*- coding: utf-8 -*-
def createPerson(result, browser):
"""
Create a Person and add a telephone number. It can be ran infinitely (e.g.
until it is interrupted by the end user) with 1 concurrent user, through
performance_tester_erp5 with the following command:
performance_tester_erp5 http://foo.bar:4242/erp5/ 1 createPerson
Please note that you must run this command from the same directory of this
script and userInfo.py. Further information about performance_tester_erp5
options and arguments are available by specifying ``--help''.
"""
# Go to Persons module (person_module)
result('Go to person module',
browser.mainForm.submitSelectModule(value='/person_module'))
# Create a new person and record the time elapsed in seconds
result('Add Person', browser.mainForm.submitNew())
# Check whether it has been successfully created
assert browser.getTransitionMessage() == 'Object created.'
# Fill the first and last name of the newly created person
browser.mainForm.getControl(name='field_my_first_name').value = 'Create'
browser.mainForm.getControl(name='field_my_last_name').value = 'Person'
# Submit the changes, record the time elapsed in seconds
result('Save', browser.mainForm.submitSave())
# Check whether the changes have been successfully updated
assert browser.getTransitionMessage() == 'Data updated.'
person_url = browser.url
# Add phone number
result('Add telephone',
browser.mainForm.submitSelectAction(value='add Telephone'))
# Fill telephone title and number
browser.mainForm.getControl(name='field_my_title'). value = 'Personal'
browser.mainForm.getControl(name='field_my_telephone_number').value = '0123456789'
# Submit the changes, record the time elapsed in seconds
result('Save', browser.mainForm.submitSave())
# Check whether the changes have been successfully updated
assert browser.getTransitionMessage() == 'Data updated.'
# Go back to the Person page before validating
browser.open(person_url)
# Validate it (as the workflow action may not be available yet, try 5 times
# and sleep 5s between each attempts before failing)
show_validate_time, waiting_for_validate_action = \
browser.mainForm.submitSelectWorkflow(value='validate_action',
maximum_attempt_number=5,
sleep_between_attempt=5)
result('Waiting for validate_action', waiting_for_validate_action)
result('Show validate', show_validate_time)
result('Validated', browser.mainForm.submitDialogConfirm())
assert browser.getTransitionMessage() == 'Status changed.'
# Specify user login/password used to run the tests
user_tuple = (('zope', 'zope'),)
This diff is collapsed.
##############################################################################
#
# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
# Arnaud Fontaine <arnaud.fontaine@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import multiprocessing
import csv
import traceback
import os
import logging
import signal
import sys
from ..test_browser.browser import Browser
MAXIMUM_ERROR_COUNTER = 10
RESULT_NUMBER_BEFORE_FLUSHING = 100
class BenchmarkProcess(multiprocessing.Process):
def __init__(self, exit_msg_queue, result_klass, argument_namespace,
nb_users, user_index, *args, **kwargs):
self._exit_msg_queue = exit_msg_queue
self._result_klass = result_klass
self._argument_namespace = argument_namespace
self._nb_users = nb_users
self._user_index = user_index
# Initialized when running the test
self._browser = None
self._current_repeat = 1
# TODO: Per target error counter instead of global one?
self._error_counter = 0
super(BenchmarkProcess, self).__init__(*args, **kwargs)
def stopGracefully(self, *args, **kwargs):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
raise StopIteration("Interrupted by user or because of an error from "
"another process, flushing remaining results...")
def getBrowser(self, log_file):
info_list = tuple(self._argument_namespace.url) + \
tuple(self._argument_namespace.user_tuple[self._user_index])
return Browser(*info_list,
is_debug=self._argument_namespace.enable_debug,
log_file=log_file,
is_legacy_listbox=self._argument_namespace.is_legacy_listbox)
def runBenchmarkSuiteList(self, result):
for target_idx, target in enumerate(self._argument_namespace.benchmark_suite_list):
self._logger.debug("EXECUTE: %s" % target)
result.enterSuite(target.__name__)
try:
self._browser.open()
target(result, self._browser)
except StopIteration:
raise
except Exception, e:
msg = "%s: %s" % (target, traceback.format_exc())
try:
msg += "Last response headers:\n%s\nLast response contents:\n%s" % \
(self._browser.headers, self._browser.contents)
except:
pass
if (self._current_repeat == 1 or
self._error_counter == MAXIMUM_ERROR_COUNTER):
raise RuntimeError(msg)
self._error_counter += 1
self._logger.warning(msg)
for stat in result.getCurrentSuiteStatList():
mean = stat.mean
self._logger.info("%s: min=%.3f, mean=%.3f (+/- %.3f), max=%.3f" % \
(stat.full_label,
stat.minimum,
mean,
stat.standard_deviation,
stat.maximum))
if (self._argument_namespace.max_global_average and
mean > self._argument_namespace.max_global_average):
raise RuntimeError("Stopping as mean is greater than maximum "
"global average")
result.exitSuite()
result.iterationFinished()
def run(self):
result_instance = self._result_klass(self._argument_namespace,
self._nb_users,
self._user_index)
self._logger = result_instance.getLogger()
# Ensure the data are flushed before exiting, handled by Result class
# __exit__ block
signal.signal(signal.SIGTERM, self.stopGracefully)
# Ignore KeyboardInterrupt as it is handled by the parent process
signal.signal(signal.SIGINT, signal.SIG_IGN)
exit_status = 0
exit_msg = None
try:
with result_instance as result:
self._browser = self.getBrowser(result_instance.log_file)
while self._current_repeat != (self._argument_namespace.repeat + 1):
self._logger.info("Iteration: %d" % self._current_repeat)
self.runBenchmarkSuiteList(result)
self._current_repeat += 1
if not self._current_repeat % RESULT_NUMBER_BEFORE_FLUSHING:
result.flush()
except StopIteration, e:
self._logger.error(e)
except BaseException, e:
exit_msg = str(e)
exit_status = 1
self._exit_msg_queue.put(exit_msg)
sys.exit(exit_status)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
# Arnaud Fontaine <arnaud.fontaine@nexedi.com>
#
# First version: ERP5Mechanize from Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import argparse
def parseArguments():
parser = argparse.ArgumentParser(
description='Generate reports for ERP5 benchmarking suites.')
parser.add_argument('--enable-debug',
dest='is_debug',
action='store_true',
default=False,
help='Enable debug messages')
parser.add_argument('--filename-prefix',
default='result',
metavar='PREFIX',
help='Filename prefix for results CSV files '
'(default: result)')
parser.add_argument('--output-filename',
default='results.pdf',
metavar='FILENAME',
help='PDF output file (default: results.pdf)')
parser.add_argument('report_directory',
help='Reports directory')
namespace = parser.parse_args()
return namespace
import csv
from .result import BenchmarkResultStatistic
def computeStatisticFromFilenameList(argument_namespace, filename_list):
reader_list = []
stat_list = []
label_list = []
for filename in filename_list:
reader = csv.reader(open(filename, 'rb'), delimiter=',',
quoting=csv.QUOTE_MINIMAL)
reader_list.append(reader)
# Get headers
row_list = reader.next()
if not label_list:
label_list = row_list
for label in label_list:
stat_list.append(BenchmarkResultStatistic(*label.split(': ', 1)))
if row_list != label_list:
raise AssertionError, "ERROR: Result labels: %s != %s" % \
(label_list, row_list)
for row_list in reader:
for idx, row in enumerate(row_list):
stat_list[idx].add(float(row))
return stat_list
def formatFloatList(value_list):
return [ format(value, ".3f") for value in value_list ]
import numpy
import pylab
from matplotlib import pyplot, ticker
def drawBarDiagram(pdf, title, stat_list):
mean_list = []
yerr_list = []
minimum_list = []
maximum_list = []
label_list = []
error_list = []
for stat in stat_list:
mean_list.append(stat.mean)
yerr_list.append(stat.standard_deviation)
minimum_list.append(stat.minimum)
maximum_list.append(stat.maximum)
label_list.append(stat.label)
error_list.append(stat.error_sum)
min_array = numpy.array(minimum_list)
mean_array = numpy.array(mean_list)
max_array = numpy.array(maximum_list)
yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
## Draw diagrams
# Create the figure
figure = pyplot.figure(figsize=(11.69, 8.29))
figure.subplots_adjust(bottom=0.13, right=0.98, top=0.95)
pyplot.title(title)
# Create the axes along with their labels
axes = figure.add_subplot(111)
axes.set_ylabel('Seconds')
axes.set_xticks([])
axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
axes.yaxis.grid(True, 'major', linewidth=1.5)
axes.yaxis.grid(True, 'minor')
# Create the bars
ind = numpy.arange(len(label_list))
width = 0.33
min_rects = axes.bar(ind, minimum_list, width, color='y', label='Minimum')
avg_rects = axes.bar(ind + width, mean_list, width, color='r', label='Mean')
axes.errorbar(numpy.arange(0.5, len(stat_list)), mean_list,
yerr=[yerr_lower, yerr_upper], fmt=None,
label='Standard deviation')
max_rects = axes.bar(ind + width * 2, maximum_list, width, label='Maximum',
color='g')
# Add the legend of bars
axes.legend(loc=0)
axes.table(rowLabels=['Minimum', 'Average', 'Std. deviation', 'Maximum', 'Errors'],
colLabels=label_list,
cellText=[formatFloatList(minimum_list),
formatFloatList(mean_list),
formatFloatList(yerr_list),
formatFloatList(maximum_list),
error_list],
rowColours=('y', 'r', 'b', 'g', 'w'),
loc='bottom',
colLoc='center',
rowLoc='center',
cellLoc='center')
pdf.savefig()
pylab.close()
def drawConcurrentUsersPlot(pdf, title, nb_users_list, stat_list):
figure = pyplot.figure(figsize=(11.69, 8.29), frameon=False)
figure.subplots_adjust(bottom=0.1, right=0.98, left=0.07, top=0.95)
pyplot.title(title)
pyplot.grid(True, linewidth=1.5)
axes = figure.add_subplot(111)
min_array = numpy.array([stat.minimum for stat in stat_list])
mean_array = numpy.array([stat.mean for stat in stat_list])
max_array = numpy.array([stat.maximum for stat in stat_list])
yerr_list = [stat.standard_deviation for stat in stat_list]
yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
axes.plot(nb_users_list, min_array, 'yo-', label='Minimum')
axes.errorbar(nb_users_list,
mean_array,
yerr=[yerr_lower, yerr_upper],
color='r',
ecolor='b',
label='Mean',
elinewidth=2,
fmt='D-',
capsize=10.0)
axes.plot(nb_users_list, max_array, 'gs-', label='Maximum')
axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
axes.yaxis.grid(True, 'minor')
axes.xaxis.set_major_locator(ticker.FixedLocator(nb_users_list))
axes.set_xticks(nb_users_list)
axes.legend(loc=0)
axes.set_xlabel('Concurrent users')
axes.set_ylabel('Seconds')
pyplot.xlim(xmin=nb_users_list[0])
pdf.savefig()
pylab.close()
from matplotlib.backends.backend_pdf import PdfPages
import glob
import os
import re
user_re = re.compile('-(\d+)users-')
def generateReport():
argument_namespace = parseArguments()
filename_iter = glob.iglob("%s-*repeat*-*users*-*process*.csv" % os.path.join(
argument_namespace.report_directory,
argument_namespace.filename_prefix))
per_nb_users_report_dict = {}
for filename in filename_iter:
report_dict = per_nb_users_report_dict.setdefault(
int(user_re.search(filename).group(1)), {'filename': []})
report_dict['filename'].append(filename)
pdf = PdfPages(argument_namespace.output_filename)
for nb_users, report_dict in per_nb_users_report_dict.items():
stat_list = computeStatisticFromFilenameList(
argument_namespace, report_dict['filename'])
title = "Ran suites with %d users" % len(report_dict['filename'])
for slice_start_idx in range(0, len(stat_list), 12):
if slice_start_idx != 0:
title += ' (Ctd.)'
drawBarDiagram(pdf, title, stat_list[slice_start_idx:slice_start_idx + 12])
report_dict['stats'] = stat_list
if len(per_nb_users_report_dict) != 1:
for i in range(len(report_dict['stats'])):
stat_list = []
nb_users_list = per_nb_users_report_dict.keys()
for report_dict in per_nb_users_report_dict.values():
stat_list.append(report_dict['stats'][i])
drawConcurrentUsersPlot(
pdf,
"%s from %d to %d users (step: %d)" % (stat_list[0].full_label,
nb_users_list[0],
nb_users_list[-1],
nb_users_list[1] - nb_users_list[0]),
nb_users_list,
stat_list)
pdf.close()
if __name__ == '__main__':
generateReport()
##############################################################################
#
# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
# Arnaud Fontaine <arnaud.fontaine@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import sys
import math
import os
import csv
import logging
import signal
class BenchmarkResultStatistic(object):
def __init__(self, suite, label):
self.suite = suite
self.label = label
self.full_label = '%s: %s' % (self.suite, self.label)
self.minimum = sys.maxint
self.maximum = -1
self.n = 0
self.error_sum = 0
# For calculating the mean
self._value_sum = 0
# For calculating the standard deviation
self._variance_sum = 0
self._mean = 0
def add_error(self):
self.error_sum += 1
def add(self, value):
if value < self.minimum:
self.minimum = value
if value > self.maximum:
self.maximum = value
self._value_sum += value
self.n += 1
delta = value - self._mean
self._mean += delta / self.n
self._variance_sum += delta * (value - self._mean)
@property
def mean(self):
return self._value_sum / self.n
@property
def standard_deviation(self):
return math.sqrt(self._variance_sum / self.n)
import abc