Import erp5.utils.{benchmark,test_browser} from svn.erp5.org:public/erp5/trunk/utils

CHANGES.erp5.util.txt

@@ -4,7 +4,12 @@ Changes
 0.2 (unreleased)
 ----------------
 
- * No changes yet.
+ * Imported from https://svn.erp5.org/repos/public/erp5/trunk/utils/
+
+   - erp5.util.test_browser:
+     Programmable browser for functional and performance tests for ERP5
+   - erp5.util.benchmark:
+     Performance benchmarks for ERP5 with erp5.utils.test_browser
 
 0.1 (2011-08-08)
 ----------------

erp5/util/README.test_browser.txt

+API Documentation
+-----------------
+
+You can generate the API documentation using ``epydoc'':
+
+$ epydoc src/erp5

erp5/util/benchmark/argument.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import argparse
+import functools
+
+class ArgumentType(object):
+  @classmethod
+  def directoryType(cls, path):
+    if not (os.path.isdir(path) and os.access(path, os.W_OK)):
+      raise argparse.ArgumentTypeError("'%s' is not a valid directory or is "\
+                                         "not writable" % path)
+
+    return path
+
+  @classmethod
+  def objectFromModule(cls, module_name, object_name=None,
+                       callable_object=False):
+    if module_name.endswith('.py'):
+      module_name = module_name[:-3]
+
+    if not object_name:
+      object_name = module_name
+
+    import sys
+    sys.path.append(os.getcwd())
+
+    try:
+      module = __import__(module_name, globals(), locals(), [object_name], -1)
+    except Exception, e:
+      raise argparse.ArgumentTypeError("Cannot import '%s.%s': %s" % \
+                                         (module_name, object_name, str(e)))
+
+    try:
+      obj = getattr(module, object_name)
+    except AttributeError:
+      raise argparse.ArgumentTypeError("Could not get '%s' in '%s'" % \
+                                         (object_name, module_name))
+
+    if callable_object and not callable(obj):
+      raise argparse.ArgumentTypeError(
+        "'%s.%s' is not callable" % (module_name, object_name))
+
+    return obj
+
+  @classmethod
+  def strictlyPositiveIntType(cls, value):
+    try:
+      converted_value = int(value)
+    except ValueError:
+      pass
+    else:
+      if converted_value > 0:
+        return converted_value
+
+    raise argparse.ArgumentTypeError('expects a strictly positive integer')
+
+  @classmethod
+  def strictlyPositiveIntOrRangeType(cls, value):
+    try:
+      return cls.strictlyPositiveIntType(value)
+    except argparse.ArgumentTypeError:
+      try:
+        min_max_list = value.split(',')
+      except ValueError:
+        pass
+      else:
+        if len(min_max_list) == 2:
+          minimum, maximum = cls.strictlyPositiveIntType(min_max_list[0]), \
+              cls.strictlyPositiveIntType(min_max_list[1])
+
+          if minimum >= maximum:
+            raise argparse.ArgumentTypeError('%d >= %d' % (minimum, maximum))
+
+          return (minimum, maximum)
+
+    raise argparse.ArgumentTypeError(
+      'expects either a strictly positive integer or a range of strictly '
+      'positive integer separated by a comma')
+
+  @classmethod
+  def ERP5UrlType(cls, url):
+    if url[-1] == '/':
+      url_list = url.rsplit('/', 2)[:-1]
+    else:
+      url_list = url.rsplit('/', 1)
+
+    url_list[0] = url_list[0] + '/'
+    if len(url_list) != 2:
+      raise argparse.ArgumentTypeError("Invalid URL given")
+
+    return url_list

erp5/util/benchmark/examples/createPerson.py

+# -*- coding: utf-8 -*-
+
+def createPerson(result, browser):
+  """
+  Create a Person and add a  telephone number.  It can be ran infinitely (e.g.
+  until it  is interrupted by  the end user)  with 1 concurrent  user, through
+  performance_tester_erp5 with the following command:
+
+  performance_tester_erp5 http://foo.bar:4242/erp5/ 1 createPerson
+
+  Please note that  you must run this command from the  same directory of this
+  script  and userInfo.py.  Further information  about performance_tester_erp5
+  options and arguments are available by specifying ``--help''.
+  """
+  # Go to Persons module (person_module)
+  result('Go to person module',
+         browser.mainForm.submitSelectModule(value='/person_module'))
+
+  # Create a new person and record the time elapsed in seconds
+  result('Add Person', browser.mainForm.submitNew())
+
+  # Check whether it has been successfully created
+  assert browser.getTransitionMessage() == 'Object created.'
+
+  # Fill the first and last name of the newly created person
+  browser.mainForm.getControl(name='field_my_first_name').value = 'Create'
+  browser.mainForm.getControl(name='field_my_last_name').value = 'Person'
+
+  # Submit the changes, record the time elapsed in seconds
+  result('Save', browser.mainForm.submitSave())
+
+  # Check whether the changes have been successfully updated
+  assert browser.getTransitionMessage() == 'Data updated.'
+  person_url = browser.url
+
+  # Add phone number
+  result('Add telephone',
+         browser.mainForm.submitSelectAction(value='add Telephone'))
+
+  # Fill telephone title and number
+  browser.mainForm.getControl(name='field_my_title'). value = 'Personal'
+  browser.mainForm.getControl(name='field_my_telephone_number').value = '0123456789'
+
+  # Submit the changes, record the time elapsed in seconds
+  result('Save', browser.mainForm.submitSave())
+
+  # Check whether the changes have been successfully updated
+  assert browser.getTransitionMessage() == 'Data updated.'
+
+  # Go back to the Person page before validating
+  browser.open(person_url)
+
+  # Validate it (as the workflow action may not be available yet, try 5 times
+  # and sleep 5s between each attempts before failing)
+  show_validate_time, waiting_for_validate_action = \
+      browser.mainForm.submitSelectWorkflow(value='validate_action',
+                                            maximum_attempt_number=5,
+                                            sleep_between_attempt=5)
+
+  result('Waiting for validate_action', waiting_for_validate_action)
+  result('Show validate', show_validate_time)
+  result('Validated', browser.mainForm.submitDialogConfirm())
+  assert browser.getTransitionMessage() == 'Status changed.'

erp5/util/benchmark/examples/userInfo.py

+# Specify user login/password used to run the tests
+user_tuple = (('zope', 'zope'),)

erp5/util/benchmark/process.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import multiprocessing
+import csv
+import traceback
+import os
+import logging
+import signal
+import sys
+
+from ..test_browser.browser import Browser
+
+MAXIMUM_ERROR_COUNTER = 10
+RESULT_NUMBER_BEFORE_FLUSHING = 100
+
+class BenchmarkProcess(multiprocessing.Process):
+  def __init__(self, exit_msg_queue, result_klass, argument_namespace,
+               nb_users, user_index, *args, **kwargs):
+    self._exit_msg_queue = exit_msg_queue
+    self._result_klass = result_klass
+    self._argument_namespace = argument_namespace
+    self._nb_users = nb_users
+    self._user_index = user_index
+
+    # Initialized when running the test
+    self._browser = None
+    self._current_repeat = 1
+
+    # TODO: Per target error counter instead of global one?
+    self._error_counter = 0
+
+    super(BenchmarkProcess, self).__init__(*args, **kwargs)
+
+  def stopGracefully(self, *args, **kwargs):
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    raise StopIteration("Interrupted by user or because of an error from "
+                        "another process, flushing remaining results...")
+
+  def getBrowser(self, log_file):
+    info_list = tuple(self._argument_namespace.url) + \
+        tuple(self._argument_namespace.user_tuple[self._user_index])
+
+    return Browser(*info_list,
+                   is_debug=self._argument_namespace.enable_debug,
+                   log_file=log_file,
+                   is_legacy_listbox=self._argument_namespace.is_legacy_listbox)
+
+  def runBenchmarkSuiteList(self, result):
+    for target_idx, target in enumerate(self._argument_namespace.benchmark_suite_list):
+      self._logger.debug("EXECUTE: %s" % target)
+      result.enterSuite(target.__name__)
+
+      try:
+        self._browser.open()
+        target(result, self._browser)
+      except StopIteration:
+        raise
+      except Exception, e:
+        msg = "%s: %s" % (target, traceback.format_exc())
+
+        try:
+          msg += "Last response headers:\n%s\nLast response contents:\n%s" % \
+              (self._browser.headers, self._browser.contents)
+        except:
+          pass
+
+        if (self._current_repeat == 1 or
+            self._error_counter == MAXIMUM_ERROR_COUNTER):
+          raise RuntimeError(msg)
+
+        self._error_counter += 1
+        self._logger.warning(msg)
+
+      for stat in result.getCurrentSuiteStatList():
+        mean = stat.mean
+
+        self._logger.info("%s: min=%.3f, mean=%.3f (+/- %.3f), max=%.3f" % \
+                            (stat.full_label,
+                             stat.minimum,
+                             mean,
+                             stat.standard_deviation,
+                             stat.maximum))
+
+        if (self._argument_namespace.max_global_average and
+            mean > self._argument_namespace.max_global_average):
+          raise RuntimeError("Stopping as mean is greater than maximum "
+                             "global average")
+
+      result.exitSuite()
+
+    result.iterationFinished()
+
+  def run(self):
+    result_instance = self._result_klass(self._argument_namespace,
+                                         self._nb_users,
+                                         self._user_index)
+
+    self._logger = result_instance.getLogger()
+
+    # Ensure the data are flushed before exiting, handled by Result class 
+    # __exit__ block
+    signal.signal(signal.SIGTERM, self.stopGracefully)
+ 
+    # Ignore KeyboardInterrupt as it is handled by the parent process
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    exit_status = 0
+    exit_msg = None
+
+    try:
+      with result_instance as result:
+        self._browser = self.getBrowser(result_instance.log_file)
+
+        while self._current_repeat != (self._argument_namespace.repeat + 1):
+          self._logger.info("Iteration: %d" % self._current_repeat)
+          self.runBenchmarkSuiteList(result)
+          self._current_repeat += 1
+
+          if not self._current_repeat % RESULT_NUMBER_BEFORE_FLUSHING:
+            result.flush()
+
+    except StopIteration, e:
+      self._logger.error(e)
+
+    except BaseException, e:
+      exit_msg = str(e)
+      exit_status = 1
+
+    self._exit_msg_queue.put(exit_msg)
+    sys.exit(exit_status)

erp5/util/benchmark/report.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# First version: ERP5Mechanize from Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+
+def parseArguments():
+  parser = argparse.ArgumentParser(
+    description='Generate reports for ERP5 benchmarking suites.')
+
+  parser.add_argument('--enable-debug',
+                      dest='is_debug',
+                      action='store_true',
+                      default=False,
+                      help='Enable debug messages')
+
+  parser.add_argument('--filename-prefix',
+                      default='result',
+                      metavar='PREFIX',
+                      help='Filename prefix for results CSV files '
+                           '(default: result)')
+
+  parser.add_argument('--output-filename',
+                      default='results.pdf',
+                      metavar='FILENAME',
+                      help='PDF output file (default: results.pdf)')
+
+  parser.add_argument('report_directory',
+                      help='Reports directory')
+
+  namespace = parser.parse_args()
+
+  return namespace
+
+import csv
+
+from .result import BenchmarkResultStatistic
+
+def computeStatisticFromFilenameList(argument_namespace, filename_list):
+  reader_list = []
+  stat_list = []
+  label_list = []
+
+  for filename in filename_list:
+    reader = csv.reader(open(filename, 'rb'), delimiter=',',
+                        quoting=csv.QUOTE_MINIMAL)
+
+    reader_list.append(reader)
+
+    # Get headers
+    row_list = reader.next()
+    if not label_list:
+      label_list = row_list
+      for label in label_list:
+        stat_list.append(BenchmarkResultStatistic(*label.split(': ', 1)))
+
+    if row_list != label_list:
+      raise AssertionError, "ERROR: Result labels: %s != %s" % \
+          (label_list, row_list)
+
+    for row_list in reader:
+      for idx, row in enumerate(row_list):
+        stat_list[idx].add(float(row))
+
+  return stat_list
+
+def formatFloatList(value_list):
+  return [ format(value, ".3f") for value in value_list ]
+
+import numpy
+import pylab
+
+from matplotlib import pyplot, ticker
+
+def drawBarDiagram(pdf, title, stat_list):
+  mean_list = []
+  yerr_list = []
+  minimum_list = []
+  maximum_list = []
+  label_list = []
+  error_list = []
+
+  for stat in stat_list:
+    mean_list.append(stat.mean)
+    yerr_list.append(stat.standard_deviation)
+    minimum_list.append(stat.minimum)
+    maximum_list.append(stat.maximum)
+    label_list.append(stat.label)
+    error_list.append(stat.error_sum)
+
+  min_array = numpy.array(minimum_list)
+  mean_array = numpy.array(mean_list)
+  max_array = numpy.array(maximum_list)
+
+  yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
+  yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
+
+  ## Draw diagrams
+  # Create the figure
+  figure = pyplot.figure(figsize=(11.69, 8.29))
+  figure.subplots_adjust(bottom=0.13, right=0.98, top=0.95)
+  pyplot.title(title)
+
+  # Create the axes along with their labels
+  axes = figure.add_subplot(111)
+  axes.set_ylabel('Seconds')
+  axes.set_xticks([])
+
+  axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
+  axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
+  axes.yaxis.grid(True, 'major', linewidth=1.5)
+  axes.yaxis.grid(True, 'minor')
+ 
+  # Create the bars
+  ind = numpy.arange(len(label_list))
+  width = 0.33
+
+  min_rects = axes.bar(ind, minimum_list, width, color='y', label='Minimum')
+
+  avg_rects = axes.bar(ind + width, mean_list, width, color='r', label='Mean')
+
+  axes.errorbar(numpy.arange(0.5, len(stat_list)), mean_list,
+                yerr=[yerr_lower, yerr_upper], fmt=None,
+                label='Standard deviation')
+
+  max_rects = axes.bar(ind + width * 2, maximum_list, width, label='Maximum',
+                       color='g')
+
+  # Add the legend of bars
+  axes.legend(loc=0)
+
+  axes.table(rowLabels=['Minimum', 'Average', 'Std. deviation', 'Maximum', 'Errors'],
+             colLabels=label_list,
+             cellText=[formatFloatList(minimum_list),
+                       formatFloatList(mean_list),
+                       formatFloatList(yerr_list),
+                       formatFloatList(maximum_list),
+                       error_list],
+             rowColours=('y', 'r', 'b', 'g', 'w'),
+             loc='bottom',
+             colLoc='center',
+             rowLoc='center',
+             cellLoc='center')
+
+  pdf.savefig()
+  pylab.close()
+
+def drawConcurrentUsersPlot(pdf, title, nb_users_list, stat_list):
+  figure = pyplot.figure(figsize=(11.69, 8.29), frameon=False)
+  figure.subplots_adjust(bottom=0.1, right=0.98, left=0.07, top=0.95)
+  pyplot.title(title)
+  pyplot.grid(True, linewidth=1.5)
+
+  axes = figure.add_subplot(111)
+
+  min_array = numpy.array([stat.minimum for stat in stat_list])
+  mean_array = numpy.array([stat.mean for stat in stat_list])
+  max_array = numpy.array([stat.maximum for stat in stat_list])
+
+  yerr_list = [stat.standard_deviation for stat in stat_list]
+  yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
+  yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
+
+  axes.plot(nb_users_list, min_array, 'yo-', label='Minimum')
+
+  axes.errorbar(nb_users_list,
+                mean_array,
+                yerr=[yerr_lower, yerr_upper],
+                color='r',
+                ecolor='b',
+                label='Mean',
+                elinewidth=2,
+                fmt='D-',
+                capsize=10.0)
+
+  axes.plot(nb_users_list, max_array, 'gs-', label='Maximum')
+
+  axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
+  axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
+  axes.yaxis.grid(True, 'minor')
+
+  axes.xaxis.set_major_locator(ticker.FixedLocator(nb_users_list))
+
+  axes.set_xticks(nb_users_list)
+  axes.legend(loc=0)
+  axes.set_xlabel('Concurrent users')
+  axes.set_ylabel('Seconds')
+
+  pyplot.xlim(xmin=nb_users_list[0])
+  pdf.savefig()
+  pylab.close()
+
+from matplotlib.backends.backend_pdf import PdfPages
+
+import glob
+import os
+import re
+
+user_re = re.compile('-(\d+)users-')
+
+def generateReport():
+  argument_namespace = parseArguments()
+
+  filename_iter = glob.iglob("%s-*repeat*-*users*-*process*.csv" % os.path.join(
+      argument_namespace.report_directory,
+      argument_namespace.filename_prefix))
+
+  per_nb_users_report_dict = {}
+  for filename in filename_iter:
+    report_dict = per_nb_users_report_dict.setdefault(
+      int(user_re.search(filename).group(1)), {'filename': []})
+
+    report_dict['filename'].append(filename)
+
+  pdf = PdfPages(argument_namespace.output_filename)
+
+  for nb_users, report_dict in per_nb_users_report_dict.items():
+    stat_list = computeStatisticFromFilenameList(
+      argument_namespace, report_dict['filename'])
+
+    title = "Ran suites with %d users" % len(report_dict['filename'])
+    for slice_start_idx in range(0, len(stat_list), 12):
+      if slice_start_idx != 0:
+        title += ' (Ctd.)'
+
+      drawBarDiagram(pdf, title, stat_list[slice_start_idx:slice_start_idx + 12])
+
+    report_dict['stats'] = stat_list
+
+  if len(per_nb_users_report_dict) != 1:
+    for i in range(len(report_dict['stats'])):
+      stat_list = []
+      nb_users_list = per_nb_users_report_dict.keys()
+      for report_dict in per_nb_users_report_dict.values():
+        stat_list.append(report_dict['stats'][i])
+
+      drawConcurrentUsersPlot(
+        pdf,
+        "%s from %d to %d users (step: %d)" % (stat_list[0].full_label,
+                                               nb_users_list[0],
+                                               nb_users_list[-1],
+                                               nb_users_list[1] - nb_users_list[0]),
+        nb_users_list,
+        stat_list)
+
+  pdf.close()
+
+if __name__ == '__main__':
+  generateReport()

erp5/util/benchmark/result.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import sys
+import math
+import os
+import csv
+import logging
+import signal
+
+class BenchmarkResultStatistic(object):
+  def __init__(self, suite, label):
+    self.suite = suite
+    self.label = label
+
+    self.full_label = '%s: %s' % (self.suite, self.label)
+
+    self.minimum = sys.maxint
+    self.maximum = -1
+    self.n = 0
+    self.error_sum = 0
+
+    # For calculating the mean
+    self._value_sum = 0
+
+    # For calculating the standard deviation
+    self._variance_sum = 0
+    self._mean = 0
+
+  def add_error(self):
+    self.error_sum += 1
+
+  def add(self, value):
+    if value < self.minimum:
+      self.minimum = value
+    if value > self.maximum:
+      self.maximum = value
+
+    self._value_sum += value
+    self.n += 1
+
+    delta = value - self._mean
+    self._mean += delta / self.n
+    self._variance_sum += delta * (value - self._mean)
+
+  @property
+  def mean(self):
+    return self._value_sum / self.n
+
+  @property
+  def standard_deviation(self):
+    return math.sqrt(self._variance_sum / self.n)
+
+import abc
+
+class BenchmarkResult(object):
+  __metaclass__ = abc.ABCMeta
+
+  def __init__(self, argument_namespace, nb_users, user_index):
+    self._argument_namespace = argument_namespace
+    self._nb_users = nb_users
+    self._user_index = user_index
+
+    self._log_level = self._argument_namespace.enable_debug and \
+        logging.DEBUG or logging.INFO
+
+    self._stat_list = []
+    self._suite_idx = 0
+    self._result_idx = 0
+    self.result_list = []
+    self._all_result_list = []
+    self._first_iteration = True
+    self._current_suite_name = None
+    self._result_idx_checkpoint_list = []
+    self.label_list = []
+    self._logger = None
+
+  def getLogger(self):
+    if not self._logger:
+      logging.basicConfig(stream=self.log_file, level=self._log_level)
+      self._logger = logging.getLogger('erp5.util.benchmark')
+      return self._logger
+
+    return self._logger
+
+  def __enter__(self):
+    return self
+
+  def enterSuite(self, name):
+    self._current_suite_name = name
+
+  def __call__(self, label, value):
+    self.result_list.append(value)
+    if self._first_iteration:
+      self._stat_list.append(BenchmarkResultStatistic(self._current_suite_name,
+                                                      label))
+
+    self._stat_list[self._result_idx].add(value)
+    self._result_idx += 1
+
+  def getLabelList(self):
+    return [ stat.full_label for stat in self._stat_list ]
+
+  def iterationFinished(self):
+    self._all_result_list.append(self.result_list)
+    if self._first_iteration:
+      self.label_list = self.getLabelList()
+
+    self.getLogger().debug("RESULTS: %s" % self.result_list)
+    self.result_list = []
+    self._first_iteration = False
+    self._suite_idx = 0
+    self._result_idx = 0
+
+  def getStatList(self):
+    return self._stat_list
+
+  def getCurrentSuiteStatList(self):
+    start_index = self._suite_idx and \
+        self._result_idx_checkpoint_list[self._suite_idx - 1] or 0
+
+    return self._stat_list[start_index:self._result_idx]
+
+  def exitSuite(self):
+    if self._first_iteration:
+      self._result_idx_checkpoint_list.append(self._result_idx)
+    else:
+      expected_result_idx = self._result_idx_checkpoint_list[self._suite_idx]
+      while self._result_idx != expected_result_idx:
+        self.result_list.append(0)
+        self._stat_list[self._result_idx].add_error()
+        self._result_idx += 1
+
+    self._suite_idx += 1
+
+  @abc.abstractmethod
+  def flush(self, partial=True):
+    self._all_result_list = []
+
+  @abc.abstractmethod
+  def __exit__(self, exc_type, exc_value, traceback):
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    self.flush(partial=False)
+    return True
+
+class CSVBenchmarkResult(BenchmarkResult):
+  def __init__(self, *args, **kwargs):
+    super(CSVBenchmarkResult, self).__init__(*args, **kwargs)
+
+    filename_prefix = self._getFilenamePrefix()
+
+    self._result_filename = "%s.csv" % filename_prefix
+    self._result_filename_path = os.path.join(
+      self._argument_namespace.report_directory, self._result_filename)
+
+    self._log_filename = "%s.log" % filename_prefix
+    self._log_filename_path = os.path.join(
+      self._argument_namespace.report_directory, self._log_filename)
+
+    self.log_file = open(self._log_filename_path, 'w')
+
+  def _getFilenamePrefix(self):
+    max_nb_users = isinstance(self._argument_namespace.users, int) and \
+        self._argument_namespace.users or self._argument_namespace.users[1]
+
+    fmt = "%%s-%%drepeat-%%0%ddusers-process%%0%dd" % \
+        (len(str(max_nb_users)), len(str(self._nb_users)))
+
+    return fmt % (self._argument_namespace.filename_prefix,
+                  self._argument_namespace.repeat,
+                  self._nb_users,
+                  self._user_index)
+
+  def __enter__(self):
+    self._result_file = open(self._result_filename_path, 'wb')
+    self._csv_writer = csv.writer(self._result_file, delimiter=',',
+                                  quoting=csv.QUOTE_MINIMAL)
+
+    return self
+
+  def flush(self, partial=True):
+    if self._result_file.tell() == 0:
+      self._csv_writer.writerow(self.label_list)
+
+    self._csv_writer.writerows(self._all_result_list)
+    self._result_file.flush()
+    os.fsync(self._result_file.fileno())
+
+    super(CSVBenchmarkResult, self).flush(partial)
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    super(CSVBenchmarkResult, self).__exit__(exc_type, exc_value, traceback)
+    self._result_file.close()
+
+    if exc_type and not issubclass(exc_type, StopIteration):
+      msg = "An error occured, see: %s" % self._log_filename_path
+      self.getLogger().error("%s: %s" % (exc_type, exc_value))
+      raise RuntimeError(msg)
+
+from cStringIO import StringIO
+
+import xmlrpclib
+import datetime
+
+class ERP5BenchmarkResult(BenchmarkResult):
+  def __init__(self, *args, **kwargs):
+    self.log_file = StringIO()
+    self._log_buffer_list = []
+
+    super(ERP5BenchmarkResult, self).__init__(*args, **kwargs)
+
+  def iterationFinished(self):
+    super(ERP5BenchmarkResult, self).iterationFinished()
+
+    # TODO: garbage?
+    self._log_buffer_list.append(self.log_file.getvalue())
+    self.log_file.seek(0)
+
+  def flush(self, partial=True):
+    benchmark_result = xmlrpclib.ServerProxy(
+      self._argument_namespace.erp5_publish_url,
+      verbose=True,
+      allow_none=True)
+
+    benchmark_result.BenchmarkResult_addResultLineList(
+      self._argument_namespace.user_tuple[self._user_index][0],
+      self._argument_namespace.repeat,
+      self._nb_users,
+      self._argument_namespace.benchmark_suite_name_list,
+      self.getLabelList(),
+      self._all_result_list,
+      self._log_buffer_list)
+
+    super(ERP5BenchmarkResult, self).flush()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    super(ERP5BenchmarkResult, self).__exit__(exc_type, exc_value, traceback)
+
+  @staticmethod
+  def createResultDocument(publish_url, publish_project, repeat, nb_users):
+    test_result_module = xmlrpclib.ServerProxy(publish_url,
+                                               verbose=True,
+                                               allow_none=True)
+
+    if isinstance(nb_users, tuple):
+      nb_users_str = '%d to %d' % nb_users
+    else:
+      nb_users_str = '%d' % nb_users
+
+    benchmark_result = test_result_module.TestResultModule_addBenchmarkResult(
+      '%d repeat with %s concurrent users' % (repeat, nb_users_str),
+      publish_project, ' '.join(sys.argv), datetime.datetime.now())
+
+    return benchmark_result['id']
+
+  @staticmethod
+  def closeResultDocument(publish_document_url, error_message_set):
+    result = xmlrpclib.ServerProxy(publish_document_url,
+                                   verbose=True,
+                                   allow_none=True)
+
+    result.BenchmarkResult_completed(error_message_set and 'FAIL' or 'PASS',
+                                     error_message_set)

erp5/util/benchmark/scalability_tester.py

+#!/usr/bin/env python
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from .result import CSVBenchmarkResult
+
+class CSVScalabilityBenchmarkResult(CSVBenchmarkResult):
+  def flush(self, partial=True):
+    super(CSVScalabilityBenchmarkResult, self).flush(partial)
+    self._argument_namespace.notify_method(self._result_filename,
+                                           self._result_file.tell(),
+                                           partial=partial)
+
+from .performance_tester import PerformanceTester
+
+class ScalabilityTester(PerformanceTester):
+  def preRun(self, *args, **kwargs):
+    pass
+
+  def postRun(self, error_message_set):
+    from logging import Formatter
+    import sys
+    import urllib
+    import urllib2
+
+    try:
+      urllib2.urlopen("http://[%s]:%d/report" % \
+                        (self._argument_namespace.manager_address,
+                         self._argument_namespace.manager_port),
+                      urllib.urlencode({'error_message_set': '|'.join(error_message_set)})).close()
+
+    except:
+      print >>sys.stderr, "ERROR: %s" % Formatter().formatException(sys.exc_info())
+
+  def getResultClass(self):
+    if not self._argument_namespace.erp5_publish_url:
+      return CSVScalabilityBenchmarkResult
+
+    return super(ScalabilityTester, self).getResultClass()
+
+from slapos.tool.nosqltester import NoSQLTester
+
+class RunScalabilityTester(NoSQLTester):
+  def __init__(self):
+    super(RunScalabilityTester, self).__init__()
+
+  def _add_parser_arguments(self, parser):
+    super(RunScalabilityTester, self)._add_parser_arguments(parser)
+    ScalabilityTester._add_parser_arguments(parser)
+
+  def _parse_arguments(self, parser):
+    namespace = super(RunScalabilityTester, self)._parse_arguments(parser)
+    ScalabilityTester._check_parsed_arguments(namespace)
+    namespace.notify_method = self.send_result_availability_notification
+    return namespace
+
+  def run_tester(self):
+    ScalabilityTester(self.argument_namespace).run()
+
+def main():
+  RunScalabilityTester().run()
+
+if __name__ == '__main__':
+  main()

erp5/util/test_browser/examples/testAddPerson.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from erp5.util.test_browser.browser import Browser
+
+ITERATION = 20
+
+def benchmarkAddPerson(iteration_counter, result_dict):
+  """
+  Benchmark adding a person.
+  """
+  # Create a browser instance
+  browser = Browser('http://localhost:18080/', 'erp5',
+                    username='zope', password='zope')
+
+  # Open ERP5 homepage
+  browser.open()
+
+  # Go to Persons module (person_module)
+  browser.mainForm.submitSelectModule(value='/person_module')
+
+  # Create a new person and record the time elapsed in seconds
+  result_dict.setdefault('Create', []).append(
+    browser.mainForm.submitNew())
+
+  # Check whether it has been successfully created
+  assert browser.getTransitionMessage() == 'Object created.'
+
+  # Fill the first and last name of the newly created person
+  browser.mainForm.getControl(name='field_my_first_name').value = 'Foo%d' % \
+      iteration_counter
+
+  browser.mainForm.getControl(name='field_my_last_name').value = 'Bar%d' % \
+      iteration_counter
+
+  # Submit the changes, record the time elapsed in seconds
+  result_dict.setdefault('Save', []).append(
+    browser.mainForm.submitSave())
+
+  # Check whether the changes have been successfully updated
+  assert browser.getTransitionMessage() == 'Data updated.'
+
+  # Validate the person (as the workflow action may not be available yet, try
+  # 5 times and sleep 5s between each attempts before failing) and record
+  # time spent on confirmation
+  browser.mainForm.submitSelectWorkflow(value='validate_action',
+                                        maximum_attempt_number=5,
+                                        sleep_between_attempt=5)
+  result_dict.setdefault('Validate', []).append(
+    browser.mainForm.submitDialogConfirm())
+
+  # Check whether it has been successfully validated
+  assert browser.getTransitionMessage() == 'Status changed.'
+
+
+  ## Go to the new person from the Persons module, showing how to use
+  ## listbox API
+  # Go to Persons module first (person_module)
+  browser.mainForm.submitSelectModule(value='/person_module')
+
+  # Select all the persons whose Usual Name starts with Foo
+  browser.mainForm.getListboxControl(2, 2).value = 'Foo%'
+
+  result_dict.setdefault('Filter', []).append(
+    browser.mainForm.submit())
+
+  # Get the line number
+  line_number = browser.getListboxPosition("Foo%(counter)d Bar%(counter)d" % \
+                                             {'counter': iteration_counter},
+                                           column_number=2)
+
+  # From the column and line_number, we can now get the Link instance
+  link = browser.getListboxLink(line_number=line_number, column_number=2)
+
+  # Click on the link
+  link.click()
+
+  assert browser.mainForm.getControl(name='field_my_first_name').value == \
+      'Foo%d' % iteration_counter
+
+if __name__ == '__main__':
+  # Run benchmarkAddPerson ITERATION times and compute the average time it
+  # took for each operation
+  result_dict = {}
+  counter = 0
+  while counter != ITERATION:
+    benchmarkAddPerson(counter, result_dict)
+    counter += 1
+
+  for title, time_list in result_dict.iteritems():
+    print "%s: %.4fs" % (title, float(sum(time_list)) / ITERATION)

setup.py

@@ -39,6 +39,10 @@ setup(name=name,
       ],
       extras_require={
         'testnode': ['slapos.core', 'xml_marshaller'],
+        'test_browser': ['zope.testbrowser >= 3.11.1', 'z3c.etestbrowser'],
+        'benchmark': [name+'[test_browser]'],
+        'benchmark-report': [name+'[benchmark]', 'matplotlib', 'numpy'],
+        'scalability_tester': [name+'[benchmark]', 'slapos.tool.nosqltester'],
       },
       zip_safe=True,
       packages=find_packages(),
@@ -46,6 +50,9 @@ setup(name=name,
       entry_points={
         'console_scripts': [
           'testnode = erp5.util.testnode:main [testnode]',
+          'performance_tester_erp5 = erp5.util.benchmark.performance_tester:main [benchmark]',
+          'scalability_tester_erp5 = erp5.util.benchmark.scalability_tester:main [scalability_tester]',
+          'generate_erp5_tester_report = erp5.util.benchmark.report:generateReport [benchmark-report]',
         ],
       }
     )