Import erp5.utils.{benchmark,test_browser} from svn.erp5.org:public/erp5/trunk/utils

CHANGES.erp5.util.txt

@@ -4,7 +4,12 @@ Changes
 0.2 (unreleased)
 ----------------
 
- * No changes yet.
+ * Imported from https://svn.erp5.org/repos/public/erp5/trunk/utils/
+
+   - erp5.util.test_browser:
+     Programmable browser for functional and performance tests for ERP5
+   - erp5.util.benchmark:
+     Performance benchmarks for ERP5 with erp5.utils.test_browser
 
 0.1 (2011-08-08)
 ----------------

erp5/util/README.test_browser.txt

+API Documentation
+-----------------
+
+You can generate the API documentation using ``epydoc'':
+
+$ epydoc src/erp5

erp5/util/benchmark/argument.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import argparse
+import functools
+
+class ArgumentType(object):
+  @classmethod
+  def directoryType(cls, path):
+    if not (os.path.isdir(path) and os.access(path, os.W_OK)):
+      raise argparse.ArgumentTypeError("'%s' is not a valid directory or is "\
+                                         "not writable" % path)
+
+    return path
+
+  @classmethod
+  def objectFromModule(cls, module_name, object_name=None,
+                       callable_object=False):
+    if module_name.endswith('.py'):
+      module_name = module_name[:-3]
+
+    if not object_name:
+      object_name = module_name
+
+    import sys
+    sys.path.append(os.getcwd())
+
+    try:
+      module = __import__(module_name, globals(), locals(), [object_name], -1)
+    except Exception, e:
+      raise argparse.ArgumentTypeError("Cannot import '%s.%s': %s" % \
+                                         (module_name, object_name, str(e)))
+
+    try:
+      obj = getattr(module, object_name)
+    except AttributeError:
+      raise argparse.ArgumentTypeError("Could not get '%s' in '%s'" % \
+                                         (object_name, module_name))
+
+    if callable_object and not callable(obj):
+      raise argparse.ArgumentTypeError(
+        "'%s.%s' is not callable" % (module_name, object_name))
+
+    return obj
+
+  @classmethod
+  def strictlyPositiveIntType(cls, value):
+    try:
+      converted_value = int(value)
+    except ValueError:
+      pass
+    else:
+      if converted_value > 0:
+        return converted_value
+
+    raise argparse.ArgumentTypeError('expects a strictly positive integer')
+
+  @classmethod
+  def strictlyPositiveIntOrRangeType(cls, value):
+    try:
+      return cls.strictlyPositiveIntType(value)
+    except argparse.ArgumentTypeError:
+      try:
+        min_max_list = value.split(',')
+      except ValueError:
+        pass
+      else:
+        if len(min_max_list) == 2:
+          minimum, maximum = cls.strictlyPositiveIntType(min_max_list[0]), \
+              cls.strictlyPositiveIntType(min_max_list[1])
+
+          if minimum >= maximum:
+            raise argparse.ArgumentTypeError('%d >= %d' % (minimum, maximum))
+
+          return (minimum, maximum)
+
+    raise argparse.ArgumentTypeError(
+      'expects either a strictly positive integer or a range of strictly '
+      'positive integer separated by a comma')
+
+  @classmethod
+  def ERP5UrlType(cls, url):
+    if url[-1] == '/':
+      url_list = url.rsplit('/', 2)[:-1]
+    else:
+      url_list = url.rsplit('/', 1)
+
+    url_list[0] = url_list[0] + '/'
+    if len(url_list) != 2:
+      raise argparse.ArgumentTypeError("Invalid URL given")
+
+    return url_list

erp5/util/benchmark/examples/createPerson.py

+# -*- coding: utf-8 -*-
+
+def createPerson(result, browser):
+  """
+  Create a Person and add a  telephone number.  It can be ran infinitely (e.g.
+  until it  is interrupted by  the end user)  with 1 concurrent  user, through
+  performance_tester_erp5 with the following command:
+
+  performance_tester_erp5 http://foo.bar:4242/erp5/ 1 createPerson
+
+  Please note that  you must run this command from the  same directory of this
+  script  and userInfo.py.  Further information  about performance_tester_erp5
+  options and arguments are available by specifying ``--help''.
+  """
+  # Go to Persons module (person_module)
+  result('Go to person module',
+         browser.mainForm.submitSelectModule(value='/person_module'))
+
+  # Create a new person and record the time elapsed in seconds
+  result('Add Person', browser.mainForm.submitNew())
+
+  # Check whether it has been successfully created
+  assert browser.getTransitionMessage() == 'Object created.'
+
+  # Fill the first and last name of the newly created person
+  browser.mainForm.getControl(name='field_my_first_name').value = 'Create'
+  browser.mainForm.getControl(name='field_my_last_name').value = 'Person'
+
+  # Submit the changes, record the time elapsed in seconds
+  result('Save', browser.mainForm.submitSave())
+
+  # Check whether the changes have been successfully updated
+  assert browser.getTransitionMessage() == 'Data updated.'
+  person_url = browser.url
+
+  # Add phone number
+  result('Add telephone',
+         browser.mainForm.submitSelectAction(value='add Telephone'))
+
+  # Fill telephone title and number
+  browser.mainForm.getControl(name='field_my_title'). value = 'Personal'
+  browser.mainForm.getControl(name='field_my_telephone_number').value = '0123456789'
+
+  # Submit the changes, record the time elapsed in seconds
+  result('Save', browser.mainForm.submitSave())
+
+  # Check whether the changes have been successfully updated
+  assert browser.getTransitionMessage() == 'Data updated.'
+
+  # Go back to the Person page before validating
+  browser.open(person_url)
+
+  # Validate it (as the workflow action may not be available yet, try 5 times
+  # and sleep 5s between each attempts before failing)
+  show_validate_time, waiting_for_validate_action = \
+      browser.mainForm.submitSelectWorkflow(value='validate_action',
+                                            maximum_attempt_number=5,
+                                            sleep_between_attempt=5)
+
+  result('Waiting for validate_action', waiting_for_validate_action)
+  result('Show validate', show_validate_time)
+  result('Validated', browser.mainForm.submitDialogConfirm())
+  assert browser.getTransitionMessage() == 'Status changed.'

erp5/util/benchmark/examples/userInfo.py

+# Specify user login/password used to run the tests
+user_tuple = (('zope', 'zope'),)

erp5/util/benchmark/process.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import multiprocessing
+import csv
+import traceback
+import os
+import logging
+import signal
+import sys
+
+from ..test_browser.browser import Browser
+
+MAXIMUM_ERROR_COUNTER = 10
+RESULT_NUMBER_BEFORE_FLUSHING = 100
+
+class BenchmarkProcess(multiprocessing.Process):
+  def __init__(self, exit_msg_queue, result_klass, argument_namespace,
+               nb_users, user_index, *args, **kwargs):
+    self._exit_msg_queue = exit_msg_queue
+    self._result_klass = result_klass
+    self._argument_namespace = argument_namespace
+    self._nb_users = nb_users
+    self._user_index = user_index
+
+    # Initialized when running the test
+    self._browser = None
+    self._current_repeat = 1
+
+    # TODO: Per target error counter instead of global one?
+    self._error_counter = 0
+
+    super(BenchmarkProcess, self).__init__(*args, **kwargs)
+
+  def stopGracefully(self, *args, **kwargs):
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    raise StopIteration("Interrupted by user or because of an error from "
+                        "another process, flushing remaining results...")
+
+  def getBrowser(self, log_file):
+    info_list = tuple(self._argument_namespace.url) + \
+        tuple(self._argument_namespace.user_tuple[self._user_index])
+
+    return Browser(*info_list,
+                   is_debug=self._argument_namespace.enable_debug,
+                   log_file=log_file,
+                   is_legacy_listbox=self._argument_namespace.is_legacy_listbox)
+
+  def runBenchmarkSuiteList(self, result):
+    for target_idx, target in enumerate(self._argument_namespace.benchmark_suite_list):
+      self._logger.debug("EXECUTE: %s" % target)
+      result.enterSuite(target.__name__)
+
+      try:
+        self._browser.open()
+        target(result, self._browser)
+      except StopIteration:
+        raise
+      except Exception, e:
+        msg = "%s: %s" % (target, traceback.format_exc())
+
+        try:
+          msg += "Last response headers:\n%s\nLast response contents:\n%s" % \
+              (self._browser.headers, self._browser.contents)
+        except:
+          pass
+
+        if (self._current_repeat == 1 or
+            self._error_counter == MAXIMUM_ERROR_COUNTER):
+          raise RuntimeError(msg)
+
+        self._error_counter += 1
+        self._logger.warning(msg)
+
+      for stat in result.getCurrentSuiteStatList():
+        mean = stat.mean
+
+        self._logger.info("%s: min=%.3f, mean=%.3f (+/- %.3f), max=%.3f" % \
+                            (stat.full_label,
+                             stat.minimum,
+                             mean,
+                             stat.standard_deviation,
+                             stat.maximum))
+
+        if (self._argument_namespace.max_global_average and
+            mean > self._argument_namespace.max_global_average):
+          raise RuntimeError("Stopping as mean is greater than maximum "
+                             "global average")
+
+      result.exitSuite()
+
+    result.iterationFinished()
+
+  def run(self):
+    result_instance = self._result_klass(self._argument_namespace,
+                                         self._nb_users,
+                                         self._user_index)
+
+    self._logger = result_instance.getLogger()
+
+    # Ensure the data are flushed before exiting, handled by Result class 
+    # __exit__ block
+    signal.signal(signal.SIGTERM, self.stopGracefully)
+ 
+    # Ignore KeyboardInterrupt as it is handled by the parent process
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    exit_status = 0
+    exit_msg = None
+
+    try:
+      with result_instance as result:
+        self._browser = self.getBrowser(result_instance.log_file)
+
+        while self._current_repeat != (self._argument_namespace.repeat + 1):
+          self._logger.info("Iteration: %d" % self._current_repeat)
+          self.runBenchmarkSuiteList(result)
+          self._current_repeat += 1
+
+          if not self._current_repeat % RESULT_NUMBER_BEFORE_FLUSHING:
+            result.flush()
+
+    except StopIteration, e:
+      self._logger.error(e)
+
+    except BaseException, e:
+      exit_msg = str(e)
+      exit_status = 1
+
+    self._exit_msg_queue.put(exit_msg)
+    sys.exit(exit_status)

erp5/util/benchmark/report.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# First version: ERP5Mechanize from Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+
+def parseArguments():
+  parser = argparse.ArgumentParser(
+    description='Generate reports for ERP5 benchmarking suites.')
+
+  parser.add_argument('--enable-debug',
+                      dest='is_debug',
+                      action='store_true',
+                      default=False,
+                      help='Enable debug messages')
+
+  parser.add_argument('--filename-prefix',
+                      default='result',
+                      metavar='PREFIX',
+                      help='Filename prefix for results CSV files '
+                           '(default: result)')
+
+  parser.add_argument('--output-filename',
+                      default='results.pdf',
+                      metavar='FILENAME',
+                      help='PDF output file (default: results.pdf)')
+
+  parser.add_argument('report_directory',
+                      help='Reports directory')
+
+  namespace = parser.parse_args()
+
+  return namespace
+
+import csv
+
+from .result import BenchmarkResultStatistic
+
+def computeStatisticFromFilenameList(argument_namespace, filename_list):
+  reader_list = []
+  stat_list = []
+  label_list = []
+
+  for filename in filename_list:
+    reader = csv.reader(open(filename, 'rb'), delimiter=',',
+                        quoting=csv.QUOTE_MINIMAL)
+
+    reader_list.append(reader)
+
+    # Get headers
+    row_list = reader.next()
+    if not label_list:
+      label_list = row_list
+      for label in label_list:
+        stat_list.append(BenchmarkResultStatistic(*label.split(': ', 1)))
+
+    if row_list != label_list:
+      raise AssertionError, "ERROR: Result labels: %s != %s" % \
+          (label_list, row_list)
+
+    for row_list in reader:
+      for idx, row in enumerate(row_list):
+        stat_list[idx].add(float(row))
+
+  return stat_list
+
+def formatFloatList(value_list):
+  return [ format(value, ".3f") for value in value_list ]
+
+import numpy
+import pylab
+
+from matplotlib import pyplot, ticker
+
+def drawBarDiagram(pdf, title, stat_list):
+  mean_list = []
+  yerr_list = []
+  minimum_list = []
+  maximum_list = []
+  label_list = []
+  error_list = []
+
+  for stat in stat_list:
+    mean_list.append(stat.mean)
+    yerr_list.append(stat.standard_deviation)
+    minimum_list.append(stat.minimum)
+    maximum_list.append(stat.maximum)
+    label_list.append(stat.label)
+    error_list.append(stat.error_sum)
+
+  min_array = numpy.array(minimum_list)
+  mean_array = numpy.array(mean_list)
+  max_array = numpy.array(maximum_list)
+
+  yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
+  yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
+
+  ## Draw diagrams
+  # Create the figure
+  figure = pyplot.figure(figsize=(11.69, 8.29))
+  figure.subplots_adjust(bottom=0.13, right=0.98, top=0.95)
+  pyplot.title(title)
+
+  # Create the axes along with their labels
+  axes = figure.add_subplot(111)
+  axes.set_ylabel('Seconds')
+  axes.set_xticks([])
+
+  axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
+  axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
+  axes.yaxis.grid(True, 'major', linewidth=1.5)
+  axes.yaxis.grid(True, 'minor')
+ 
+  # Create the bars
+  ind = numpy.arange(len(label_list))
+  width = 0.33
+
+  min_rects = axes.bar(ind, minimum_list, width, color='y', label='Minimum')
+
+  avg_rects = axes.bar(ind + width, mean_list, width, color='r', label='Mean')
+
+  axes.errorbar(numpy.arange(0.5, len(stat_list)), mean_list,
+                yerr=[yerr_lower, yerr_upper], fmt=None,
+                label='Standard deviation')
+
+  max_rects = axes.bar(ind + width * 2, maximum_list, width, label='Maximum',
+                       color='g')
+
+  # Add the legend of bars
+  axes.legend(loc=0)
+
+  axes.table(rowLabels=['Minimum', 'Average', 'Std. deviation', 'Maximum', 'Errors'],
+             colLabels=label_list,
+             cellText=[formatFloatList(minimum_list),
+                       formatFloatList(mean_list),
+                       formatFloatList(yerr_list),
+                       formatFloatList(maximum_list),
+                       error_list],
+             rowColours=('y', 'r', 'b', 'g', 'w'),
+             loc='bottom',
+             colLoc='center',
+             rowLoc='center',
+             cellLoc='center')
+
+  pdf.savefig()
+  pylab.close()
+
+def drawConcurrentUsersPlot(pdf, title, nb_users_list, stat_list):
+  figure = pyplot.figure(figsize=(11.69, 8.29), frameon=False)
+  figure.subplots_adjust(bottom=0.1, right=0.98, left=0.07, top=0.95)
+  pyplot.title(title)
+  pyplot.grid(True, linewidth=1.5)
+
+  axes = figure.add_subplot(111)
+
+  min_array = numpy.array([stat.minimum for stat in stat_list])
+  mean_array = numpy.array([stat.mean for stat in stat_list])
+  max_array = numpy.array([stat.maximum for stat in stat_list])
+
+  yerr_list = [stat.standard_deviation for stat in stat_list]
+  yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
+  yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
+
+  axes.plot(nb_users_list, min_array, 'yo-', label='Minimum')
+
+  axes.errorbar(nb_users_list,
+                mean_array,
+                yerr=[yerr_lower, yerr_upper],
+                color='r',
+                ecolor='b',
+                label='Mean',
+                elinewidth=2,
+                fmt='D-',
+                capsize=10.0)
+
+  axes.plot(nb_users_list, max_array, 'gs-', label='Maximum')
+
+  axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
+  axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
+  axes.yaxis.grid(True, 'minor')
+
+  axes.xaxis.set_major_locator(ticker.FixedLocator(nb_users_list))
+
+  axes.set_xticks(nb_users_list)
+  axes.legend(loc=0)
+  axes.set_xlabel('Concurrent users')
+  axes.set_ylabel('Seconds')
+
+  pyplot.xlim(xmin=nb_users_list[0])
+  pdf.savefig()
+  pylab.close()
+
+from matplotlib.backends.backend_pdf import PdfPages
+
+import glob
+import os
+import re
+
+user_re = re.compile('-(\d+)users-')
+
+def generateReport():
+  argument_namespace = parseArguments()
+
+  filename_iter = glob.iglob("%s-*repeat*-*users*-*process*.csv" % os.path.join(
+      argument_namespace.report_directory,
+      argument_namespace.filename_prefix))
+
+  per_nb_users_report_dict = {}
+  for filename in filename_iter:
+    report_dict = per_nb_users_report_dict.setdefault(
+      int(user_re.search(filename).group(1)), {'filename': []})
+
+    report_dict['filename'].append(filename)
+
+  pdf = PdfPages(argument_namespace.output_filename)
+
+  for nb_users, report_dict in per_nb_users_report_dict.items():
+    stat_list = computeStatisticFromFilenameList(
+      argument_namespace, report_dict['filename'])
+
+    title = "Ran suites with %d users" % len(report_dict['filename'])
+    for slice_start_idx in range(0, len(stat_list), 12):
+      if slice_start_idx != 0:
+        title += ' (Ctd.)'
+
+      drawBarDiagram(pdf, title, stat_list[slice_start_idx:slice_start_idx + 12])
+
+    report_dict['stats'] = stat_list
+
+  if len(per_nb_users_report_dict) != 1:
+    for i in range(len(report_dict['stats'])):
+      stat_list = []
+      nb_users_list = per_nb_users_report_dict.keys()
+      for report_dict in per_nb_users_report_dict.values():
+        stat_list.append(report_dict['stats'][i])
+
+      drawConcurrentUsersPlot(
+        pdf,
+        "%s from %d to %d users (step: %d)" % (stat_list[0].full_label,
+                                               nb_users_list[0],
+                                               nb_users_list[-1],
+                                               nb_users_list[1] - nb_users_list[0]),
+        nb_users_list,
+        stat_list)
+
+  pdf.close()
+
+if __name__ == '__main__':
+  generateReport()

erp5/util/benchmark/result.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import sys
+import math
+import os
+import csv
+import logging
+import signal
+
+class BenchmarkResultStatistic(object):
+  def __init__(self, suite, label):
+    self.suite = suite
+    self.label = label
+
+    self.full_label = '%s: %s' % (self.suite, self.label)
+
+    self.minimum = sys.maxint
+    self.maximum = -1
+    self.n = 0
+    self.error_sum = 0
+
+    # For calculating the mean
+    self._value_sum = 0
+
+    # For calculating the standard deviation
+    self._variance_sum = 0
+    self._mean = 0
+
+  def add_error(self):
+    self.error_sum += 1
+
+  def add(self, value):
+    if value < self.minimum:
+      self.minimum = value
+    if value > self.maximum:
+      self.maximum = value
+
+    self._value_sum += value
+    self.n += 1
+
+    delta = value - self._mean
+    self._mean += delta / self.n
+    self._variance_sum += delta * (value - self._mean)
+
+  @property
+  def mean(self):
+    return self._value_sum / self.n
+
+  @property
+  def standard_deviation(self):
+    return math.sqrt(self._variance_sum / self.n)