Add erp5.utils.benchmark

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk/utils@45502 20353a03-c40f-0410-a6d1-a30d3c3de9de

erp5/util/benchmark/generateReport.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# First version: ERP5Mechanize from Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+
+def parseArguments():
+  parser = argparse.ArgumentParser(
+    description='Generate reports for ERP5 benchmarking suites.')
+
+  parser.add_argument('--enable-debug',
+                      dest='is_debug',
+                      action='store_true',
+                      default=False,
+                      help='Enable debug messages')
+
+  parser.add_argument('--filename-prefix',
+                      default='result',
+                      metavar='PREFIX',
+                      help='Filename prefix for results CSV files '
+                           '(default: result)')
+
+  parser.add_argument('--output-filename',
+                      default='results.pdf',
+                      metavar='FILENAME',
+                      help='PDF output file (default: results.pdf)')
+
+  parser.add_argument('report_directory',
+                      help='Reports directory')
+
+  namespace = parser.parse_args()
+
+  return namespace
+
+import csv
+
+from benchmark import BenchmarkResultStatistic
+
+def computeStatisticFromFilenameList(argument_namespace, filename_list):
+  reader_list = []
+  stat_list = []
+  label_list = []
+
+  for filename in filename_list:
+    reader = csv.reader(open(filename, 'rb'), delimiter=',',
+                        quoting=csv.QUOTE_MINIMAL)
+
+    reader_list.append(reader)
+
+    # Get headers
+    row_list = reader.next()
+    if not label_list:
+      label_list = row_list
+      for label in label_list:
+        stat_list.append(BenchmarkResultStatistic(*label.split(': ', 1)))
+
+    if row_list != label_list:
+      raise AssertionError, "ERROR: Result labels: %s != %s" % \
+          (label_list, row_list)
+
+    for row_list in reader:
+      for idx, row in enumerate(row_list):
+        stat_list[idx].add(float(row))
+
+  return stat_list
+
+def formatFloatList(value_list):
+  return [ format(value, ".3f") for value in value_list ]
+
+import numpy
+import pylab
+
+from matplotlib import pyplot, ticker
+
+def drawBarDiagram(pdf, title, stat_list):
+  mean_list = []
+  yerr_list = []
+  minimum_list = []
+  maximum_list = []
+  label_list = []
+  error_list = []
+
+  for stat in stat_list:
+    mean_list.append(stat.mean)
+    yerr_list.append(stat.standard_deviation)
+    minimum_list.append(stat.minimum)
+    maximum_list.append(stat.maximum)
+    label_list.append(stat.label)
+    error_list.append(stat.error_sum)
+
+  min_array = numpy.array(minimum_list)
+  mean_array = numpy.array(mean_list)
+  max_array = numpy.array(maximum_list)
+
+  yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
+  yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
+
+  ## Draw diagrams
+  # Create the figure
+  figure = pyplot.figure(figsize=(11.69, 8.29))
+  figure.subplots_adjust(bottom=0.13, right=0.98, top=0.95)
+  pyplot.title(title)
+
+  # Create the axes along with their labels
+  axes = figure.add_subplot(111)
+  axes.set_ylabel('Seconds')
+  axes.set_xticks([])
+
+  axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
+  axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
+  axes.yaxis.grid(True, 'major', linewidth=1.5)
+  axes.yaxis.grid(True, 'minor')
+ 
+  # Create the bars
+  ind = numpy.arange(len(label_list))
+  width = 0.33
+
+  min_rects = axes.bar(ind, minimum_list, width, color='y', label='Minimum')
+
+  avg_rects = axes.bar(ind + width, mean_list, width, color='r', label='Mean')
+
+  axes.errorbar(numpy.arange(0.5, len(stat_list)), mean_list,
+                yerr=[yerr_lower, yerr_upper], fmt=None,
+                label='Standard deviation')
+
+  max_rects = axes.bar(ind + width * 2, maximum_list, width, label='Maximum',
+                       color='g')
+
+  # Add the legend of bars
+  axes.legend(loc=0)
+
+  axes.table(rowLabels=['Minimum', 'Average', 'Std. deviation', 'Maximum', 'Errors'],
+             colLabels=label_list,
+             cellText=[formatFloatList(minimum_list),
+                       formatFloatList(mean_list),
+                       formatFloatList(yerr_list),
+                       formatFloatList(maximum_list),
+                       error_list],
+             rowColours=('y', 'r', 'b', 'g', 'w'),
+             loc='bottom',
+             colLoc='center',
+             rowLoc='center',
+             cellLoc='center')
+
+  pdf.savefig()
+  pylab.close()
+
+def drawConcurrentUsersPlot(pdf, title, nb_users_list, stat_list):
+  figure = pyplot.figure(figsize=(11.69, 8.29), frameon=False)
+  figure.subplots_adjust(bottom=0.1, right=0.98, left=0.07, top=0.95)
+  pyplot.title(title)
+  pyplot.grid(True, linewidth=1.5)
+
+  axes = figure.add_subplot(111)
+
+  min_array = numpy.array([stat.minimum for stat in stat_list])
+  mean_array = numpy.array([stat.mean for stat in stat_list])
+  max_array = numpy.array([stat.maximum for stat in stat_list])
+
+  yerr_list = [stat.standard_deviation for stat in stat_list]
+  yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
+  yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
+
+  axes.plot(nb_users_list, min_array, 'yo-', label='Minimum')
+
+  axes.errorbar(nb_users_list,
+                mean_array,
+                yerr=[yerr_lower, yerr_upper],
+                color='r',
+                ecolor='b',
+                label='Mean',
+                elinewidth=2,
+                fmt='D-',
+                capsize=10.0)
+
+  axes.plot(nb_users_list, max_array, 'gs-', label='Maximum')
+
+  axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
+  axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
+  axes.yaxis.grid(True, 'minor')
+
+  axes.xaxis.set_major_locator(ticker.FixedLocator(nb_users_list))
+
+  axes.set_xticks(nb_users_list)
+  axes.legend(loc=0)
+  axes.set_xlabel('Concurrent users')
+  axes.set_ylabel('Seconds')
+
+  pyplot.xlim(xmin=nb_users_list[0])
+  pdf.savefig()
+  pylab.close()
+
+from matplotlib.backends.backend_pdf import PdfPages
+
+import glob
+import os
+import re
+
+user_re = re.compile('-(\d+)users-')
+
+def generateReport():
+  argument_namespace = parseArguments()
+
+  filename_iter = glob.iglob("%s-*repeat*-*users*-*process*.csv" % os.path.join(
+      argument_namespace.report_directory,
+      argument_namespace.filename_prefix))
+
+  per_nb_users_report_dict = {}
+  for filename in filename_iter:
+    report_dict = per_nb_users_report_dict.setdefault(
+      int(user_re.search(filename).group(1)), {'filename': []})
+
+    report_dict['filename'].append(filename)
+
+  pdf = PdfPages(argument_namespace.output_filename)
+
+  for nb_users, report_dict in per_nb_users_report_dict.items():
+    stat_list = computeStatisticFromFilenameList(
+      argument_namespace, report_dict['filename'])
+
+    title = "Ran suites with %d users" % len(report_dict['filename'])
+    for slice_start_idx in range(0, len(stat_list), 12):
+      if slice_start_idx != 0:
+        title += ' (Ctd.)'
+
+      drawBarDiagram(pdf, title, stat_list[slice_start_idx:slice_start_idx + 12])
+
+    report_dict['stats'] = stat_list
+
+  if len(per_nb_users_report_dict) != 1:
+    for i in range(len(report_dict['stats'])):
+      stat_list = []
+      nb_users_list = per_nb_users_report_dict.keys()
+      for report_dict in per_nb_users_report_dict.values():
+        stat_list.append(report_dict['stats'][i])
+
+      drawConcurrentUsersPlot(
+        pdf,
+        "%s from %d to %d users (step: %d)" % (stat_list[0].full_label,
+                                               nb_users_list[0],
+                                               nb_users_list[-1],
+                                               nb_users_list[1] - nb_users_list[0]),
+        nb_users_list,
+        stat_list)
+
+  pdf.close()
+
+if __name__ == '__main__':
+  generateReport()

erp5/util/benchmark/runBenchmark.py

+#!/usr/bin/env python
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+import os
+
+from benchmark import ArgumentType
+
+def parseArguments():
+  parser = argparse.ArgumentParser(description='Run ERP5 benchmarking suites.')
+
+  # Optional arguments
+  parser.add_argument('--filename-prefix',
+                      default='result',
+                      metavar='PREFIX',
+                      help='Filename prefix for results and logs files '
+                           '(default: result)')
+
+  parser.add_argument('--report-directory',
+                      type=ArgumentType.directoryType,
+                      default=os.getcwd(),
+                      metavar='DIRECTORY',
+                      help='Directory where the results and logs will be stored '
+                           '(default: current directory)')
+
+  parser.add_argument('--max-global-average',
+                      type=float,
+                      default=0,
+                      metavar='N',
+                      help='Stop when any suite operation is over this value '
+                           '(default: disable)')
+
+  parser.add_argument('--users-file',
+                      dest='user_info_filename',
+                      default='userInfo',
+                      metavar='MODULE',
+                      help="Import users from ``user_tuple'' in MODULE")
+
+  parser.add_argument('--users-range-increment',
+                      type=ArgumentType.strictlyPositiveIntType,
+                      default=1,
+                      metavar='N',
+                      help='Number of users being added after each repetition '
+                           '(default: 1)')
+
+  parser.add_argument('--enable-debug',
+                      dest='is_debug',
+                      action='store_true',
+                      default=False,
+                      help='Enable debug messages')
+
+  parser.add_argument('--enable-legacy-listbox',
+                      dest='is_legacy_listbox',
+                      action='store_true',
+                      default=False,
+                      help='Enable legacy listbox for Browser')
+
+  parser.add_argument('--repeat',
+                      type=ArgumentType.strictlyPositiveIntType,
+                      default=-1,
+                      metavar='N',
+                      help='Repeat the benchmark suite N times '
+                           '(default: infinite)')
+
+  # Mandatory arguments
+  parser.add_argument('url',
+                      type=ArgumentType.ERP5UrlType,
+                      metavar='URL',
+                      help='ERP5 base URL')
+
+  parser.add_argument('users',
+                      type=ArgumentType.strictlyPositiveIntOrRangeType,
+                      metavar='NB_USERS|MIN_NB_USERS,MAX_NB_USERS',
+                      help='Number of users (fixed or a range)')
+
+  parser.add_argument('benchmark_suite_list',
+                      nargs='+',
+                      metavar='BENCHMARK_SUITES',
+                      help='Benchmark suite modules')
+
+  namespace = parser.parse_args()
+
+  namespace.user_tuple = ArgumentType.objectFromModule(namespace.user_info_filename,
+                                                       object_name='user_tuple')
+
+  object_benchmark_suite_list = []
+  for benchmark_suite in namespace.benchmark_suite_list:
+    object_benchmark_suite_list.append(ArgumentType.objectFromModule(benchmark_suite,
+                                                                     callable_object=True))
+
+  namespace.benchmark_suite_list = object_benchmark_suite_list
+
+  max_nb_users = isinstance(namespace.users, tuple) and namespace.users[1] or \
+      namespace.users
+
+  if max_nb_users > len(namespace.user_tuple):
+    raise argparse.ArgumentTypeError("Not enough users in the given file")
+
+  return namespace
+
+import sys
+import multiprocessing
+
+from benchmark import BenchmarkProcess
+
+def runConstantBenchmark(argument_namespace, nb_users):
+  process_list = []
+
+  exit_msg_queue = multiprocessing.Queue(nb_users)
+
+  for user_index in range(nb_users):
+    process = BenchmarkProcess(exit_msg_queue, nb_users, user_index, argument_namespace)
+    process_list.append(process)
+
+  for process in process_list:
+    process.start()
+
+  error_message_set = set()
+  i = 0
+  while i != len(process_list):
+    try:
+      msg = exit_msg_queue.get()
+    except KeyboardInterrupt:
+      if argument_namespace.repeat != -1:
+        print >>sys.stderr, "Stopping gracefully"
+        for process in process_list:
+          process.terminate()
+
+        i = 0
+        continue
+
+    if msg is not None:
+      error_message_set.add(msg)
+      for process in process_list:
+        process.terminate()
+
+      break
+
+    i += 1
+
+  if error_message_set:
+    for error_message in error_message_set:
+      print >>sys.stderr, "ERROR: %s" % error_message
+
+    sys.exit(1)
+
+def runBenchmark():
+  argument_namespace = parseArguments()
+
+  if isinstance(argument_namespace.users, tuple):
+    nb_users, max_users = argument_namespace.users
+    while True:
+      runConstantBenchmark(argument_namespace, nb_users)
+
+      if nb_users == max_users:
+        break
+
+      nb_users = min(nb_users + argument_namespace.users_range_increment,
+                     max_users)
+
+  else:
+    runConstantBenchmark(argument_namespace, argument_namespace.users)
+
+if __name__ == '__main__':
+  runBenchmark()