Do not consider an error on the first iteration a specific case.

Before, if the first iteration failed, the whole execution failed even if the
number of maximum errors specified on the command line argument was not
reached, which was an issue especially when running with a range of users.

The results are now adjusted dynamically after the first successful iteration
rather than at the first iteration.

erp5/util/benchmark/process.py

@@ -37,6 +37,7 @@ import datetime
 import socket
 
 from ..testbrowser.browser import Browser
+from .result import NothingFlushedException
 
 RESULT_NUMBER_BEFORE_FLUSHING = 100
 
@@ -84,6 +85,7 @@ class BenchmarkProcess(multiprocessing.Process):
     for target_idx, target in enumerate(self._argument_namespace.benchmark_suite_list):
       self._logger.debug("EXECUTE: %s" % target)
       result.enterSuite(target.__name__)
+      with_error = False
 
       try:
         self._browser.open()
@@ -100,11 +102,11 @@ class BenchmarkProcess(multiprocessing.Process):
           pass
 
         self._error_counter += 1
-        if (self._current_repeat == 1 or
-            self._error_counter >= self._argument_namespace.max_error_number):
+        if self._error_counter >= self._argument_namespace.max_error_number:
           raise RuntimeError(msg)
 
         self._logger.warning(msg)
+        with_error = True
 
       else:
         for stat in result.getCurrentSuiteStatList():
@@ -122,7 +124,7 @@ class BenchmarkProcess(multiprocessing.Process):
             raise RuntimeError("Stopping as mean is greater than maximum "
                                "global average")
 
-      result.exitSuite()
+      result.exitSuite(with_error)
 
     result.iterationFinished()
 
@@ -167,7 +169,10 @@ class BenchmarkProcess(multiprocessing.Process):
           self.runBenchmarkSuiteList(result)
 
           if not self._current_repeat % RESULT_NUMBER_BEFORE_FLUSHING:
+            try:
               result.flush()
+            except NothingFlushedException:
+              pass
 
           self._current_repeat += 1
 

erp5/util/benchmark/result.py

@@ -44,7 +44,6 @@ class BenchmarkResultStatistic(object):
     self.minimum = sys.maxint
     self.maximum = -1
     self.n = 0
-    self.error_sum = 0
 
     # For calculating the mean
     self._value_sum = 0
@@ -53,9 +52,6 @@ class BenchmarkResultStatistic(object):
     self._variance_sum = 0
     self._mean = 0
 
-  def add_error(self):
-    self.error_sum += 1
-
   def add(self, value):
     if value < self.minimum:
       self.minimum = value
@@ -77,6 +73,9 @@ class BenchmarkResultStatistic(object):
   def standard_deviation(self):
     return math.sqrt(self._variance_sum / self.n)
 
+class NothingFlushedException(Exception):
+  pass
+
 import abc
 
 class BenchmarkResult(object):
@@ -86,16 +85,14 @@ class BenchmarkResult(object):
     self._argument_namespace = argument_namespace
     self._nb_users = nb_users
     self._user_index = user_index
-    self._stat_list = []
-    self._suite_idx = 0
-    self._result_idx = 0
-    self.result_list = []
-    self._all_result_list = []
-    self._first_iteration = True
-    self._current_suite_name = None
-    self._result_idx_checkpoint_list = []
-    self.label_list = []
     self._logger = None
+    self._label_list = None
+
+    self._all_suite_list = []
+    self._current_suite_index = 0
+    self._all_result_list = []
+    self._current_suite_dict = None
+    self._current_result_list = None
 
   @property
   def logger(self):
@@ -113,62 +110,106 @@ class BenchmarkResult(object):
     return self
 
   def enterSuite(self, name):
-    self._current_suite_name = name
+    try:
+      self._current_suite_dict = self._all_suite_list[self._current_suite_index]
+    except IndexError:
+      self._current_suite_dict = {'name': name,
+                                  'all_result_list': [],
+                                  'stat_list': [],
+                                  # Number of expected results
+                                  'expected': -1}
 
-  def __call__(self, label, value):
-    self.result_list.append(value)
+      self._all_suite_list.append(self._current_suite_dict)
 
+    self._current_result_list = []
+
+  def __call__(self, label, value):
     try:
-      result_statistic = self._stat_list[self._result_idx]
+      result_statistic = \
+          self._current_suite_dict['stat_list'][len(self._current_result_list)]
+
     except IndexError:
-      result_statistic = BenchmarkResultStatistic(self._current_suite_name,
-                                                  label)
+      result_statistic = BenchmarkResultStatistic(
+        self._current_suite_dict['name'], label)
 
-      self._stat_list.append(result_statistic)
+      self._current_suite_dict['stat_list'].append(result_statistic)
 
     result_statistic.add(value)
-    self._result_idx += 1
+    self._current_result_list.append(value)
+
+  @property
+  def label_list(self):
+    if self._label_list:
+      return self._label_list
 
-  def getLabelList(self):
-    return [ stat.full_label for stat in self._stat_list ]
+    # TODO: Should perhaps be cached...
+    label_list = []
+    for suite_dict in self._all_suite_list:
+      if suite_dict['expected'] == -1:
+        return None
 
-  def iterationFinished(self):
-    self._all_result_list.append(self.result_list)
-    if self._first_iteration:
-      self.label_list = self.getLabelList()
+      label_list.extend([ stat.full_label for stat in suite_dict['stat_list'] ])
 
-    self.logger.debug("RESULTS: %s" % self.result_list)
-    self.result_list = []
-    self._first_iteration = False
-    self._suite_idx = 0
-    self._result_idx = 0
+    self._label_list = label_list
+    return label_list
 
   def getCurrentSuiteStatList(self):
-    start_index = self._suite_idx and \
-        self._result_idx_checkpoint_list[self._suite_idx - 1] or 0
+    return self._current_suite_dict['stat_list']
+
+  @staticmethod
+  def _addResultWithError(result_list, expected_len):
+    missing_result_n = expected_len - len(result_list)
+    if missing_result_n > 0:
+      result_list.extend(missing_result_n * [-1])
 
-    return self._stat_list[start_index:self._result_idx]
+  def exitSuite(self, with_error=False):
+    if with_error:
+      if self._current_suite_dict['expected'] != -1:
+        self._addResultWithError(self._current_result_list,
+                                 self._current_suite_dict['expected'])
 
-  def exitSuite(self):
-    if self._first_iteration:
-      self._result_idx_checkpoint_list.append(self._result_idx)
     else:
-      expected_result_idx = self._result_idx_checkpoint_list[self._suite_idx]
-      while self._result_idx != expected_result_idx:
-        self.result_list.append(0)
-        self._stat_list[self._result_idx].add_error()
-        self._result_idx += 1
+      if self._current_suite_dict['expected'] == -1:
+        self._current_suite_dict['expected'] = len(self._current_result_list)
 
-    self._suite_idx += 1
+        # Fix previous results
+        for result_list in self._current_suite_dict['all_result_list']:
+          self._addResultWithError(result_list,
+                                   self._current_suite_dict['expected'])
+
+    self._current_suite_dict['all_result_list'].append(self._current_result_list)
+    self._current_suite_index += 1
+
+  def iterationFinished(self):
+    self._current_suite_index = 0
 
   @abc.abstractmethod
   def flush(self, partial=True):
-    self._all_result_list = []
+    # TODO: Should perhaps be cached...
+    all_result_list = []
+    for result_dict in self._all_suite_list:
+      if result_dict['expected'] == -1:
+        raise NothingFlushedException()
+
+      for index, result_list in enumerate(result_dict['all_result_list']):
+        try:
+          all_result_list[index].extend(result_list)
+        except IndexError:
+          all_result_list.append(result_list)
+
+      result_dict['all_result_list'] = []
+
+    return all_result_list
 
   @abc.abstractmethod
   def __exit__(self, exc_type, exc_value, traceback_object):
     signal.signal(signal.SIGTERM, signal.SIG_IGN)
+
+    try:
       self.flush(partial=False)
+    except NothingFlushedException:
+      pass
+
     return True
 
 class CSVBenchmarkResult(BenchmarkResult):
@@ -207,15 +248,15 @@ class CSVBenchmarkResult(BenchmarkResult):
     return self
 
   def flush(self, partial=True):
+    result_list = super(CSVBenchmarkResult, self).flush(partial)
+
     if self._result_file.tell() == 0:
       self._csv_writer.writerow(self.label_list)
 
-    self._csv_writer.writerows(self._all_result_list)
+    self._csv_writer.writerows(result_list)
     self._result_file.flush()
     os.fsync(self._result_file.fileno())
 
-    super(CSVBenchmarkResult, self).flush(partial)
-
   def __exit__(self, exc_type, exc_value, traceback_object):
     super(CSVBenchmarkResult, self).__exit__(exc_type, exc_value,
                                              traceback_object)
@@ -251,6 +292,8 @@ class ERP5BenchmarkResult(BenchmarkResult):
     self.log_file.seek(0)
 
   def flush(self, partial=True):
+    result_list = super(ERP5BenchmarkResult, self).flush()
+
     benchmark_result = xmlrpclib.ServerProxy(
       self._argument_namespace.erp5_publish_url,
       verbose=True,
@@ -261,12 +304,10 @@ class ERP5BenchmarkResult(BenchmarkResult):
       self._argument_namespace.repeat,
       self._nb_users,
       self._argument_namespace.benchmark_suite_name_list,
-      self.getLabelList(),
-      self._all_result_list,
+      self.label_list,
+      result_list,
       self._log_buffer_list)
 
-    super(ERP5BenchmarkResult, self).flush()
-
   def __exit__(self, exc_type, exc_value, traceback_object):
     super(ERP5BenchmarkResult, self).__exit__(exc_type, exc_value,
                                               traceback_object)

erp5/util/benchmark/scalability_tester.py

@@ -28,11 +28,15 @@
 #
 ##############################################################################
 
-from .result import CSVBenchmarkResult
+from .result import CSVBenchmarkResult, NothingFlushedException
 
 class CSVScalabilityBenchmarkResult(CSVBenchmarkResult):
   def flush(self, partial=True):
+    try:
       super(CSVScalabilityBenchmarkResult, self).flush(partial)
+    except NothingFlushedException:
+      pass
+    else:
       self._argument_namespace.notify_method(self._result_filename,
                                              self._result_file.tell(),
                                              partial=partial)