Commit 922a597b authored by Arnaud Fontaine's avatar Arnaud Fontaine

Fix KeyboardInterrupt on control process where the results were not

properly flushed before exiting and terminate children properly.

Also, terminate children after a given number of errors (based upon a
patch from jm).

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk/utils@46015 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent fa44baa3
......@@ -33,11 +33,15 @@ import os
import sys
import multiprocessing
import xmlrpclib
import signal
import errno
from erp5.utils.benchmark.argument import ArgumentType
from erp5.utils.benchmark.process import BenchmarkProcess
from erp5.utils.benchmark.result import ERP5BenchmarkResult, CSVBenchmarkResult
MAXIMUM_KEYBOARD_INTERRUPT = 5
class PerformanceTester(object):
def __init__(self, namespace=None):
if not namespace:
......@@ -46,6 +50,8 @@ class PerformanceTester(object):
else:
self._argument_namespace = namespace
self._process_terminated_counter = 0
@staticmethod
def _add_parser_arguments(parser):
# Optional arguments
......@@ -189,6 +195,9 @@ class PerformanceTester(object):
ERP5BenchmarkResult.closeResultDocument(self._argument_namespace.erp5_publish_url,
error_message_set)
def _child_terminated_handler(self, *args, **kwargs):
self._process_terminated_counter += 1
def _run_constant(self, nb_users):
process_list = []
exit_msg_queue = multiprocessing.Queue(nb_users)
......@@ -202,33 +211,41 @@ class PerformanceTester(object):
process_list.append(process)
signal.signal(signal.SIGCHLD, self._child_terminated_handler)
for process in process_list:
process.start()
error_message_set = set()
i = 0
while i != len(process_list):
interrupted_counter = 0
while self._process_terminated_counter != len(process_list):
try:
msg = exit_msg_queue.get()
except KeyboardInterrupt:
if self._argument_namespace.repeat != -1:
print >>sys.stderr, "Stopping gracefully"
for process in process_list:
process.terminate()
error_message = exit_msg_queue.get()
i = 0
continue
else:
msg = None
except KeyboardInterrupt, e:
if interrupted_counter == 0:
print >>sys.stderr, "\nInterrupted by user, stopping gracefully " \
"unless interrupted %d times" % MAXIMUM_KEYBOARD_INTERRUPT
interrupted_counter += 1
if msg is not None:
error_message_set.add(msg)
for process in process_list:
process.terminate()
if (not getattr(process, '_stopping', False) or
interrupted_counter == MAXIMUM_KEYBOARD_INTERRUPT):
process._stopping = True
process.terminate()
break
# An IOError may be raised when receiving a SIGCHLD which interrupts the
# blocking system call above and the system call should not be restarted
# (using siginterrupt), otherwise the process will stall forever as its
# child has already exited
except IOError, e:
if e.errno == errno.EINTR:
continue
i += 1
else:
if error_message is not None:
error_message_set.add(error_message)
if error_message_set:
return (error_message_set, 1)
......
......@@ -36,6 +36,9 @@ import sys
from erp5.utils.test_browser.browser import Browser
MAXIMUM_ERROR_COUNTER = 10
RESULT_NUMBER_BEFORE_FLUSHING = 100
class BenchmarkProcess(multiprocessing.Process):
def __init__(self, exit_msg_queue, result_klass, argument_namespace,
nb_users, user_index, *args, **kwargs):
......@@ -48,6 +51,7 @@ class BenchmarkProcess(multiprocessing.Process):
# Initialized when running the test
self._browser = None
self._current_repeat = 1
self._error_counter = 0
super(BenchmarkProcess, self).__init__(*args, **kwargs)
......@@ -71,6 +75,9 @@ class BenchmarkProcess(multiprocessing.Process):
try:
target(result, self._browser)
except BaseException, e:
if isinstance(e, StopIteration):
raise
msg = "%s: %s" % (target, traceback.format_exc())
if (self._argument_namespace.enable_debug and isinstance(e, Exception)):
......@@ -79,10 +86,11 @@ class BenchmarkProcess(multiprocessing.Process):
except:
pass
if self._current_repeat == 1:
self._logger.error(msg)
raise
if (self._current_repeat == 1 or
self._error_counter == MAXIMUM_ERROR_COUNTER):
raise RuntimeError(msg)
self._error_counter += 1
self._logger.warning(msg)
for stat in result.getCurrentSuiteStatList():
......@@ -95,12 +103,10 @@ class BenchmarkProcess(multiprocessing.Process):
stat.standard_deviation,
stat.maximum))
if self._argument_namespace.max_global_average and \
mean > self._argument_namespace.max_global_average:
self._logger.info("Stopping as mean is greater than maximum "
"global average")
raise StopIteration
if (self._argument_namespace.max_global_average and
mean > self._argument_namespace.max_global_average):
raise RuntimeError("Stopping as mean is greater than maximum "
"global average")
result.exitSuite()
......@@ -113,8 +119,12 @@ class BenchmarkProcess(multiprocessing.Process):
self._logger = result_instance.getLogger()
if self._argument_namespace.repeat != -1:
signal.signal(signal.SIGTERM, self.stopGracefully)
# Ensure the data are flushed before exiting, handled by Result class
# __exit__ block
signal.signal(signal.SIGTERM, self.stopGracefully)
# Ignore KeyboardInterrupt as it is handled by the parent process
signal.signal(signal.SIGINT, signal.SIG_IGN)
exit_status = 0
exit_msg = None
......@@ -128,16 +138,15 @@ class BenchmarkProcess(multiprocessing.Process):
self.runBenchmarkSuiteList(result)
self._current_repeat += 1
if self._current_repeat == 100:
if self._current_repeat == RESULT_NUMBER_BEFORE_FLUSHING:
result.flush()
except StopIteration, e:
exit_msg = str(e)
exit_status = 1
self._logger.error(e)
except BaseException, e:
exit_msg = e
exit_status = 2
exit_status = 1
self._exit_msg_queue.put(exit_msg)
sys.exit(exit_status)
......@@ -214,15 +214,10 @@ class CSVBenchmarkResult(BenchmarkResult):
super(CSVBenchmarkResult, self).__exit__(exc_type, exc_value, traceback)
self._result_file.close()
if exc_type:
if exc_type and not issubclass(exc_type, StopIteration):
msg = "An error occured, see: %s" % self._log_filename_path
from traceback import format_tb
self.getLogger().error("%s: %s\n%s" % (exc_type, exc_value,
''.join(format_tb(traceback))))
if isinstance(exc_type, StopIteration):
raise StopIteration, msg
else:
raise RuntimeError, msg
self.getLogger().error("%s: %s" % (exc_type, exc_value))
raise RuntimeError(msg)
from cStringIO import StringIO
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment