Add xunit output support to tests

product/ERP5Type/tests/runUnitTest.py

@@ -18,6 +18,7 @@ try:
   from coverage import coverage
 except ImportError:
   coverage = None
+import xunit
 
 WIN = os.name == 'nt'
 
@@ -145,6 +146,7 @@ Options:
   --sys_path=path,path       Comma-separated list of paths which will be used to
                              extend sys.path
   --instance_home=PATH       Create/use test instance in given path
+  --xunit_file=PATH          File for xunit xml report
 
 When no unit test is specified, only activities are processed.
 """
@@ -450,10 +452,16 @@ getattr(unittest, 'loader', unittest).TestLoader = ERP5TypeTestLoader
 class DebugTestResult:
   """Wrap an unittest.TestResult, invoking pdb on errors / failures
   """
-  def __init__(self, result):
+  xunit = None
+  def __init__(self, result, debug, xunit_file):
     self.result = result
+    if xunit_file:
+      self.xunit = xunit.Xunit(xunit_file)
+    self._debug = debug
 
   def _start_debugger(self, tb):
+    if not self._debug:
+      return
     import Lifetime
     if Lifetime._shutdown_phase:
       return
@@ -476,17 +484,36 @@ class DebugTestResult:
   def addError(self, test, err):
     self._start_debugger(err[2])
     self.result.addError(test, err)
+    if self.xunit:
+      self.xunit.addError(test, err)
 
   def addFailure(self, test, err):
     self._start_debugger(err[2])
     self.result.addFailure(test, err)
+    if self.xunit:
+      self.xunit.addFailure(test, err)
+
+  def addSuccess(self, test):
+    self.result.addSuccess(test)
+    if self.xunit:
+      self.xunit.addSuccess(test)
+
+  def startTest(self, test):
+    self.result.startTest(test)
+    if self.xunit:
+      self.xunit.startTest(test)
+
+  def printErrors(self):
+    self.result.printErrors()
+    if self.xunit:
+      self.xunit.report(unittest._WritelnDecorator(sys.stdout))
 
   def __getattr__(self, attr):
     return getattr(self.result, attr)
 
 _print = sys.stderr.write
 
-def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
+def runUnitTestList(test_list, verbosity=1, debug=0, xunit_file=None, run_only=None):
   if "zeo_client" in os.environ and "zeo_server" in os.environ:
     _print("conflicting options: --zeo_client and --zeo_server")
     sys.exit(1)
@@ -648,13 +675,13 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
         ERP5TypeTestLoader._testMethodPrefix = 'dummy_test'
         PortalTestCase.setUp = dummy_setUp
         PortalTestCase.tearDown = dummy_tearDown
-      elif debug:
+      else:
         # Hack the profiler to run only specified test methods,
         # and wrap results when running in debug mode.
         class DebugTextTestRunner(TestRunner):
           def _makeResult(self):
             result = super(DebugTextTestRunner, self)._makeResult()
-            return DebugTestResult(result)
+            return DebugTestResult(result, debug, xunit_file)
         TestRunner = DebugTextTestRunner
       loader = ERP5TypeTestLoader()
       if run_only:
@@ -783,6 +810,7 @@ def main(argument_list=None):
         "products_path=",
         "sys_path=",
         "instance_home=",
+        "xunit_file=",
         ])
   except getopt.GetoptError, msg:
     usage(sys.stderr, msg)
@@ -795,6 +823,7 @@ def main(argument_list=None):
   verbosity = 1
   debug = 0
   run_only = None
+  xunit_file = None
   instance_home = os.path.join(real_instance_home, 'unit_test')
 
   for opt, arg in opts:
@@ -900,6 +929,8 @@ def main(argument_list=None):
       sys.path.extend(arg.split(','))
     elif opt == "--instance_home":
       instance_home = os.path.abspath(arg)
+    elif opt == "--xunit_file":
+      xunit_file = os.path.abspath(arg)
 
   global tests_home
   os.environ['INSTANCE_HOME'] = instance_home
@@ -911,6 +942,7 @@ def main(argument_list=None):
   result = runUnitTestList(test_list=args,
                            verbosity=verbosity,
                            debug=debug,
+                           xunit_file=xunit_file,
                            run_only=run_only,
                            )
   return result and not result.wasSuccessful()

product/ERP5Type/tests/xunit.py

+"""This plugin provides test results in the standard XUnit XML format.
+
+It's designed for the `Jenkins`_ (previously Hudson) continuous build
+system, but will probably work for anything else that understands an
+XUnit-formatted XML representation of test results.
+
+Add this shell command to your builder ::
+
+    nosetests --with-xunit
+
+And by default a file named nosetests.xml will be written to the
+working directory.
+
+In a Jenkins builder, tick the box named "Publish JUnit test result report"
+under the Post-build Actions and enter this value for Test report XMLs::
+
+    **/nosetests.xml
+
+If you need to change the name or location of the file, you can set the
+``--xunit-file`` option.
+
+Here is an abbreviated version of what an XML test report might look like::
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <testsuite name="nosetests" tests="1" errors="1" failures="0" skip="0">
+        <testcase classname="path_to_test_suite.TestSomething"
+                  name="test_it" time="0">
+            <error type="exceptions.TypeError" message="oops, wrong type">
+            Traceback (most recent call last):
+            ...
+            TypeError: oops, wrong type
+            </error>
+        </testcase>
+    </testsuite>
+
+.. _Jenkins: http://jenkins-ci.org/
+
+"""
+import codecs
+import doctest
+import os
+import traceback
+import re
+import inspect
+from time import time
+from xml.sax import saxutils
+
+from backportUnittest import SkipTest
+UNICODE_STRINGS = False
+
+# Invalid XML characters, control characters 0-31 sans \t, \n and \r
+CONTROL_CHARACTERS = re.compile(r"[\000-\010\013\014\016-\037]")
+
+TEST_ID = re.compile(r'^(.*?)(\(.*\))$')
+
+def xml_safe(value):
+    """Replaces invalid XML characters with '?'."""
+    return CONTROL_CHARACTERS.sub('?', value)
+
+def escape_cdata(cdata):
+    """Escape a string for an XML CDATA section."""
+    return xml_safe(cdata).replace(']]>', ']]>]]&gt;<![CDATA[')
+
+def id_split(idval):
+    m = TEST_ID.match(idval)
+    if m:
+        name, fargs = m.groups()
+        head, tail = name.rsplit(".", 1)
+        return [head, tail+fargs]
+    else:
+        return idval.rsplit(".", 1)
+
+def nice_classname(obj):
+    """Returns a nice name for class object or class instance.
+
+        >>> nice_classname(Exception()) # doctest: +ELLIPSIS
+        '...Exception'
+        >>> nice_classname(Exception) # doctest: +ELLIPSIS
+        '...Exception'
+
+    """
+    if inspect.isclass(obj):
+        cls_name = obj.__name__
+    else:
+        cls_name = obj.__class__.__name__
+    mod = inspect.getmodule(obj)
+    if mod:
+        name = mod.__name__
+        # jython
+        if name.startswith('org.python.core.'):
+            name = name[len('org.python.core.'):]
+        return "%s.%s" % (name, cls_name)
+    else:
+        return cls_name
+
+def exc_message(exc_info):
+    """Return the exception's message."""
+    exc = exc_info[1]
+    if exc is None:
+        # str exception
+        result = exc_info[0]
+    else:
+        try:
+            result = str(exc)
+        except UnicodeEncodeError:
+            try:
+                result = unicode(exc)
+            except UnicodeError:
+                # Fallback to args as neither str nor
+                # unicode(Exception(u'\xe6')) work in Python < 2.6
+                result = exc.args[0]
+    return xml_safe(result)
+
+class Xunit(object):
+    """This plugin provides test results in the standard XUnit XML format."""
+    name = 'xunit'
+    score = 2000
+    encoding = 'UTF-8'
+    error_report_file = None
+
+    def _timeTaken(self):
+        if hasattr(self, '_timer'):
+            taken = time() - self._timer
+        else:
+            # test died before it ran (probably error in setup())
+            # or success/failure added before test started probably 
+            # due to custom TestResult munging
+            taken = 0.0
+        return taken
+
+    def _quoteattr(self, attr):
+        """Escape an XML attribute. Value can be unicode."""
+        attr = xml_safe(attr)
+        if isinstance(attr, unicode) and not UNICODE_STRINGS:
+            attr = attr.encode(self.encoding)
+        return saxutils.quoteattr(attr)
+
+    def options(self, parser, env):
+        """Sets additional command line options."""
+        Plugin.options(self, parser, env)
+        parser.add_option(
+            '--xunit-file', action='store',
+            dest='xunit_file', metavar="FILE",
+            default=env.get('NOSE_XUNIT_FILE', 'nosetests.xml'),
+            help=("Path to xml file to store the xunit report in. "
+                  "Default is nosetests.xml in the working directory "
+                  "[NOSE_XUNIT_FILE]"))
+
+    def __init__(self, xunit_file):
+        """ We don't call configure.
+        """
+        self.stats = {'errors': 0,
+                      'failures': 0,
+                      'passes': 0,
+                      'skipped': 0
+                      }
+        self.errorlist = []
+        self.error_report_file = codecs.open(xunit_file, 'w',
+                                             self.encoding, 'replace')
+
+    def configure(self, options, config):
+        """Configures the xunit plugin."""
+        Plugin.configure(self, options, config)
+        self.config = config
+        if self.enabled:
+            self.stats = {'errors': 0,
+                          'failures': 0,
+                          'passes': 0,
+                          'skipped': 0
+                          }
+            self.errorlist = []
+            self.error_report_file = codecs.open(options.xunit_file, 'w',
+                                                 self.encoding, 'replace')
+
+    def report(self, stream):
+        """Writes an Xunit-formatted XML file
+
+        The file includes a report of test errors and failures.
+
+        """
+        self.stats['encoding'] = self.encoding
+        self.stats['total'] = (self.stats['errors'] + self.stats['failures']
+                               + self.stats['passes'] + self.stats['skipped'])
+        self.error_report_file.write(
+            u'<?xml version="1.0" encoding="%(encoding)s"?>'
+            u'<testsuite name="nosetests" tests="%(total)d" '
+            u'errors="%(errors)d" failures="%(failures)d" '
+            u'skip="%(skipped)d">' % self.stats)
+        self.error_report_file.write(u''.join([self._forceUnicode(e)
+                                               for e in self.errorlist]))
+        self.error_report_file.write(u'</testsuite>')
+        self.error_report_file.close()
+        if 1:
+            stream.writeln("-" * 70)
+            stream.writeln("XML: %s" % self.error_report_file.name)
+
+    def startTest(self, test):
+        """Initializes a timer before starting a test."""
+        self._timer = time()
+
+    def addError(self, test, err, capt=None):
+        """Add error output to Xunit report.
+        """
+        taken = self._timeTaken()
+
+        if issubclass(err[0], SkipTest):
+            type = 'skipped'
+            self.stats['skipped'] += 1
+        else:
+            type = 'error'
+            self.stats['errors'] += 1
+        tb = ''.join(traceback.format_exception(*err))
+        id = test.id()
+        self.errorlist.append(
+            '<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
+            '<%(type)s type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>'
+            '</%(type)s></testcase>' %
+            {'cls': self._quoteattr(id_split(id)[0]),
+             'name': self._quoteattr(id_split(id)[-1]),
+             'taken': taken,
+             'type': type,
+             'errtype': self._quoteattr(nice_classname(err[0])),
+             'message': self._quoteattr(exc_message(err)),
+             'tb': escape_cdata(tb),
+             })
+
+    def addFailure(self, test, err, capt=None, tb_info=None):
+        """Add failure output to Xunit report.
+        """
+        taken = self._timeTaken()
+        tb = ''.join(traceback.format_exception(*err))
+        self.stats['failures'] += 1
+        id = test.id()
+        self.errorlist.append(
+            '<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
+            '<failure type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>'
+            '</failure></testcase>' %
+            {'cls': self._quoteattr(id_split(id)[0]),
+             'name': self._quoteattr(id_split(id)[-1]),
+             'taken': taken,
+             'errtype': self._quoteattr(nice_classname(err[0])),
+             'message': self._quoteattr(exc_message(err)),
+             'tb': escape_cdata(tb),
+             })
+
+    def addSuccess(self, test, capt=None):
+        """Add success output to Xunit report.
+        """
+        taken = self._timeTaken()
+        self.stats['passes'] += 1
+        id = test.id()
+        self.errorlist.append(
+            '<testcase classname=%(cls)s name=%(name)s '
+            'time="%(taken).3f" />' %
+            {'cls': self._quoteattr(id_split(id)[0]),
+             'name': self._quoteattr(id_split(id)[-1]),
+             'taken': taken,
+             })
+
+    def _forceUnicode(self, s):
+        if not UNICODE_STRINGS:
+            if isinstance(s, str):
+                s = s.decode(self.encoding, 'replace')
+        return s