stack/erp5: support coverage when running tests

This replaces the broken --coverage argument from runUnitTest, coverage
needs to be started earlier and also introduces a coverage plugin to
collect coverage data for code in ZODB; python scripts and components
are supported.

To use on test nodes, set up a web dav server somewhere, configure the
instance parameters of the test suite on ERP5 to enable coverage and
upload individual results to this webdav server and then combine the
coverage data and produce a report, using the bin/coverage script from
the software release.

For the steps below, it is necessary to change working directory to the
root of the software folder.

Step 1: combine the coverage data:

    ./bin/coverage combine \
      --keep \
      /path/to/all/coverage/files/*coverage.sqlite3

( using --keep is optional, but it helps in case of mistakes )

Step 2: build an html report:

   ./bin/coverage html \
     --skip-covered \
     --omit parts/erp5/product/ERP5/Document/UnitTest.py \
     --directory /path/for/html/report/

Note that we want to omit UnitTest.py that is created during test
(see testBusinessTemplate.TestDocumentTemplateItem) and get coverage
result because it is executed in the test, but it does not exist as a
source file in the repository, so we skip it.

Of course, to produce a correct html report from a test that have been
running on test nodes, it's necessary that the software release used to
produce the html report has exactly the same version ans the one that
has been running on test nodes.

Another simpler, but slower approach is to run all the tests on the same
machine, then only running step 2 is necessary.

software/erp5/instance-erp5-input-schema.json

@@ -525,6 +525,46 @@
           "default": true,
           "type": "boolean"
         },
+        "coverage": {
+          "type": "object",
+          "title": "Coverage",
+          "description": "Coverage configuration",
+          "additionalProperties": false,
+          "properties": {
+            "enabled": {
+              "description": "Collect python coverage data during test run.",
+              "default": false,
+              "type": "boolean"
+            },
+            "include": {
+              "description": "File name patterns to include in coverage data, relative to software buildout's directory. Default to all repositories defined in software by ${erp5_repository_list:repository_id_list}.",
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "examples": [
+                [
+                  "parts/erp5/*",
+                  "parts/custom-repository/*",
+                  "develop-eggs/custom-egg/*"
+                ]
+              ]
+            },
+            "branch": {
+              "description": "Enable branch coverage",
+              "type": "boolean",
+              "default": false
+            },
+            "upload-url": {
+              "description": "URL to upload coverage data. This is interpreted as a RFC 6570 URI Template, with the following parameters: test_name, test_result_id and test_result_revision. The request will be a PUT request with the coverage file content as body, suitable for WebDav servers. If the URL contains user and password, they will be used to attempt authentication using Digest and Basic authentication schemes.",
+              "type": "string",
+              "format": "uri",
+              "examples": [
+                "https://user:password@example.com/{test_result_id}/{test_name}.coverage.sqlite3"
+              ]
+            }
+          }
+        },
         "node-count": {
           "description": "Number of tests this instance can execute in parallel. This must be at least equal to the number of nodes configured on testnode running the test",
           "default": 3,

stack/erp5/buildout.cfg

@@ -72,6 +72,7 @@ parts +=
   mroonga-mariadb
   tesseract
   zabbix-agent
+  .coveragerc
 
 # Buildoutish
   eggs-all-scripts
@@ -351,14 +352,52 @@ entry-points =
   runUnitTest=runUnitTest:main
 scripts = runUnitTest
 initialization =
-  import glob, os, sys
+  import glob, os, sys, json
+  buildout_directory = '''${buildout:directory}'''
+  parts_directory = '''${buildout:parts-directory}'''
+  repository_id_list = \
+    '''${erp5_repository_list:repository_id_list}'''.split()[::-1]
+
+  # read testrunner configuration from slapos instance parameters to
+  # configure coverage if enabled.
+  with open(os.environ['ERP5_TEST_RUNNER_CONFIGURATION']) as f:
+    test_runner_configuration = json.load(f)
+  test_runner_configuration.setdefault('coverage', {})
+  test_runner_configuration['coverage'].setdefault('enabled', False)
+
+  coverage_process = None
+  if test_runner_configuration['coverage']['enabled']:
+    test_runner_configuration['coverage'].setdefault(
+      'include', [os.path.join('parts', repo, '*') for repo in repository_id_list])
+
+    assets_directory = ''
+    test_name = sys.argv[-1].replace(':', '_')
+    if os.environ.get('SLAPOS_TEST_LOG_DIRECTORY'):
+      assets_directory = os.path.join(os.environ['SLAPOS_TEST_LOG_DIRECTORY'], test_name)
+      if not os.path.exists(assets_directory):
+        os.makedirs(assets_directory)
+    coverage_data_file = os.path.abspath(
+      os.path.join(assets_directory, 'coverage.sqlite3'))
+
+    curdir = os.path.abspath(os.curdir)
+    # change current directory when importing coverage so that it considers paths
+    # relative to the root of the software
+    os.chdir(buildout_directory)
+    import coverage
+    coverage_process = coverage.Coverage(
+      include=test_runner_configuration['coverage']['include'],
+      data_file=coverage_data_file,
+      branch=test_runner_configuration['coverage'].get('branch'),
+    )
+    coverage_process.set_option('run:relative_files', 'true')
+    coverage_process.set_option('run:plugins', ['erp5_coverage_plugin'])
+    coverage_process.start()
+    os.chdir(curdir)
+
   import Products
   Products.__path__[:0] = filter(None,
     os.getenv('INSERT_PRODUCTS_PATH', '').split(os.pathsep))
   os.environ['ZOPE_SCRIPTS'] = ''
-  parts_directory = '''${buildout:parts-directory}'''
-  repository_id_list = \
-    '''${erp5_repository_list:repository_id_list}'''.split()[::-1]
   os.environ['erp5_tests_bt5_path'] = ','.join(
     os.path.join(parts_directory, x, 'bt5') for x in repository_id_list)
   extra_path_list = '''${:extra-paths}'''.split()
@@ -371,6 +410,64 @@ initialization =
   sys.path[:0] = sum((
     glob.glob(os.path.join(x, 'Products', '*', 'tests'))
     for x in os.getenv('INSERT_PRODUCTS_PATH', '').split(os.pathsep)), [])
+  import runUnitTest
+  try:
+    sys.exit(runUnitTest.main())
+  finally:
+    if coverage_process:
+      coverage_process.stop()
+      coverage_process.save()
+      # Make html report for this test
+      os.chdir(buildout_directory)
+      coverage_process.html_report(
+        directory=os.path.join(assets_directory, 'coverage_report'),
+      )
+      # upload the coverage so that they can be combined from another machine
+      upload_url = test_runner_configuration['coverage'].get('upload-url')
+      if upload_url:
+        import requests
+        import time
+        import uritemplate
+        from six.moves.urllib.parse import urlparse
+
+        auth_list = (None, )
+        parsed_url = urlparse(upload_url)
+        if parsed_url.username:
+          # try Digest and Basic authentication and retry 5 times to tolerate transiant errors
+          auth_list = (
+            requests.auth.HTTPDigestAuth(parsed_url.username, parsed_url.password),
+            requests.auth.HTTPBasicAuth(parsed_url.username, parsed_url.password),
+          ) *  5
+
+        url = uritemplate.URITemplate(upload_url).expand(
+          test_name=test_name,
+          # Environment variables are set in parts/erp5/product/ERP5Type/tests/runTestSuite.py
+          test_result_id=os.environ.get('ERP5_TEST_RESULT_ID', 'unknown_test_result_id'),
+          test_result_revision=os.environ.get('ERP5_TEST_RESULT_REVISION', 'unknown_test_result_revision'),
+        )
+
+        for auth in auth_list:
+          with open(coverage_data_file, 'rb') as f:
+            resp = requests.put(url, data=f, auth=auth)
+            if resp.ok:
+              # print just the hostname, not to include the auth part
+              print('Uploaded coverage data to {parsed_url.hostname}'.format(parsed_url=parsed_url))
+              break
+            print('Error {resp.status_code} uploading coverage data to {parsed_url.hostname} with {auth.__class__.__name__}'.format(
+              resp=resp, parsed_url=parsed_url, auth=auth))
+            time.sleep(1)
+        else:
+          sys.stderr.write('Error uploading coverage data to {parsed_url.hostname}\n'.format(parsed_url=parsed_url))
+
+[.coveragerc]
+recipe = slapos.recipe.template
+output = ${buildout:directory}/${:_buildout_section_name_}
+inline =
+  # coverage configuration file, useful when making html report
+  [run]
+  plugins =
+    erp5_coverage_plugin
+  relative_files = true
 
 [test-suite-runner]
 # XXX: Workaround for fact ERP5Type is not an distribution and does not
@@ -466,7 +563,6 @@ eggs = ${neoppod:eggs}
   SOAPpy
   chardet
   collective.recipe.template
-  coverage
   erp5diff
   interval
   ipdb
@@ -606,6 +702,12 @@ Acquisition-patch-options = -p1
 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
 python-magic-patch-options = -p1
 
+
+# neoppod installs bin/coverage so we inject erp5 plugin here so that coverage script can use it in report
+[neoppod]
+eggs +=
+  erp5_coverage_plugin
+
 [eggs-all-scripts]
 recipe = zc.recipe.egg
 eggs =
@@ -683,6 +785,7 @@ PyStemmer = 1.3.0
 Pympler = 0.4.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
+erp5-coverage-plugin = 0.0.1
 erp5diff = 0.8.1.8
 five.formlib = 1.0.4
 five.localsitemanager = 2.0.5