[HARDCODED][stack erp5] add a runZeleniumTest script

To run zelenium tests on saucelabs
Desactivate usuable runTestSuite

stack/erp5/buildout.cfg

@@ -153,6 +153,13 @@ url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-imp
 md5sum = b32d9ee1cb85f85d8d2f2b58f58459f1
 mode = 755
 
+[template-run-zelenium]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template/run-zelenium-test.py.in
+md5sum = b78c7f5db0e9c9de9b96ac600da675a6
+output = ${buildout:directory}/run-zelenium-test.py.in
+mode = 755
+
 [template-mariadb]
 <= download-base
 filename = instance-mariadb.cfg.in
@@ -234,7 +241,7 @@ recipe = slapos.recipe.template:jinja2
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 rendered = ${buildout:directory}/template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 29e60884bb09ba5461a7c4717d67877e
+md5sum = 90916b88414f5a02a535e35c776cbbc9
 mode = 640
 context =
     key mariadb_link_binary template-mariadb:link-binary
@@ -312,6 +319,7 @@ context =
     key template_zeo template-zeo:target
     key template_zope template-zope:target
     key template_zope_conf template-zope-conf:target
+    key template_run_zelenium template-run-zelenium:output
     key userhosts_location userhosts:location
     key wget_location wget:location
     key xdamage_location xdamage:location
@@ -331,7 +339,7 @@ rendered = ${monitor-template-dummy:target}
 [template-erp5]
 <= download-base
 filename = instance-erp5.cfg.in
-md5sum = b5f49c90017684aa3389ef3f97ece509
+md5sum = bd7d240bc8c1edf01cb717e5bba74a61
 
 [template-zeo]
 <= download-base
@@ -341,7 +349,7 @@ md5sum = b0cb0ee97cddc79112a718e065806037
 [template-zope]
 <= download-base
 filename = instance-zope.cfg.in
-md5sum = 83304f1940e0d7555bd678e2cea4c738
+md5sum = 49ccb2e83e0ed55b616c6d95ef5d810e
 link-binary =
   ${aspell:location}/bin/aspell
   ${dmtx-utils:location}/bin/dmtxwrite
@@ -565,6 +573,7 @@ eggs =
 
 # Needed for parsing .po files from our Localizer subset
   polib
+  selenium
 
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the
@@ -825,3 +834,4 @@ zope.app.dependable = 3.5.1
 # Products.CMFCalendar==2.2.3
 # five.formlib==1.0.4
 zope.app.form = 4.0.2
+selenium = 2.53.1

stack/erp5/instance-erp5.cfg.in

@@ -135,6 +135,7 @@ config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url'
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
+config-saucelabs-dict = {{ dumps(slapparameter_dict.get('saucelabs-dict', {})) }}
 config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
 config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
 config-inituser-login = {{ dumps(inituser_login) }}

stack/erp5/instance-zope.cfg.in

@@ -2,6 +2,7 @@
 {% set next_port = slapparameter_dict['port-base'] -%}
 {% set site_id = slapparameter_dict['site-id'] -%}
 {% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
+{% set saucelabs_dict = slapparameter_dict.get('saucelabs-dict', None) -%}
 {% set node_id_base = slapparameter_dict['name'] -%}
 {% set part_list = [] -%}
 {% set publish_list = [] -%}
@@ -393,6 +394,25 @@ monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ next_port }}
 monitor-title = Zope monitor
 
+{% if saucelabs_dict -%}
+[test-zelenium-runner-parameter]
+configuration = {{ dumps(saucelabs_dict) }}
+user = {{ dumps(slapparameter_dict['inituser-login']) }}
+password = {{ dumps(slapparameter_dict['inituser-password']) }}
+
+[{{ section('test-zelenium-runner') }}]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['run-zelenium-template'] }}
+# XXX Surcharged normal runTestSuite brutally
+rendered = ${directory:bin}/runTestSuite
+extensions = jinja2.ext.do
+context =
+    import json_module json
+    key configuration test-zelenium-runner-parameter:configuration
+    key user test-zelenium-runner-parameter:user
+    key password test-zelenium-runner-parameter:password
+{%- endif %}
+
 [buildout]
 extends =
   {{ logrotate_cfg }}

stack/erp5/instance.cfg.in

@@ -115,6 +115,7 @@ extra-context =
 [dynamic-template-zope-parameters]
 bin-directory = {{ bin_directory }}
 zope-conf-template = {{ template_zope_conf }}
+run-zelenium-template = {{ template_run_zelenium }}
 buildout-bin-directory = {{ buildout_bin_directory }}
 6tunnel = {{ sixtunnel_location }}
 coreutils = {{ coreutils_location }}

stack/erp5/template/run-zelenium-test.py.in

+#!${buildout:directory}/bin/${eggs:interpreter}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+import argparse, os, sys, traceback
+from erp5.util import taskdistribution
+import time
+from lxml import etree
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import time
+import json
+
+
+def main():
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', help='ignored', type=int)
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites')
+  parser.add_argument('--frontend_url',
+                      help='The url of frontend of the test suite')
+  parser.add_argument('--target',
+                      help='Target OS to run tests on',
+                      type=str)
+  parser.add_argument('--target_version',
+                      help='Target OS version to use',
+                      type=str,)
+  parser.add_argument('--target_browser',
+                      help='The desired browser of the target OS to be used. Example: Firefox if target is Android.',
+                      type=str,)
+  parser.add_argument('--target_device',
+                      help='The desired device running the target OS. Example: iPad Simulator, if target is iOS.',
+                      type=str,)
+  parser.add_argument('--appium_server_auth',
+                      help='Combination of user and token to access SauceLabs service. (i.e. user:token)',
+                      type=str)
+
+  args = parser.parse_args()
+
+  parsed_parameters = json.loads('{{ json_module.dumps(configuration) }}')
+
+
+  if not getattr(args, 'target', None):
+    args.target = parsed_parameters.get('target', 'firefox')
+  if not getattr(args, 'test_suite', None):
+    args.test_suite = parsed_parameters.get('test-suite')
+  if not getattr(args, 'target_version', None):
+    args.target_version = parsed_parameters.get('target-version')
+  if not getattr(args, 'appium_server_auth', None):
+    args.appium_server_auth = parsed_parameters.get('appium-server-auth')
+  if not getattr(args, 'target_browser', None):
+    args.target_browser = parsed_parameters.get('target-browser')
+  if not getattr(args, 'target_device', None):
+    args.target_device = parsed_parameters.get('target-device')
+  if not getattr(args, 'frontend_url', None):
+    args.frontend_url = parsed_parameters.get('frontend-url')
+
+  is_browser_running = False
+  test_line_dict = {}
+  test_suite_title = args.test_suite_title or args.test_suite
+  test_suite = args.test_suite
+  revision = args.revision
+
+  # curl https://saucelabs.com/rest/v1/info/platforms/all
+  # https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/
+  if args.target in ['iOS', 'Android']:
+    # parameters for mobile emulators have different names then parameters for
+    # desktop OSes
+    capabilities = {
+      'platformName': args.target,
+      'platformVersion': args.target_version,
+      'deviceName': args.target_device,
+      'browserName': args.target_browser
+    }
+  elif 'Windows' in args.target or 'OS X' in args.target:
+    capabilities = {
+      'browserName': args.target_browser,
+      'platform': args.target,
+      'version': args.target_version
+    }
+
+  if not args.appium_server_auth:
+    raise RuntimeError('--appium_server_auth is required.')
+  if not args.frontend_url:
+    raise RuntimeError('--frontend_url is required.')
+
+  appium_url = "http://%s@ondemand.saucelabs.com/wd/hub" % (args.appium_server_auth)
+
+  # adjust make path to frontend
+  # Do not store any test result in the ZMI
+  url = "%s/erp5/portal_tests/core/TestRunner.html" \
+    "?test=../test_suite_html" \
+    "&auto=on" \
+    "&resultsUrl=../getId" \
+    "&__ac_name=%s" \
+    "&__ac_password=%s" % (args.frontend_url, "{{ user }}", "{{ password }}")
+
+  # Wait until all bt5 are installed...
+  time.sleep(120)
+
+  try:
+    browser = webdriver.Remote(appium_url, capabilities)
+    is_browser_running = True
+
+    agent = browser.execute_script("return navigator.userAgent")
+    print url
+    print agent
+
+    start_time = time.time()
+
+    browser.get(url)
+
+    # Wait for Zelenium to be loaded
+    WebDriverWait(browser, 10).until(EC.presence_of_element_located((
+      By.XPATH, '//iframe[@id="testSuiteFrame"]'
+    )))
+    # XXX No idea how to wait for the iframe content to be loaded
+    time.sleep(5)
+    # Count number of test to be executed
+    test_count = browser.execute_script(
+      "return document.getElementById('testSuiteFrame').contentDocument.querySelector('tbody').children.length"
+    ) - 1 # First child is the file name
+
+    # Wait for test to be executed
+    erp5_zelenium_test_timeout = 90
+    WebDriverWait(browser, erp5_zelenium_test_timeout * (test_count + 1)).until(EC.presence_of_element_located((
+      By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count
+    )))
+    execution_duration = round(time.time() - start_time, 2)
+    if test_count:
+      test_execution_duration = execution_duration / test_count
+    else:
+      test_execution_duration = 0
+    html_parser = etree.HTMLParser(recover=True)
+
+    # body = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
+    # test_count = int(body.xpath('//td[@id="testRuns"]')[0].text)
+    # failed_test_count = int(body.xpath('//td[@id="testFailures"]')[0].text)
+    # print 'Run %i, failed %i' % (test_count, failed_test_count)
+
+    # https://github.com/appium/appium/issues/5199
+    # browser.switch_to.frame(browser.find_element_by_id("testSuiteFrame"))
+    # iframe = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
+    iframe = etree.fromstring(
+      browser.execute_script(
+        "return document.getElementById('testSuiteFrame').contentDocument.querySelector('html').innerHTML"
+      ).encode('UTF-8'),
+      html_parser
+    )
+
+    browser.quit()
+    is_browser_running = False
+
+    tbody = iframe.xpath('.//body/table/tbody')[0]
+    tr_count = 0
+    for tr in tbody:
+      if tr_count:
+        # First td is the main title
+        test_name = tr[0][0].text
+
+        skip_count = success_count = error_count = 0
+        if len(tr) == 1:
+          # Test was not executed
+          tr_count = 1
+          test_table = 'Test not executed!'
+          test_tbody = 'Test not executed!'
+        else:
+          test_table = tr[1].xpath('.//table')[0]
+          test_tbody = tr[1].xpath('.//tbody')[0]
+
+          tr_count = len(test_tbody)
+          for tr in test_tbody:
+            # print etree.tostring(tr).split('\n')[0]
+            status = tr.attrib.get('class')
+            if 'status_done' in status:
+              skip_count += 1
+            elif 'status_passed' in status:
+              success_count += 1
+            elif 'status_failed' in status:
+              error_count += 1
+
+        test_line_dict[test_name] = {
+          'test_count': tr_count,
+          'error_count': error_count,
+          'failure_count': tr_count - (skip_count + success_count + error_count),
+          'skip_count': skip_count,
+          'duration': test_execution_duration,
+          'command': url,
+          'stdout': agent,
+          'stderr': '',
+          'html_test_result': etree.tostring(test_table)
+        }
+
+      tr_count += 1
+
+
+    tool = taskdistribution.TaskDistributionTool(portal_url=args.master_url)
+    test_result = tool.createTestResult(revision = revision,
+                                        test_name_list = test_line_dict.keys(),
+                                        node_title = args.test_node_title,
+                                        test_title = test_suite_title,
+                                        project_title = args.project_title)
+    if test_result is None or not hasattr(args, 'master_url'):
+      return
+    # report test results
+    while 1:
+      test_result_line = test_result.start()
+      if not test_result_line:
+        print 'No test result anymore.'
+        break
+
+      print 'Submitting: "%s"' % test_result_line.name
+      # report status back to Nexedi ERP5
+      test_result_line.stop(**test_line_dict[test_result_line.name])
+
+
+  except:
+    # Catch any exception here, to warn user instead of being silent,
+    # by generating fake error result
+    print traceback.format_exc()
+    result = dict(status_code=-1,
+                  command=url,
+                  stderr=traceback.format_exc(),
+                  stdout='')
+    # XXX: inform test node master of error
+    raise EnvironmentError(result)
+  
+  finally:
+    if is_browser_running:
+      # if by any chance browser is still running due to
+      # traceback raised make sure we cleanup
+      browser.quit()
+
+
+if __name__ == "__main__":
+    main()