run-zelenium-test.py.in 9.1 KB
Newer Older
1
#!{{ bin_path }}
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically

import argparse, os, sys, traceback
from erp5.util import taskdistribution
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from urllib import urlopen

def main():
  parser = argparse.ArgumentParser(description='Run a test suite.')
Xiaowu Zhang's avatar
Xiaowu Zhang committed
18
  parser_configuration = {{ repr(configuration) }}
19 20 21 22 23 24 25 26 27
  parser.add_argument('--test_suite', help='The test suite name')
  parser.add_argument('--test_suite_title', help='The test suite title')
  parser.add_argument('--test_node_title', help='The test node title')
  parser.add_argument('--project_title', help='The project title')
  parser.add_argument('--revision', help='The revision to test',
                      default='dummy_revision')
  parser.add_argument('--node_quantity', help='ignored', type=int)
  parser.add_argument('--master_url',
                      help='The Url of Master controling many suites')
28 29 30
  parser.add_argument('--remote_access_url',
                      help='The access url',
                      default = parser_configuration.get('remote-access-url')
Xiaowu Zhang's avatar
Xiaowu Zhang committed
31
                     )
32 33
  parser.add_argument('--target',
                      help='Target OS to run tests on',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
34 35
                      default = parser_configuration.get('target')
                     )
36 37
  parser.add_argument('--target_version',
                      help='Target OS version to use',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
38 39
                      default = parser_configuration.get('target-version')
                     )
40 41
  parser.add_argument('--target_browser',
                      help='The desired browser of the target OS to be used. Example: Firefox if target is Android.',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
42 43
                      default = parser_configuration.get('target-browser')
                     )
44 45
  parser.add_argument('--target_device',
                      help='The desired device running the target OS. Example: iPad Simulator, if target is iOS.',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
46 47
                      default = parser_configuration.get('target-device')
                     )
48 49
  parser.add_argument('--appium_server_auth',
                      help='Combination of user and token to access SauceLabs service. (i.e. user:token)',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
50 51 52 53 54 55
                      default = parser_configuration.get('appium-server-auth')
                     )
  parser.add_argument('--run_only',
                      help='zuite to run',
                      default = parser_configuration.get('run-only')
                     )
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
  args = parser.parse_args()
  test_line_dict = {}
  test_suite_title = args.test_suite_title or args.test_suite
  test_suite = args.test_suite
  revision = args.revision

  # curl https://saucelabs.com/rest/v1/info/platforms/all
  # https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/
  if args.target in ['iOS', 'Android']:
    # parameters for mobile emulators have different names then parameters for
    # desktop OSes
    capabilities = {
      'platformName': args.target,
      'platformVersion': args.target_version,
      'deviceName': args.target_device,
      'browserName': args.target_browser
    }
  elif 'Windows' in args.target or 'OS X' in args.target:
    capabilities = {
      'browserName': args.target_browser,
      'platform': args.target,
      'version': args.target_version
    }

80
  if not args.appium_server_auth or not args.remote_access_url:
Xiaowu Zhang's avatar
Xiaowu Zhang committed
81
    sys.exit(-1)
82 83

  appium_url = "http://%s@ondemand.saucelabs.com/wd/hub" % (args.appium_server_auth)
84
  # adjust make path to access url
85 86 87 88 89 90 91
  # Do not store any test result in the ZMI
  if args.run_only:
    url = "%s/erp5/portal_tests/%s/core/TestRunner.html" \
      "?test=../test_suite_html" \
      "&auto=on" \
      "&resultsUrl=../getId" \
      "&__ac_name=%s" \
92
      "&__ac_password=%s" % (args.remote_access_url, args.run_only, {{ repr(user) }}, {{ repr(password) }})
93 94 95 96 97 98
  else:
    url = "%s/erp5/portal_tests/core/TestRunner.html" \
      "?test=../test_suite_html" \
      "&auto=on" \
      "&resultsUrl=../getId" \
      "&__ac_name=%s" \
99
      "&__ac_password=%s" % (args.remote_access_url, {{ repr(user) }}, {{ repr(password) }})
100 101
    
  # Wait until all activities are finished...
102
  wait_url = args.remote_access_url + '/erp5/Zuite_waitForActivities'
103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
  while 1:
    try:
      response = urlopen(wait_url)
      try:
        if response.code == 500:
          sys.exit(-1)
        if response.code == 200 and response.read() == 'Done.':
          break
      finally:
        response.close()
    except Exception:
      traceback.print_exc()
    time.sleep(10)
  
  tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
Xiaowu Zhang's avatar
Xiaowu Zhang committed
118 119
  browser = webdriver.Remote(appium_url, capabilities)
  
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
  try:
    agent = browser.execute_script("return navigator.userAgent")
    print url
    print agent
    start_time = time.time()
    browser.get(url)
    # Wait for Zelenium to be loaded
    WebDriverWait(browser, 10).until(EC.presence_of_element_located((
      By.XPATH, '//iframe[@id="testSuiteFrame"]'
    )))
    # XXX No idea how to wait for the iframe content to be loaded
    time.sleep(5)
    # Count number of test to be executed
    test_count = browser.execute_script(
      "return document.getElementById('testSuiteFrame').contentDocument.querySelector('tbody').children.length"
    ) - 1 # First child is the file name

    # Wait for test to be executed
    erp5_zelenium_test_timeout = 90
    WebDriverWait(browser, erp5_zelenium_test_timeout * (test_count + 1)).until(EC.presence_of_element_located((
      By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count
    )))
    execution_duration = round(time.time() - start_time, 2)
    if test_count:
      test_execution_duration = execution_duration / test_count
    else:
      test_execution_duration = 0
    html_parser = etree.HTMLParser(recover=True)

    # body = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
    # test_count = int(body.xpath('//td[@id="testRuns"]')[0].text)
    # failed_test_count = int(body.xpath('//td[@id="testFailures"]')[0].text)
    # print 'Run %i, failed %i' % (test_count, failed_test_count)

    # https://github.com/appium/appium/issues/5199
    # browser.switch_to.frame(browser.find_element_by_id("testSuiteFrame"))
    # iframe = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
    iframe = etree.fromstring(
      browser.execute_script(
        "return document.getElementById('testSuiteFrame').contentDocument.querySelector('html').innerHTML"
      ).encode('UTF-8'),
      html_parser
    )
Xiaowu Zhang's avatar
Xiaowu Zhang committed
163
    # parse test result
164 165 166 167 168 169
    tbody = iframe.xpath('.//body/table/tbody')[0]
    for tr in tbody[1:]:
      # First td is the main title
      test_name = tr[0][0].text

      skip_count = success_count = error_count = 0
Xiaowu Zhang's avatar
Xiaowu Zhang committed
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
     
      test_table = tr[1].xpath('.//table')[0]
      test_tbody = tr[1].xpath('.//tbody')[0]

      tr_count = len(test_tbody)
      for tr in test_tbody:
        # print etree.tostring(tr).split('\n')[0]
        status = tr.attrib.get('class')
        if status is None or 'status_done' in status:
          skip_count += 1
        elif 'status_passed' in status:
          success_count += 1
        elif 'status_failed' in status:
          error_count += 1

      test_line_dict[test_name] = {
        'test_count': tr_count,
        'error_count': error_count,
        'failure_count': tr_count - (skip_count + success_count + error_count),
        'skip_count': skip_count,
        'duration': test_execution_duration,
        'command': url,
        'stdout': agent,
        'stderr': '',
        'html_test_result': etree.tostring(test_table)
      }
196 197 198 199 200 201 202 203 204 205 206
  except:
    test_line_dict['UnexpectedException'] = {
      'test_count': 1,
      'error_count': 0,
      'failure_count': 1,
      'skip_count': 0,
      'duration': 1,
      'command': url,
      'stdout': agent,
      'stderr': traceback.format_exc()
    }
Xiaowu Zhang's avatar
Xiaowu Zhang committed
207 208
  finally:
    browser.quit()
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231

  try:
    test_result = tool.createTestResult(revision = revision,
                                        test_name_list = test_line_dict.keys(),
                                        node_title = args.test_node_title,
                                        test_title = test_suite_title,
                                        project_title = args.project_title)
    if test_result is None or not hasattr(args, 'master_url'):
      return
    # report test results
    while 1:
      test_result_line = test_result.start()
      if not test_result_line:
        print 'No test result anymore.'
        break

      print 'Submitting: "%s"' % test_result_line.name
      # report status back to Nexedi ERP5
      test_result_line.stop(**test_line_dict[test_result_line.name])

  except:
    # Catch any exception here, to warn user instead of being silent,
    # by generating fake error result
Xiaowu Zhang's avatar
Xiaowu Zhang committed
232
    traceback.print_exc()
233 234 235 236 237 238 239 240 241
    result = dict(status_code=-1,
                  command=url,
                  stderr=traceback.format_exc(),
                  stdout='')
    # XXX: inform test node master of error
    raise EnvironmentError(result)

if __name__ == "__main__":
    main()