run-zelenium-test.py.in 10.1 KB
Newer Older
1
#!{{ bin_path }}
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically

import argparse, os, sys, traceback
from erp5.util import taskdistribution
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from urllib import urlopen

def main():
  parser = argparse.ArgumentParser(description='Run a test suite.')
Xiaowu Zhang's avatar
Xiaowu Zhang committed
18
  parser_configuration = {{ repr(configuration) }}
19 20 21 22 23 24 25 26 27
  parser.add_argument('--test_suite', help='The test suite name')
  parser.add_argument('--test_suite_title', help='The test suite title')
  parser.add_argument('--test_node_title', help='The test node title')
  parser.add_argument('--project_title', help='The project title')
  parser.add_argument('--revision', help='The revision to test',
                      default='dummy_revision')
  parser.add_argument('--node_quantity', help='ignored', type=int)
  parser.add_argument('--master_url',
                      help='The Url of Master controling many suites')
28 29 30
  parser.add_argument('--remote_access_url',
                      help='The access url',
                      default = parser_configuration.get('remote-access-url')
Xiaowu Zhang's avatar
Xiaowu Zhang committed
31
                     )
32 33
  parser.add_argument('--target',
                      help='Target OS to run tests on',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
34 35
                      default = parser_configuration.get('target')
                     )
36 37
  parser.add_argument('--target_version',
                      help='Target OS version to use',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
38 39
                      default = parser_configuration.get('target-version')
                     )
40 41
  parser.add_argument('--target_browser',
                      help='The desired browser of the target OS to be used. Example: Firefox if target is Android.',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
42 43
                      default = parser_configuration.get('target-browser')
                     )
44 45
  parser.add_argument('--target_device',
                      help='The desired device running the target OS. Example: iPad Simulator, if target is iOS.',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
46 47
                      default = parser_configuration.get('target-device')
                     )
48 49
  parser.add_argument('--appium_server_auth',
                      help='Combination of user and token to access SauceLabs service. (i.e. user:token)',
Xiaowu Zhang's avatar
Xiaowu Zhang committed
50 51 52 53 54 55
                      default = parser_configuration.get('appium-server-auth')
                     )
  parser.add_argument('--run_only',
                      help='zuite to run',
                      default = parser_configuration.get('run-only')
                     )
56 57 58 59 60
  parser.add_argument('--max_duration',
                      type=int,
                      help='max duration for running test en second',
                      default = parser_configuration.get('max-duration', 1800)
                     )
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
  args = parser.parse_args()
  test_line_dict = {}
  test_suite_title = args.test_suite_title or args.test_suite
  test_suite = args.test_suite
  revision = args.revision

  # curl https://saucelabs.com/rest/v1/info/platforms/all
  # https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/
  if args.target in ['iOS', 'Android']:
    # parameters for mobile emulators have different names then parameters for
    # desktop OSes
    capabilities = {
      'platformName': args.target,
      'platformVersion': args.target_version,
      'deviceName': args.target_device,
76
      'browserName': args.target_browser,
77 78
      'maxDuration': args.max_duration,
      'name': test_suite_title
79 80 81 82 83
    }
  elif 'Windows' in args.target or 'OS X' in args.target:
    capabilities = {
      'browserName': args.target_browser,
      'platform': args.target,
84
      'version': args.target_version,
85 86
      'maxDuration': args.max_duration,
      'name': test_suite_title
87 88
    }

89
  if not args.appium_server_auth or not args.remote_access_url:
Xiaowu Zhang's avatar
Xiaowu Zhang committed
90
    sys.exit(-1)
91

92 93
  #Authentication over HTTPS is not supported in saucelab
  #https://wiki.saucelabs.com/display/DOCS/Instant+Selenium+Python+Tests
94
  appium_url = "http://%s@ondemand.saucelabs.com/wd/hub" % (args.appium_server_auth)
95
  # adjust make path to access url
96 97 98 99 100 101 102
  # Do not store any test result in the ZMI
  if args.run_only:
    url = "%s/erp5/portal_tests/%s/core/TestRunner.html" \
      "?test=../test_suite_html" \
      "&auto=on" \
      "&resultsUrl=../getId" \
      "&__ac_name=%s" \
103
      "&__ac_password=%s" % (args.remote_access_url, args.run_only, {{ repr(user) }}, {{ repr(password) }})
104 105 106 107 108 109
  else:
    url = "%s/erp5/portal_tests/core/TestRunner.html" \
      "?test=../test_suite_html" \
      "&auto=on" \
      "&resultsUrl=../getId" \
      "&__ac_name=%s" \
110
      "&__ac_password=%s" % (args.remote_access_url, {{ repr(user) }}, {{ repr(password) }})
111 112
    
  # Wait until all activities are finished...
113
  wait_url = args.remote_access_url + '/erp5/Zuite_waitForActivities'
114 115 116 117 118 119 120 121 122 123 124 125 126
  while 1:
    try:
      response = urlopen(wait_url)
      try:
        if response.code == 500:
          sys.exit(-1)
        if response.code == 200 and response.read() == 'Done.':
          break
      finally:
        response.close()
    except Exception:
      traceback.print_exc()
    time.sleep(10)
127
 
128
  tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
Xiaowu Zhang's avatar
Xiaowu Zhang committed
129 130
  browser = webdriver.Remote(appium_url, capabilities)
  
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
  try:
    agent = browser.execute_script("return navigator.userAgent")
    print url
    print agent
    start_time = time.time()
    browser.get(url)
    # Wait for Zelenium to be loaded
    WebDriverWait(browser, 10).until(EC.presence_of_element_located((
      By.XPATH, '//iframe[@id="testSuiteFrame"]'
    )))
    # XXX No idea how to wait for the iframe content to be loaded
    time.sleep(5)
    # Count number of test to be executed
    test_count = browser.execute_script(
      "return document.getElementById('testSuiteFrame').contentDocument.querySelector('tbody').children.length"
    ) - 1 # First child is the file name

    # Wait for test to be executed
149 150 151 152 153 154 155 156 157 158
    while 1:
      try:
        WebDriverWait(browser, args.max_duration).until(EC.presence_of_element_located((
          By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count
        )))
        break
      except TimeoutException as error:
        raise error
      except Exception:
        traceback.print_exc()
159 160 161 162 163 164 165 166 167 168 169 170
  except:
    test_line_dict['UnexpectedException'] = {
      'test_count': 1,
      'error_count': 0,
      'failure_count': 1,
      'skip_count': 0,
      'duration': 1,
      'command': url,
      'stdout': agent,
      'stderr': traceback.format_exc()
    }
  finally:
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
    execution_duration = round(time.time() - start_time, 2)
    if test_count:
      test_execution_duration = execution_duration / test_count
    else:
      test_execution_duration = 0
    html_parser = etree.HTMLParser(recover=True)

    # body = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
    # test_count = int(body.xpath('//td[@id="testRuns"]')[0].text)
    # failed_test_count = int(body.xpath('//td[@id="testFailures"]')[0].text)
    # print 'Run %i, failed %i' % (test_count, failed_test_count)

    # https://github.com/appium/appium/issues/5199
    # browser.switch_to.frame(browser.find_element_by_id("testSuiteFrame"))
    # iframe = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
    iframe = etree.fromstring(
      browser.execute_script(
        "return document.getElementById('testSuiteFrame').contentDocument.querySelector('html').innerHTML"
      ).encode('UTF-8'),
      html_parser
    )
Xiaowu Zhang's avatar
Xiaowu Zhang committed
192
    # parse test result
193 194 195 196
    tbody = iframe.xpath('.//body/table/tbody')[0]
    for tr in tbody[1:]:
      # First td is the main title
      test_name = tr[0][0].text
197 198 199 200 201 202 203
      if len(tr) == 1:
        test_line_dict[test_name] = {
        'test_count': 1,
        'error_count': 0,
        'failure_count': 1,
        'skip_count': 0,
        'duration': 0,
Xiaowu Zhang's avatar
Xiaowu Zhang committed
204 205 206
        'command': url,
        'stdout': agent,
        'stderr': '',
207
        'html_test_result': 'Not Executed due to saucelabs related error'
Xiaowu Zhang's avatar
Xiaowu Zhang committed
208
      }
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
      else:
        skip_count = success_count = error_count = 0
        test_table = tr[1].xpath('.//table')[0]
        test_tbody = tr[1].xpath('.//tbody')[0]

        tr_count = len(test_tbody)
        for tr in test_tbody:
          # print etree.tostring(tr).split('\n')[0]
          status = tr.attrib.get('class')
          if status is None or 'status_done' in status:
            skip_count += 1
          elif 'status_passed' in status:
            success_count += 1
          elif 'status_failed' in status:
            error_count += 1

        test_line_dict[test_name] = {
          'test_count': tr_count,
          'error_count': error_count,
          'failure_count': tr_count - (skip_count + success_count + error_count),
          'skip_count': skip_count,
          'duration': test_execution_duration,
          'command': url,
          'stdout': agent,
          'stderr': '',
          'html_test_result': etree.tostring(test_table)
        }
Xiaowu Zhang's avatar
Xiaowu Zhang committed
236
    browser.quit()
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259

  try:
    test_result = tool.createTestResult(revision = revision,
                                        test_name_list = test_line_dict.keys(),
                                        node_title = args.test_node_title,
                                        test_title = test_suite_title,
                                        project_title = args.project_title)
    if test_result is None or not hasattr(args, 'master_url'):
      return
    # report test results
    while 1:
      test_result_line = test_result.start()
      if not test_result_line:
        print 'No test result anymore.'
        break

      print 'Submitting: "%s"' % test_result_line.name
      # report status back to Nexedi ERP5
      test_result_line.stop(**test_line_dict[test_result_line.name])

  except:
    # Catch any exception here, to warn user instead of being silent,
    # by generating fake error result
Xiaowu Zhang's avatar
Xiaowu Zhang committed
260
    traceback.print_exc()
261 262 263 264 265 266 267 268 269
    result = dict(status_code=-1,
                  command=url,
                  stderr=traceback.format_exc(),
                  stdout='')
    # XXX: inform test node master of error
    raise EnvironmentError(result)

if __name__ == "__main__":
    main()