Commit a3af1eac authored by Kirill Smelkov's avatar Kirill Smelkov

Turn go/neo/t/nxd/runTestSuite into -> nxdtest

Make the tool generic: load list of testcases to run from .nxdtest file
and process the correspondingly. See added top-level documentation for
details.
parent 90410109
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018 Nexedi SA and Contributors.
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -17,21 +17,79 @@
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""runTestSuite - run neotest under Nexedi testing infrastructure.
"""nxdtest - tox-like tool to run tests under Nexedi testing infrastructure(*).
neotest must be on $PATH.
Nxdtest runs tests defined by .nxdtest file.
A project defines set of tests cases to verify itself in that file.
.nxdtest file is Python program executed in special environment described below.
Test cases are declared with `TestCase`. Each test case specifies:
- its name,
- a program to run,
- (optionally) an environment for the run,
- (optionally) a summary function to extract summary from test output.
For example the following .nxdtest defines 3 test cases that run Wendelin.core
`test.py` tests with FileStorage, ZEO and NEO as database backend::
for stor in ['fs1', 'zeo', 'neo']:
TestCase('test.py/%s' % stor, ['make', 'test.py'],
env={'WENDELIN_CORE_TEST_DB': '<%s>' % stor,
summaryf=PyTest.summary)
Nxdtest only runs tests, but - unlike tox - does not prepare variants of
software build. Nxdtest assumes that the software build is already fully
prepared. This matches SlapOS environment, where software building is separate
step handled by SlapOS.
(*) https://www.erp5.com/NXD-Presentation.ci.testing.system.buildout
https://www.erp5.com/erp5-Guideline.Nexedi.Testing.Extended
https://stack.nexedi.com/test_status
"""
from __future__ import print_function, absolute_import
from erp5.util.taskdistribution import TaskDistributor
from subprocess import Popen, PIPE
from time import time, strftime, gmtime
import os, sys, threading, argparse, logging, traceback, re
import six
# loadNXDTestFile loads .nxdtest file located @path.
def loadNXDTestFile(path): # -> TestEnv
t = TestEnv()
g = {'TestCase': t.TestCase} # TODO + all other public TestEnv methods
with open(path, "r") as f:
src = f.read()
six.exec_(src, g)
return t
# TestCase defines one test case to run.
class TestCase:
def __init__(self, name, argv, summaryf=None, **kw):
self.name = name # testcase name
self.argv = argv # program to run
self.kw = kw # **kw is passed to Popen
self.summaryf = summaryf # function to extract summary from test output
# TestEnv represents a testing environment with set of TestCases to run.
class TestEnv:
def __init__(self):
self.byname = {} # name -> TestCase
self.testv = [] # of TestCase
# TestCase adds new test case to the environment.
def TestCase(self, name, argv, **kw):
assert name not in self.byname
t = TestCase(name, argv, **kw)
self.testv.append(t)
self.byname[name] = t
def main():
# testnode executes us giving URL to master results collecting instance and other details
# https://lab.nexedi.com/nexedi/erp5/blob/744f3fde/erp5/util/testnode/UnitTestRunner.py#L137
parser = argparse.ArgumentParser(description=__doc__)
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--master_url', help='The URL of Master controling many suites')
parser.add_argument('--revision', help='The revision to test', default='dummy_revision')
parser.add_argument('--test_suite', help='The test suite name')
......@@ -49,11 +107,14 @@ def main():
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger()
# load list of tests to run
tenv = loadNXDTestFile('.nxdtest')
# connect to master and create 'test result' object with list of tests to run
tool = TaskDistributor(portal_url = args.master_url, logger = logger)
test_result = tool.createTestResult(
revision = args.revision,
test_name_list = ['test-go', 'test-py', 'bench-local'],
test_name_list = [t.name for t in tenv.testv],
node_title = args.test_node_title,
test_title = args.test_suite_title or args.test_suite,
project_title = args.project_title)
......@@ -75,15 +136,15 @@ def main():
if test_result_line is None:
break
# run `neotest <test-name>`
testname = test_result_line.name
argv = ['neotest', testname]
# run tenv[name]
t = tenv.byname[test_result_line.name]
tstart = time()
try:
# NOTE runs with unchanged cwd. Instance wrapper cares to set cwd before running us.
# Run t.argv in t.kw['env'] environment.
# Test command is spawned with unchanged cwd. Instance wrapper cares to set cwd before running us.
# bufsize=1 means 'line buffered'
p = Popen(argv, stdin=devnull, stdout=PIPE, stderr=PIPE, bufsize=1)
p = Popen(t.argv, stdin=devnull, stdout=PIPE, stderr=PIPE, bufsize=1, **t.kw)
except:
stdout, stderr = '', traceback.format_exc()
sys.stderr.write(stderr)
......@@ -113,10 +174,9 @@ def main():
}
# postprocess output, if we can
summaryf = globals().get(testname.replace('-', '_') + '_summary')
if summaryf is not None:
if t.summaryf is not None:
try:
summary = summaryf(stdout)
summary = t.summaryf(stdout)
except:
bad = traceback.format_exc()
sys.stderr.write(bad)
......@@ -131,7 +191,7 @@ def main():
# report result of test run back to master
test_result_line.stop(
command = ' '.join(argv),
command = ' '.join(t.argv),
duration = tend - tstart,
date = strftime("%Y/%m/%d %H:%M:%S", gmtime(tend)),
......
......@@ -13,7 +13,7 @@ setup(
keywords = 'Nexedi testing infrastructure tool tox',
packages = [],
install_requires = ['erp5.util'],
install_requires = ['erp5.util', 'six'],
scripts = ['nxdtest'],
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment