Commit ee96d25f authored by Daniel Latypov's avatar Daniel Latypov Committed by Shuah Khan

kunit: tool: refactor how we plumb metadata into JSON

When using --json, kunit.py run/exec/parse will produce results in
KernelCI json format.
As part of that, we include the build_dir that was used, and we
(incorrectly) hardcode in the arch, etc.

We'll want a way to plumb more values (as well as the correct `arch`),
so this patch groups those fields into kunit_json.Metadata type.
This patch should have no user visible changes.

And since we only used build_dir in KunitParseRequest for json, we can
now move it out of that struct and add it into KunitExecRequest, which
needs it and used to get it via inheritance.
Signed-off-by: default avatarDaniel Latypov <dlatypov@google.com>
Reviewed-by: default avatarDavid Gow <davidgow@google.com>
Reviewed-by: default avatarBrendan Higgins <brendanhiggins@google.com>
Signed-off-by: default avatarShuah Khan <skhan@linuxfoundation.org>
parent 6bd0f52e
...@@ -47,11 +47,11 @@ class KunitBuildRequest(KunitConfigRequest): ...@@ -47,11 +47,11 @@ class KunitBuildRequest(KunitConfigRequest):
@dataclass @dataclass
class KunitParseRequest: class KunitParseRequest:
raw_output: Optional[str] raw_output: Optional[str]
build_dir: str
json: Optional[str] json: Optional[str]
@dataclass @dataclass
class KunitExecRequest(KunitParseRequest): class KunitExecRequest(KunitParseRequest):
build_dir: str
timeout: int timeout: int
alltests: bool alltests: bool
filter_glob: str filter_glob: str
...@@ -153,6 +153,8 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) - ...@@ -153,6 +153,8 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
test_glob = request.filter_glob.split('.', maxsplit=2)[1] test_glob = request.filter_glob.split('.', maxsplit=2)[1]
filter_globs = [g + '.'+ test_glob for g in filter_globs] filter_globs = [g + '.'+ test_glob for g in filter_globs]
metadata = kunit_json.Metadata(build_dir=request.build_dir)
test_counts = kunit_parser.TestCounts() test_counts = kunit_parser.TestCounts()
exec_time = 0.0 exec_time = 0.0
for i, filter_glob in enumerate(filter_globs): for i, filter_glob in enumerate(filter_globs):
...@@ -165,7 +167,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) - ...@@ -165,7 +167,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
filter_glob=filter_glob, filter_glob=filter_glob,
build_dir=request.build_dir) build_dir=request.build_dir)
_, test_result = parse_tests(request, run_result) _, test_result = parse_tests(request, metadata, run_result)
# run_kernel() doesn't block on the kernel exiting. # run_kernel() doesn't block on the kernel exiting.
# That only happens after we get the last line of output from `run_result`. # That only happens after we get the last line of output from `run_result`.
# So exec_time here actually contains parsing + execution time, which is fine. # So exec_time here actually contains parsing + execution time, which is fine.
...@@ -189,7 +191,7 @@ def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: ...@@ -189,7 +191,7 @@ def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus:
else: else:
return KunitStatus.TEST_FAILURE return KunitStatus.TEST_FAILURE
def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]: def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]:
parse_start = time.time() parse_start = time.time()
test_result = kunit_parser.Test() test_result = kunit_parser.Test()
...@@ -216,8 +218,7 @@ def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> Tuple[ ...@@ -216,8 +218,7 @@ def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> Tuple[
if request.json: if request.json:
json_str = kunit_json.get_json_result( json_str = kunit_json.get_json_result(
test=test_result, test=test_result,
def_config='kunit_defconfig', metadata=metadata)
build_dir=request.build_dir)
if request.json == 'stdout': if request.json == 'stdout':
print(json_str) print(json_str)
else: else:
...@@ -504,10 +505,11 @@ def main(argv, linux=None): ...@@ -504,10 +505,11 @@ def main(argv, linux=None):
else: else:
with open(cli_args.file, 'r', errors='backslashreplace') as f: with open(cli_args.file, 'r', errors='backslashreplace') as f:
kunit_output = f.read().splitlines() kunit_output = f.read().splitlines()
# We know nothing about how the result was created!
metadata = kunit_json.Metadata()
request = KunitParseRequest(raw_output=cli_args.raw_output, request = KunitParseRequest(raw_output=cli_args.raw_output,
build_dir='',
json=cli_args.json) json=cli_args.json)
result, _ = parse_tests(request, kunit_output) result, _ = parse_tests(request, metadata, kunit_output)
if result.status != KunitStatus.SUCCESS: if result.status != KunitStatus.SUCCESS:
sys.exit(1) sys.exit(1)
else: else:
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
# Copyright (C) 2020, Google LLC. # Copyright (C) 2020, Google LLC.
# Author: Heidi Fahim <heidifahim@google.com> # Author: Heidi Fahim <heidifahim@google.com>
from dataclasses import dataclass
import json import json
import os import os
...@@ -14,6 +15,13 @@ import kunit_parser ...@@ -14,6 +15,13 @@ import kunit_parser
from kunit_parser import Test, TestStatus from kunit_parser import Test, TestStatus
from typing import Any, Dict from typing import Any, Dict
@dataclass
class Metadata:
"""Stores metadata about this run to include in get_json_result()."""
arch: str = 'UM'
def_config: str = 'kunit_defconfig'
build_dir: str = ''
JsonObj = Dict[str, Any] JsonObj = Dict[str, Any]
_status_map: Dict[TestStatus, str] = { _status_map: Dict[TestStatus, str] = {
...@@ -22,14 +30,13 @@ _status_map: Dict[TestStatus, str] = { ...@@ -22,14 +30,13 @@ _status_map: Dict[TestStatus, str] = {
TestStatus.TEST_CRASHED: "ERROR", TestStatus.TEST_CRASHED: "ERROR",
} }
def _get_group_json(test: Test, def_config: str, build_dir: str) -> JsonObj: def _get_group_json(test: Test, common_fields: JsonObj) -> JsonObj:
sub_groups = [] # List[JsonObj] sub_groups = [] # List[JsonObj]
test_cases = [] # List[JsonObj] test_cases = [] # List[JsonObj]
for subtest in test.subtests: for subtest in test.subtests:
if subtest.subtests: if subtest.subtests:
sub_group = _get_group_json(subtest, def_config, sub_group = _get_group_json(subtest, common_fields)
build_dir)
sub_groups.append(sub_group) sub_groups.append(sub_group)
continue continue
status = _status_map.get(subtest.status, "FAIL") status = _status_map.get(subtest.status, "FAIL")
...@@ -37,19 +44,23 @@ def _get_group_json(test: Test, def_config: str, build_dir: str) -> JsonObj: ...@@ -37,19 +44,23 @@ def _get_group_json(test: Test, def_config: str, build_dir: str) -> JsonObj:
test_group = { test_group = {
"name": test.name, "name": test.name,
"arch": "UM",
"defconfig": def_config,
"build_environment": build_dir,
"sub_groups": sub_groups, "sub_groups": sub_groups,
"test_cases": test_cases, "test_cases": test_cases,
}
test_group.update(common_fields)
return test_group
def get_json_result(test: Test, metadata: Metadata) -> str:
common_fields = {
"arch": metadata.arch,
"defconfig": metadata.def_config,
"build_environment": metadata.build_dir,
"lab_name": None, "lab_name": None,
"kernel": None, "kernel": None,
"job": None, "job": None,
"git_branch": "kselftest", "git_branch": "kselftest",
} }
return test_group
def get_json_result(test: Test, def_config: str, build_dir: str) -> str: test_group = _get_group_json(test, common_fields)
test_group = _get_group_json(test, def_config, build_dir)
test_group["name"] = "KUnit Test Group" test_group["name"] = "KUnit Test Group"
return json.dumps(test_group, indent=4) return json.dumps(test_group, indent=4)
...@@ -468,8 +468,7 @@ class KUnitJsonTest(unittest.TestCase): ...@@ -468,8 +468,7 @@ class KUnitJsonTest(unittest.TestCase):
test_result = kunit_parser.parse_run_tests(file) test_result = kunit_parser.parse_run_tests(file)
json_obj = kunit_json.get_json_result( json_obj = kunit_json.get_json_result(
test=test_result, test=test_result,
def_config='kunit_defconfig', metadata=kunit_json.Metadata())
build_dir='.kunit')
return json.loads(json_obj) return json.loads(json_obj)
def test_failed_test_json(self): def test_failed_test_json(self):
...@@ -691,7 +690,7 @@ class KUnitMainTest(unittest.TestCase): ...@@ -691,7 +690,7 @@ class KUnitMainTest(unittest.TestCase):
self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want
got = kunit._list_tests(self.linux_source_mock, got = kunit._list_tests(self.linux_source_mock,
kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*', None, 'suite')) kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*', None, 'suite'))
self.assertEqual(got, want) self.assertEqual(got, want)
# Should respect the user's filter glob when listing tests. # Should respect the user's filter glob when listing tests.
...@@ -706,7 +705,7 @@ class KUnitMainTest(unittest.TestCase): ...@@ -706,7 +705,7 @@ class KUnitMainTest(unittest.TestCase):
# Should respect the user's filter glob when listing tests. # Should respect the user's filter glob when listing tests.
mock_tests.assert_called_once_with(mock.ANY, mock_tests.assert_called_once_with(mock.ANY,
kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*.test*', None, 'suite')) kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*.test*', None, 'suite'))
self.linux_source_mock.run_kernel.assert_has_calls([ self.linux_source_mock.run_kernel.assert_has_calls([
mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300),
mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300),
...@@ -719,7 +718,7 @@ class KUnitMainTest(unittest.TestCase): ...@@ -719,7 +718,7 @@ class KUnitMainTest(unittest.TestCase):
# Should respect the user's filter glob when listing tests. # Should respect the user's filter glob when listing tests.
mock_tests.assert_called_once_with(mock.ANY, mock_tests.assert_called_once_with(mock.ANY,
kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*', None, 'test')) kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*', None, 'test'))
self.linux_source_mock.run_kernel.assert_has_calls([ self.linux_source_mock.run_kernel.assert_has_calls([
mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300),
mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment