Commit fc08766d authored by Kirill Smelkov's avatar Kirill Smelkov

py.bench: Fix report.bench_time collect

In pytest3 pytest_runtest_call hook is always run as multicall and there
is no way to control to disable it and force e.g. 'firstresult'
behaviour. This way call.result in pytest_runtest_makereport is always a
list of returned results, not the first result itself.

Adjust the code in pytest_runtest_makereport to emulate 'firstresult'
behaviour manually.

Without the patch with recent pytest py.bench was crashing like this:

---- 8< ----
(z-dev) kirr@link:~/src/wendelin/wendelin.core$ make bench
python setup.py ll_build_ext --inplace
running ll_build_ext
copying build/lib.linux-x86_64-2.7/wendelin/bigfile/_bigfile.so -> bigfile
python t/py.bench --ignore=3rdparty --ignore=build --ignore=t
============================================================= test session starts ==============================================================
platform linux2 -- Python 2.7.14, pytest-3.3.0, py-1.5.2, pluggy-0.6.0
rootdir: /home/kirr/src/wendelin/wendelin.core, inifile:
collected 14 items

bigfile/tests/bench_0virtmem.py                          	INTERNALERROR> Traceback (most recent call last):
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/_pytest/main.py", line 103, in wrap_session
INTERNALERROR>     session.exitstatus = doit(config, session) or 0
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/_pytest/main.py", line 141, in _main
INTERNALERROR>     config.hook.pytest_runtestloop(session=session)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 617, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 222, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 216, in <lambda>
INTERNALERROR>     firstresult=hook.spec_opts.get('firstresult'),
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 201, in _multicall
INTERNALERROR>     return outcome.get_result()
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 77, in get_result
INTERNALERROR>     _reraise(*ex)  # noqa
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 180, in _multicall
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/_pytest/main.py", line 164, in pytest_runtestloop
INTERNALERROR>     item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 617, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 222, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 216, in <lambda>
INTERNALERROR>     firstresult=hook.spec_opts.get('firstresult'),
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 201, in _multicall
INTERNALERROR>     return outcome.get_result()
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 77, in get_result
INTERNALERROR>     _reraise(*ex)  # noqa
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 180, in _multicall
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/_pytest/runner.py", line 63, in pytest_runtest_protocol
INTERNALERROR>     runtestprotocol(item, nextitem=nextitem)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/_pytest/runner.py", line 77, in runtestprotocol
INTERNALERROR>     reports.append(call_and_report(item, "call", log))
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/_pytest/runner.py", line 161, in call_and_report
INTERNALERROR>     hook.pytest_runtest_logreport(report=report)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 617, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 222, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/__init__.py", line 216, in <lambda>
INTERNALERROR>     firstresult=hook.spec_opts.get('firstresult'),
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 201, in _multicall
INTERNALERROR>     return outcome.get_result()
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 77, in get_result
INTERNALERROR>     _reraise(*ex)  # noqa
INTERNALERROR>   File "/home/kirr/src/wendelin/venv/z-dev/local/lib/python2.7/site-packages/pluggy/callers.py", line 180, in _multicall
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "t/py.bench", line 165, in pytest_runtest_logreport
INTERNALERROR>     self._tw.write('%.2f' % min(report.bench_times))
INTERNALERROR> TypeError: float argument required, not list
parent 5a1ed45a
...@@ -75,10 +75,7 @@ class BenchPlugin: ...@@ -75,10 +75,7 @@ class BenchPlugin:
# run benchmark several times, and get best timing # run benchmark several times, and get best timing
# each benchmark run is executed in separate process (if not told otherwise) # each benchmark run is executed in separate process (if not told otherwise)
def pytest_runtest_call(self, item, __multicall__): def pytest_runtest_call(self, item):
# tell MultiCall not to execut other runtest_calls (e.g. runner.pytest_runtest_call)
__multicall__.firstresult = True
def run(): def run():
tstart = time() tstart = time()
item.runtest() item.runtest()
...@@ -110,7 +107,10 @@ class BenchPlugin: ...@@ -110,7 +107,10 @@ class BenchPlugin:
def pytest_runtest_makereport(self, item, call): def pytest_runtest_makereport(self, item, call):
report = _pytest.runner.pytest_runtest_makereport(item, call) report = _pytest.runner.pytest_runtest_makereport(item, call)
if call.when == 'call' and not call.excinfo: if call.when == 'call' and not call.excinfo:
report.bench_times = call.result # in pytest3 there is no way to mark pytest_runtest_call as 'firstresult'
# let's emulate firstresult logic here
assert len(call.result) == 1
report.bench_times = call.result[0]
return report return report
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment