diff --git a/AUTHORS b/AUTHORS index ac8caf905..48d9e26d6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -31,6 +31,7 @@ Christopher Gilling Daniel Grana Daniel Hahler Daniel Nuri +Danielle Jenkins Dave Hunt David Díaz-Barquero David Mohr @@ -96,6 +97,7 @@ Ross Lawley Russel Winder Ryan Wooden Samuele Pedroni +Steffen Allner Stephan Obermann Tareq Alayan Simon Gomizelj @@ -104,6 +106,7 @@ Stefan Farmbauer Thomas Grainger Tom Viner Trevor Bekolay +Vasily Kuznetsov Wouter van Ackooy Bernard Pratz Stefan Zimmermann diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3fee592b9..9ec08b03c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -68,6 +68,18 @@ `#1629`_. Thanks `@obestwalter`_ and `@davehunt`_ for the complete PR (`#1633`_) +* New cli flags: (1) ``--setup-plan`` performs normal collection and reports + the potential setup and teardown, does not execute any fixtures and tests (2) + ``--setup-only`` performs normal collection, executes setup and teardown of + fixtures and reports them. Thanks `@d6e`_, `@kvas-it`_, `@sallner`_ + and `@omarkohl`_ for the PR. + +* Added two new hooks: ``pytest_fixture_setup`` which executes the fixture + setup and ``pytest_fixture_post_finalizer`` which is called after the fixture's + finalizer and has access to the fixture's result cache. + Thanks `@d6e`_, `@sallner`_ + + **Changes** * Fixtures marked with ``@pytest.fixture`` can now use ``yield`` statements exactly like @@ -170,6 +182,9 @@ .. _@olegpidsadnyi: https://github.com/olegpidsadnyi .. _@obestwalter: https://github.com/obestwalter .. _@davehunt: https://github.com/davehunt +.. _@sallner: https://github.com/sallner +.. _@d6e: https://github.com/d6e +.. _@kvas-it: https://github.com/kvas-it .. _#1421: https://github.com/pytest-dev/pytest/issues/1421 .. _#1426: https://github.com/pytest-dev/pytest/issues/1426 diff --git a/_pytest/config.py b/_pytest/config.py index a17587d9e..3a8ce1c78 100644 --- a/_pytest/config.py +++ b/_pytest/config.py @@ -65,7 +65,7 @@ _preinit = [] default_plugins = ( "mark main terminal runner python pdb unittest capture skipping " "tmpdir monkeypatch recwarn pastebin helpconfig nose assertion genscript " - "junitxml resultlog doctest cacheprovider").split() + "junitxml resultlog doctest cacheprovider setuponly setupplan").split() builtin_plugins = set(default_plugins) builtin_plugins.add("pytester") diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py index 639e316d0..2c6d8ba51 100644 --- a/_pytest/hookspec.py +++ b/_pytest/hookspec.py @@ -218,6 +218,19 @@ def pytest_runtest_logreport(report): """ process a test setup/call/teardown report relating to the respective phase of executing a test. """ +# ------------------------------------------------------------------------- +# Fixture related hooks +# ------------------------------------------------------------------------- + +@hookspec(firstresult=True) +def pytest_fixture_setup(fixturedef, request): + """ performs fixture setup execution. """ + +def pytest_fixture_post_finalizer(fixturedef): + """ called after fixture teardown, but before the cache is cleared so + the fixture result cache ``fixturedef.cached_result`` can + still be accessed.""" + # ------------------------------------------------------------------------- # test session related hooks # ------------------------------------------------------------------------- diff --git a/_pytest/python.py b/_pytest/python.py index 0520ecef0..a0624839b 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -2481,6 +2481,8 @@ class FixtureDef: func = self._finalizer.pop() func() finally: + ihook = self._fixturemanager.session.ihook + ihook.pytest_fixture_post_finalizer(fixturedef=self) # even if finalization fails, we invalidate # the cached fixture value if hasattr(self, "cached_result"): @@ -2489,12 +2491,8 @@ class FixtureDef: def execute(self, request): # get required arguments and register our own finish() # with their finalization - kwargs = {} for argname in self.argnames: fixturedef = request._get_active_fixturedef(argname) - result, arg_cache_key, exc = fixturedef.cached_result - request._check_scope(argname, request.scope, fixturedef.scope) - kwargs[argname] = result if argname != "request": fixturedef.addfinalizer(self.finish) @@ -2512,33 +2510,44 @@ class FixtureDef: self.finish() assert not hasattr(self, "cached_result") - fixturefunc = self.func - - if self.unittest: - if request.instance is not None: - # bind the unbound method to the TestCase instance - fixturefunc = self.func.__get__(request.instance) - else: - # the fixture function needs to be bound to the actual - # request.instance so that code working with "self" behaves - # as expected. - if request.instance is not None: - fixturefunc = getimfunc(self.func) - if fixturefunc != self.func: - fixturefunc = fixturefunc.__get__(request.instance) - - try: - result = call_fixture_func(fixturefunc, request, kwargs) - except Exception: - self.cached_result = (None, my_cache_key, sys.exc_info()) - raise - self.cached_result = (result, my_cache_key, None) - return result + ihook = self._fixturemanager.session.ihook + ihook.pytest_fixture_setup(fixturedef=self, request=request) def __repr__(self): return ("" % (self.argname, self.scope, self.baseid)) +def pytest_fixture_setup(fixturedef, request): + """ Execution of fixture setup. """ + kwargs = {} + for argname in fixturedef.argnames: + fixdef = request._get_active_fixturedef(argname) + result, arg_cache_key, exc = fixdef.cached_result + request._check_scope(argname, request.scope, fixdef.scope) + kwargs[argname] = result + + fixturefunc = fixturedef.func + if fixturedef.unittest: + if request.instance is not None: + # bind the unbound method to the TestCase instance + fixturefunc = fixturedef.func.__get__(request.instance) + else: + # the fixture function needs to be bound to the actual + # request.instance so that code working with "fixturedef" behaves + # as expected. + if request.instance is not None: + fixturefunc = getimfunc(fixturedef.func) + if fixturefunc != fixturedef.func: + fixturefunc = fixturefunc.__get__(request.instance) + my_cache_key = request.param_index + try: + result = call_fixture_func(fixturefunc, request, kwargs) + except Exception: + fixturedef.cached_result = (None, my_cache_key, sys.exc_info()) + raise + fixturedef.cached_result = (result, my_cache_key, None) + return result + def num_mock_patch_args(function): """ return number of arguments used up by mock arguments (if any) """ patchings = getattr(function, "patchings", None) diff --git a/_pytest/runner.py b/_pytest/runner.py index 4cc2ef6ac..dff321a75 100644 --- a/_pytest/runner.py +++ b/_pytest/runner.py @@ -73,7 +73,10 @@ def runtestprotocol(item, log=True, nextitem=None): rep = call_and_report(item, "setup", log) reports = [rep] if rep.passed: - reports.append(call_and_report(item, "call", log)) + if item.config.option.setuponly or item.config.option.setupplan: + show_test_item(item) + else: + reports.append(call_and_report(item, "call", log)) reports.append(call_and_report(item, "teardown", log, nextitem=nextitem)) # after all teardown hooks have been called @@ -83,6 +86,16 @@ def runtestprotocol(item, log=True, nextitem=None): item.funcargs = None return reports +def show_test_item(item): + """Show test function, parameters and the fixtures of the test item.""" + tw = item.config.get_terminal_writer() + tw.line() + tw.write(' ' * 8) + tw.write(item._nodeid) + used_fixtures = sorted(item._fixtureinfo.name2fixturedefs.keys()) + if used_fixtures: + tw.write(' (fixtures used: {0})'.format(', '.join(used_fixtures))) + def pytest_runtest_setup(item): item.session._setupstate.prepare(item) diff --git a/_pytest/setuponly.py b/_pytest/setuponly.py new file mode 100644 index 000000000..abb578da7 --- /dev/null +++ b/_pytest/setuponly.py @@ -0,0 +1,59 @@ +import pytest +import sys + +def pytest_addoption(parser): + group = parser.getgroup("debugconfig") + group.addoption('--setuponly', '--setup-only', action="store_true", + help="only setup fixtures, don't execute the tests.") + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef, request): + yield + config = request.config + if config.option.setuponly: + if hasattr(request, 'param'): + # Save the fixture parameter so ._show_fixture_action() can + # display it now and during the teardown (in .finish()). + if fixturedef.ids: + if callable(fixturedef.ids): + fixturedef.cached_param = fixturedef.ids(request.param) + else: + fixturedef.cached_param = fixturedef.ids[request.param_index] + else: + fixturedef.cached_param = request.param + _show_fixture_action(fixturedef, 'SETUP') + +def pytest_fixture_post_finalizer(fixturedef): + if hasattr(fixturedef, "cached_result"): + config = fixturedef._fixturemanager.config + if config.option.setuponly: + _show_fixture_action(fixturedef, 'TEARDOWN') + if hasattr(fixturedef, "cached_param"): + del fixturedef.cached_param + +def _show_fixture_action(fixturedef, msg): + config = fixturedef._fixturemanager.config + capman = config.pluginmanager.getplugin('capturemanager') + if capman: + out, err = capman.suspendcapture() + + tw = config.get_terminal_writer() + tw.line() + tw.write(' ' * 2 * fixturedef.scopenum) + tw.write('{step} {scope} {fixture}'.format( + step=msg.ljust(8), # align the output to TEARDOWN + scope=fixturedef.scope[0].upper(), + fixture=fixturedef.argname)) + + if msg == 'SETUP': + deps = sorted(arg for arg in fixturedef.argnames if arg != 'request') + if deps: + tw.write(' (fixtures used: {0})'.format(', '.join(deps))) + + if hasattr(fixturedef, 'cached_param'): + tw.write('[{0}]'.format(fixturedef.cached_param)) + + if capman: + capman.resumecapture() + sys.stdout.write(out) + sys.stderr.write(err) diff --git a/_pytest/setupplan.py b/_pytest/setupplan.py new file mode 100644 index 000000000..c7c8ff60d --- /dev/null +++ b/_pytest/setupplan.py @@ -0,0 +1,19 @@ +import pytest + +def pytest_addoption(parser): + group = parser.getgroup("debugconfig") + group.addoption('--setupplan', '--setup-plan', action="store_true", + help="show what fixtures and tests would be executed but don't" + " execute anything.") + +@pytest.hookimpl(tryfirst=True) +def pytest_fixture_setup(fixturedef, request): + # Will return a dummy fixture if the setuponly option is provided. + if request.config.option.setupplan: + fixturedef.cached_result = (None, None, None) + return fixturedef.cached_result + +@pytest.hookimpl(tryfirst=True) +def pytest_cmdline_main(config): + if config.option.setupplan: + config.option.setuponly = True diff --git a/doc/en/writing_plugins.rst b/doc/en/writing_plugins.rst index aeb30724c..bcd795d82 100644 --- a/doc/en/writing_plugins.rst +++ b/doc/en/writing_plugins.rst @@ -498,6 +498,8 @@ Session related reporting hooks: .. autofunction:: pytest_report_header .. autofunction:: pytest_report_teststatus .. autofunction:: pytest_terminal_summary +.. autofunction:: pytest_fixture_setup +.. autofunction:: pytest_fixture_post_finalizer And here is the central hook for reporting about test execution: @@ -554,6 +556,10 @@ Reference of objects involved in hooks :members: :show-inheritance: +.. autoclass:: _pytest.python.FixtureDef() + :members: + :show-inheritance: + .. autoclass:: _pytest.runner.CallInfo() :members: diff --git a/testing/python/setup_only.py b/testing/python/setup_only.py new file mode 100644 index 000000000..e7403420b --- /dev/null +++ b/testing/python/setup_only.py @@ -0,0 +1,221 @@ +import pytest + + +@pytest.fixture(params=['--setup-only', '--setup-plan'], scope='module') +def mode(request): + return request.param + + +def test_show_only_active_fixtures(testdir, mode): + p = testdir.makepyfile(''' + import pytest + @pytest.fixture + def _arg0(): + """hidden arg0 fixture""" + @pytest.fixture + def arg1(): + """arg1 docstring""" + def test_arg1(arg1): + pass + ''') + + result = testdir.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + '*SETUP F arg1*', + '*test_arg1 (fixtures used: arg1)', + '*TEARDOWN F arg1*', + ]) + assert "_arg0" not in result.stdout.str() + + +def test_show_different_scopes(testdir, mode): + p = testdir.makepyfile(''' + import pytest + @pytest.fixture + def arg_function(): + """function scoped fixture""" + @pytest.fixture(scope='session') + def arg_session(): + """session scoped fixture""" + def test_arg1(arg_session, arg_function): + pass + ''') + + result = testdir.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + 'SETUP S arg_session*', + '*SETUP F arg_function*', + '*test_arg1 (fixtures used: arg_function, arg_session)', + '*TEARDOWN F arg_function*', + 'TEARDOWN S arg_session*', + ]) + + +def test_show_nested_fixtures(testdir, mode): + testdir.makeconftest(''' + import pytest + @pytest.fixture(scope='session') + def arg_same(): + """session scoped fixture""" + ''') + p = testdir.makepyfile(''' + import pytest + @pytest.fixture(scope='function') + def arg_same(arg_same): + """function scoped fixture""" + def test_arg1(arg_same): + pass + ''') + + result = testdir.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + 'SETUP S arg_same*', + '*SETUP F arg_same (fixtures used: arg_same)*', + '*test_arg1 (fixtures used: arg_same)', + '*TEARDOWN F arg_same*', + 'TEARDOWN S arg_same*', + ]) + + +def test_show_fixtures_with_autouse(testdir, mode): + p = testdir.makepyfile(''' + import pytest + @pytest.fixture + def arg_function(): + """function scoped fixture""" + @pytest.fixture(scope='session', autouse=True) + def arg_session(): + """session scoped fixture""" + def test_arg1(arg_function): + pass + ''') + + result = testdir.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + 'SETUP S arg_session*', + '*SETUP F arg_function*', + '*test_arg1 (fixtures used: arg_function, arg_session)', + ]) + + +def test_show_fixtures_with_parameters(testdir, mode): + testdir.makeconftest(''' + import pytest + @pytest.fixture(scope='session', params=['foo', 'bar']) + def arg_same(): + """session scoped fixture""" + ''') + p = testdir.makepyfile(''' + import pytest + @pytest.fixture(scope='function') + def arg_other(arg_same): + """function scoped fixture""" + def test_arg1(arg_other): + pass + ''') + + result = testdir.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + 'SETUP S arg_same?foo?', + 'TEARDOWN S arg_same?foo?', + 'SETUP S arg_same?bar?', + 'TEARDOWN S arg_same?bar?', + ]) + + +def test_show_fixtures_with_parameter_ids(testdir, mode): + testdir.makeconftest(''' + import pytest + @pytest.fixture( + scope='session', params=['foo', 'bar'], ids=['spam', 'ham']) + def arg_same(): + """session scoped fixture""" + ''') + p = testdir.makepyfile(''' + import pytest + @pytest.fixture(scope='function') + def arg_other(arg_same): + """function scoped fixture""" + def test_arg1(arg_other): + pass + ''') + + result = testdir.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + 'SETUP S arg_same?spam?', + 'SETUP S arg_same?ham?', + ]) + + +def test_show_fixtures_with_parameter_ids_function(testdir, mode): + p = testdir.makepyfile(''' + import pytest + @pytest.fixture(params=['foo', 'bar'], ids=lambda p: p.upper()) + def foobar(): + pass + def test_foobar(foobar): + pass + ''') + + result = testdir.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + '*SETUP F foobar?FOO?', + '*SETUP F foobar?BAR?', + ]) + + +def test_dynamic_fixture_request(testdir): + p = testdir.makepyfile(''' + import pytest + @pytest.fixture() + def dynamically_requested_fixture(): + pass + @pytest.fixture() + def dependent_fixture(request): + request.getfuncargvalue('dynamically_requested_fixture') + def test_dyn(dependent_fixture): + pass + ''') + + result = testdir.runpytest('--setup-only', p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + '*SETUP F dynamically_requested_fixture', + '*TEARDOWN F dynamically_requested_fixture' + ]) + + +def test_capturing(testdir): + p = testdir.makepyfile(''' + import pytest, sys + @pytest.fixture() + def one(): + sys.stdout.write('this should be captured') + sys.stderr.write('this should also be captured') + @pytest.fixture() + def two(one): + assert 0 + def test_capturing(two): + pass + ''') + + result = testdir.runpytest('--setup-only', p) + result.stdout.fnmatch_lines([ + 'this should be captured', + 'this should also be captured' + ]) diff --git a/testing/python/setup_plan.py b/testing/python/setup_plan.py new file mode 100644 index 000000000..8c9822469 --- /dev/null +++ b/testing/python/setup_plan.py @@ -0,0 +1,19 @@ +def test_show_fixtures_and_test(testdir): + """ Verifies that fixtures are not executed. """ + p = testdir.makepyfile(''' + import pytest + @pytest.fixture + def arg(): + assert False + def test_arg(arg): + assert False + ''') + + result = testdir.runpytest("--setup-plan", p) + assert result.ret == 0 + + result.stdout.fnmatch_lines([ + '*SETUP F arg*', + '*test_arg (fixtures used: arg)', + '*TEARDOWN F arg*', + ])