Merge pull request #1647 from sallner/features

Add new options to report fixture setup and teardown

AUTHORS

@@ -31,6 +31,7 @@ Christopher Gilling
 Daniel Grana
 Daniel Hahler
 Daniel Nuri
+Danielle Jenkins
 Dave Hunt
 David Díaz-Barquero
 David Mohr
@@ -96,6 +97,7 @@ Ross Lawley
 Russel Winder
 Ryan Wooden
 Samuele Pedroni
+Steffen Allner
 Stephan Obermann
 Tareq Alayan
 Simon Gomizelj
@@ -104,6 +106,7 @@ Stefan Farmbauer
 Thomas Grainger
 Tom Viner
 Trevor Bekolay
+Vasily Kuznetsov
 Wouter van Ackooy
 Bernard Pratz
 Stefan Zimmermann

CHANGELOG.rst

@@ -68,6 +68,18 @@
   `#1629`_. Thanks `@obestwalter`_ and `@davehunt`_ for the complete PR
   (`#1633`_)
 
+* New cli flags: (1) ``--setup-plan`` performs normal collection and reports
+  the potential setup and teardown, does not execute any fixtures and tests (2)
+  ``--setup-only`` performs normal collection, executes setup and teardown of
+  fixtures and reports them. Thanks `@d6e`_, `@kvas-it`_, `@sallner`_
+  and `@omarkohl`_ for the PR.
+
+* Added two new hooks: ``pytest_fixture_setup`` which executes the fixture
+  setup and ``pytest_fixture_post_finalizer`` which is called after the fixture's
+  finalizer and has access to the fixture's result cache.
+  Thanks `@d6e`_, `@sallner`_
+
+
 **Changes**
 
 * Fixtures marked with ``@pytest.fixture`` can now use ``yield`` statements exactly like
@@ -170,6 +182,9 @@
 .. _@olegpidsadnyi: https://github.com/olegpidsadnyi
 .. _@obestwalter: https://github.com/obestwalter
 .. _@davehunt: https://github.com/davehunt
+.. _@sallner: https://github.com/sallner
+.. _@d6e: https://github.com/d6e
+.. _@kvas-it: https://github.com/kvas-it
 
 .. _#1421: https://github.com/pytest-dev/pytest/issues/1421
 .. _#1426: https://github.com/pytest-dev/pytest/issues/1426

_pytest/config.py

@@ -65,7 +65,7 @@ _preinit = []
 default_plugins = (
      "mark main terminal runner python pdb unittest capture skipping "
      "tmpdir monkeypatch recwarn pastebin helpconfig nose assertion genscript "
-     "junitxml resultlog doctest cacheprovider").split()
+     "junitxml resultlog doctest cacheprovider setuponly setupplan").split()
 
 builtin_plugins = set(default_plugins)
 builtin_plugins.add("pytester")

_pytest/hookspec.py

@@ -218,6 +218,19 @@ def pytest_runtest_logreport(report):
     """ process a test setup/call/teardown report relating to
     the respective phase of executing a test. """
 
+# -------------------------------------------------------------------------
+# Fixture related hooks
+# -------------------------------------------------------------------------
+
+@hookspec(firstresult=True)
+def pytest_fixture_setup(fixturedef, request):
+    """ performs fixture setup execution. """
+
+def pytest_fixture_post_finalizer(fixturedef):
+    """ called after fixture teardown, but before the cache is cleared so
+    the fixture result cache ``fixturedef.cached_result`` can
+    still be accessed."""
+
 # -------------------------------------------------------------------------
 # test session related hooks
 # -------------------------------------------------------------------------

_pytest/python.py

@@ -2481,6 +2481,8 @@ class FixtureDef:
                 func = self._finalizer.pop()
                 func()
         finally:
+            ihook = self._fixturemanager.session.ihook
+            ihook.pytest_fixture_post_finalizer(fixturedef=self)
             # even if finalization fails, we invalidate
             # the cached fixture value
             if hasattr(self, "cached_result"):
@@ -2489,12 +2491,8 @@ class FixtureDef:
     def execute(self, request):
         # get required arguments and register our own finish()
         # with their finalization
-        kwargs = {}
         for argname in self.argnames:
             fixturedef = request._get_active_fixturedef(argname)
-            result, arg_cache_key, exc = fixturedef.cached_result
-            request._check_scope(argname, request.scope, fixturedef.scope)
-            kwargs[argname] = result
             if argname != "request":
                 fixturedef.addfinalizer(self.finish)
 
@@ -2512,33 +2510,44 @@ class FixtureDef:
             self.finish()
             assert not hasattr(self, "cached_result")
 
-        fixturefunc = self.func
+        ihook = self._fixturemanager.session.ihook
-
+        ihook.pytest_fixture_setup(fixturedef=self, request=request)
-        if self.unittest:
-            if request.instance is not None:
-                # bind the unbound method to the TestCase instance
-                fixturefunc = self.func.__get__(request.instance)
-        else:
-            # the fixture function needs to be bound to the actual
-            # request.instance so that code working with "self" behaves
-            # as expected.
-            if request.instance is not None:
-                fixturefunc = getimfunc(self.func)
-                if fixturefunc != self.func:
-                    fixturefunc = fixturefunc.__get__(request.instance)
-
-        try:
-            result = call_fixture_func(fixturefunc, request, kwargs)
-        except Exception:
-            self.cached_result = (None, my_cache_key, sys.exc_info())
-            raise
-        self.cached_result = (result, my_cache_key, None)
-        return result
 
     def __repr__(self):
         return ("<FixtureDef name=%r scope=%r baseid=%r >" %
                 (self.argname, self.scope, self.baseid))
 
+def pytest_fixture_setup(fixturedef, request):
+    """ Execution of fixture setup. """
+    kwargs = {}
+    for argname in fixturedef.argnames:
+        fixdef = request._get_active_fixturedef(argname)
+        result, arg_cache_key, exc = fixdef.cached_result
+        request._check_scope(argname, request.scope, fixdef.scope)
+        kwargs[argname] = result
+
+    fixturefunc = fixturedef.func
+    if fixturedef.unittest:
+        if request.instance is not None:
+            # bind the unbound method to the TestCase instance
+            fixturefunc = fixturedef.func.__get__(request.instance)
+    else:
+        # the fixture function needs to be bound to the actual
+        # request.instance so that code working with "fixturedef" behaves
+        # as expected.
+        if request.instance is not None:
+            fixturefunc = getimfunc(fixturedef.func)
+            if fixturefunc != fixturedef.func:
+                fixturefunc = fixturefunc.__get__(request.instance)
+    my_cache_key = request.param_index
+    try:
+        result = call_fixture_func(fixturefunc, request, kwargs)
+    except Exception:
+        fixturedef.cached_result = (None, my_cache_key, sys.exc_info())
+        raise
+    fixturedef.cached_result = (result, my_cache_key, None)
+    return result
+
 def num_mock_patch_args(function):
     """ return number of arguments used up by mock arguments (if any) """
     patchings = getattr(function, "patchings", None)

_pytest/runner.py

@@ -73,6 +73,9 @@ def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
+        if item.config.option.setuponly or item.config.option.setupplan:
+            show_test_item(item)
+        else:
             reports.append(call_and_report(item, "call", log))
     reports.append(call_and_report(item, "teardown", log,
         nextitem=nextitem))
@@ -83,6 +86,16 @@ def runtestprotocol(item, log=True, nextitem=None):
         item.funcargs = None
     return reports
 
+def show_test_item(item):
+    """Show test function, parameters and the fixtures of the test item."""
+    tw = item.config.get_terminal_writer()
+    tw.line()
+    tw.write(' ' * 8)
+    tw.write(item._nodeid)
+    used_fixtures = sorted(item._fixtureinfo.name2fixturedefs.keys())
+    if used_fixtures:
+        tw.write(' (fixtures used: {0})'.format(', '.join(used_fixtures)))
+
 def pytest_runtest_setup(item):
     item.session._setupstate.prepare(item)
 

_pytest/setuponly.py

@@ -0,0 +1,59 @@
+import pytest
+import sys
+
+def pytest_addoption(parser):
+    group = parser.getgroup("debugconfig")
+    group.addoption('--setuponly', '--setup-only', action="store_true",
+               help="only setup fixtures, don't execute the tests.")
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_fixture_setup(fixturedef, request):
+    yield
+    config = request.config
+    if config.option.setuponly:
+        if hasattr(request, 'param'):
+            # Save the fixture parameter so ._show_fixture_action() can
+            # display it now and during the teardown (in .finish()).
+            if fixturedef.ids:
+                if callable(fixturedef.ids):
+                    fixturedef.cached_param = fixturedef.ids(request.param)
+                else:
+                    fixturedef.cached_param = fixturedef.ids[request.param_index]
+            else:
+                fixturedef.cached_param = request.param
+        _show_fixture_action(fixturedef, 'SETUP')
+
+def pytest_fixture_post_finalizer(fixturedef):
+    if hasattr(fixturedef, "cached_result"):
+        config = fixturedef._fixturemanager.config
+        if config.option.setuponly:
+            _show_fixture_action(fixturedef, 'TEARDOWN')
+            if hasattr(fixturedef, "cached_param"):
+                del fixturedef.cached_param
+
+def _show_fixture_action(fixturedef, msg):
+    config = fixturedef._fixturemanager.config
+    capman = config.pluginmanager.getplugin('capturemanager')
+    if capman:
+        out, err = capman.suspendcapture()
+
+    tw = config.get_terminal_writer()
+    tw.line()
+    tw.write(' ' * 2 * fixturedef.scopenum)
+    tw.write('{step} {scope} {fixture}'.format(
+        step=msg.ljust(8),  # align the output to TEARDOWN
+        scope=fixturedef.scope[0].upper(),
+        fixture=fixturedef.argname))
+
+    if msg == 'SETUP':
+        deps = sorted(arg for arg in fixturedef.argnames if arg != 'request')
+        if deps:
+            tw.write(' (fixtures used: {0})'.format(', '.join(deps)))
+
+    if hasattr(fixturedef, 'cached_param'):
+        tw.write('[{0}]'.format(fixturedef.cached_param))
+
+    if capman:
+        capman.resumecapture()
+        sys.stdout.write(out)
+        sys.stderr.write(err)

_pytest/setupplan.py

@@ -0,0 +1,19 @@
+import pytest
+
+def pytest_addoption(parser):
+    group = parser.getgroup("debugconfig")
+    group.addoption('--setupplan', '--setup-plan', action="store_true",
+               help="show what fixtures and tests would be executed but don't"
+                    " execute anything.")
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_fixture_setup(fixturedef, request):
+    # Will return a dummy fixture if the setuponly option is provided.
+    if request.config.option.setupplan:
+        fixturedef.cached_result = (None, None, None)
+        return fixturedef.cached_result
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_cmdline_main(config):
+    if config.option.setupplan:
+        config.option.setuponly = True

doc/en/writing_plugins.rst

@@ -498,6 +498,8 @@ Session related reporting hooks:
 .. autofunction:: pytest_report_header
 .. autofunction:: pytest_report_teststatus
 .. autofunction:: pytest_terminal_summary
+.. autofunction:: pytest_fixture_setup
+.. autofunction:: pytest_fixture_post_finalizer
 
 And here is the central hook for reporting about
 test execution:
@@ -554,6 +556,10 @@ Reference of objects involved in hooks
     :members:
     :show-inheritance:
 
+.. autoclass:: _pytest.python.FixtureDef()
+    :members:
+    :show-inheritance:
+
 .. autoclass:: _pytest.runner.CallInfo()
     :members:
 

testing/python/setup_only.py

@@ -0,0 +1,221 @@
+import pytest
+
+
+@pytest.fixture(params=['--setup-only', '--setup-plan'], scope='module')
+def mode(request):
+    return request.param
+
+
+def test_show_only_active_fixtures(testdir, mode):
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture
+        def _arg0():
+            """hidden arg0 fixture"""
+        @pytest.fixture
+        def arg1():
+            """arg1 docstring"""
+        def test_arg1(arg1):
+            pass
+    ''')
+
+    result = testdir.runpytest(mode, p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        '*SETUP    F arg1*',
+        '*test_arg1 (fixtures used: arg1)',
+        '*TEARDOWN F arg1*',
+    ])
+    assert "_arg0" not in result.stdout.str()
+
+
+def test_show_different_scopes(testdir, mode):
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture
+        def arg_function():
+            """function scoped fixture"""
+        @pytest.fixture(scope='session')
+        def arg_session():
+            """session scoped fixture"""
+        def test_arg1(arg_session, arg_function):
+            pass
+    ''')
+
+    result = testdir.runpytest(mode, p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        'SETUP    S arg_session*',
+        '*SETUP    F arg_function*',
+        '*test_arg1 (fixtures used: arg_function, arg_session)',
+        '*TEARDOWN F arg_function*',
+        'TEARDOWN S arg_session*',
+    ])
+
+
+def test_show_nested_fixtures(testdir, mode):
+    testdir.makeconftest('''
+        import pytest
+        @pytest.fixture(scope='session')
+        def arg_same():
+            """session scoped fixture"""
+        ''')
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture(scope='function')
+        def arg_same(arg_same):
+            """function scoped fixture"""
+        def test_arg1(arg_same):
+            pass
+    ''')
+
+    result = testdir.runpytest(mode, p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        'SETUP    S arg_same*',
+        '*SETUP    F arg_same (fixtures used: arg_same)*',
+        '*test_arg1 (fixtures used: arg_same)',
+        '*TEARDOWN F arg_same*',
+        'TEARDOWN S arg_same*',
+    ])
+
+
+def test_show_fixtures_with_autouse(testdir, mode):
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture
+        def arg_function():
+            """function scoped fixture"""
+        @pytest.fixture(scope='session', autouse=True)
+        def arg_session():
+            """session scoped fixture"""
+        def test_arg1(arg_function):
+            pass
+    ''')
+
+    result = testdir.runpytest(mode, p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        'SETUP    S arg_session*',
+        '*SETUP    F arg_function*',
+        '*test_arg1 (fixtures used: arg_function, arg_session)',
+    ])
+
+
+def test_show_fixtures_with_parameters(testdir, mode):
+    testdir.makeconftest('''
+        import pytest
+        @pytest.fixture(scope='session', params=['foo', 'bar'])
+        def arg_same():
+            """session scoped fixture"""
+        ''')
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture(scope='function')
+        def arg_other(arg_same):
+            """function scoped fixture"""
+        def test_arg1(arg_other):
+            pass
+    ''')
+
+    result = testdir.runpytest(mode, p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        'SETUP    S arg_same?foo?',
+        'TEARDOWN S arg_same?foo?',
+        'SETUP    S arg_same?bar?',
+        'TEARDOWN S arg_same?bar?',
+    ])
+
+
+def test_show_fixtures_with_parameter_ids(testdir, mode):
+    testdir.makeconftest('''
+        import pytest
+        @pytest.fixture(
+            scope='session', params=['foo', 'bar'], ids=['spam', 'ham'])
+        def arg_same():
+            """session scoped fixture"""
+        ''')
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture(scope='function')
+        def arg_other(arg_same):
+            """function scoped fixture"""
+        def test_arg1(arg_other):
+            pass
+    ''')
+
+    result = testdir.runpytest(mode, p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        'SETUP    S arg_same?spam?',
+        'SETUP    S arg_same?ham?',
+    ])
+
+
+def test_show_fixtures_with_parameter_ids_function(testdir, mode):
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture(params=['foo', 'bar'], ids=lambda p: p.upper())
+        def foobar():
+            pass
+        def test_foobar(foobar):
+            pass
+    ''')
+
+    result = testdir.runpytest(mode, p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        '*SETUP    F foobar?FOO?',
+        '*SETUP    F foobar?BAR?',
+    ])
+
+
+def test_dynamic_fixture_request(testdir):
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture()
+        def dynamically_requested_fixture():
+            pass
+        @pytest.fixture()
+        def dependent_fixture(request):
+            request.getfuncargvalue('dynamically_requested_fixture')
+        def test_dyn(dependent_fixture):
+            pass
+    ''')
+
+    result = testdir.runpytest('--setup-only', p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        '*SETUP    F dynamically_requested_fixture',
+        '*TEARDOWN F dynamically_requested_fixture'
+    ])
+
+
+def test_capturing(testdir):
+    p = testdir.makepyfile('''
+        import pytest, sys
+        @pytest.fixture()
+        def one():
+            sys.stdout.write('this should be captured')
+            sys.stderr.write('this should also be captured')
+        @pytest.fixture()
+        def two(one):
+            assert 0
+        def test_capturing(two):
+            pass
+    ''')
+
+    result = testdir.runpytest('--setup-only', p)
+    result.stdout.fnmatch_lines([
+        'this should be captured',
+        'this should also be captured'
+    ])

testing/python/setup_plan.py

@@ -0,0 +1,19 @@
+def test_show_fixtures_and_test(testdir):
+    """ Verifies that fixtures are not executed. """
+    p = testdir.makepyfile('''
+        import pytest
+        @pytest.fixture
+        def arg():
+            assert False
+        def test_arg(arg):
+            assert False
+    ''')
+
+    result = testdir.runpytest("--setup-plan", p)
+    assert result.ret == 0
+
+    result.stdout.fnmatch_lines([
+        '*SETUP    F arg*',
+        '*test_arg (fixtures used: arg)',
+        '*TEARDOWN F arg*',
+    ])