From 6b0f0adf5b5dd97f718f20ba77ecf375528b4bbd Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 20 Jul 2012 14:16:50 +0200 Subject: [PATCH] implement a scope/parametrized examples using the so-far new features also fix a bug with scoping/parametrization --- _pytest/__init__.py | 2 +- _pytest/python.py | 5 +- doc/en/conf.py | 2 +- doc/en/example/index.txt | 1 + doc/en/example/newexamples.txt | 183 +++++++++++++++++++++++++++++++++ doc/en/resources.txt | 18 +++- setup.py | 2 +- testing/test_python.py | 15 ++- 8 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 doc/en/example/newexamples.txt diff --git a/_pytest/__init__.py b/_pytest/__init__.py index 39b831d87..aa0e0f06c 100644 --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.3.0.dev2' +__version__ = '2.3.0.dev3' diff --git a/_pytest/python.py b/_pytest/python.py index 62cf3d936..6baacb8cf 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1030,8 +1030,11 @@ class FuncargRequest: check_scope(self.scope, scope) __tracebackhide__ = False mp.setattr(self, "scope", scope) + kwargs = {} + if hasattr(self, "param"): + kwargs["extrakey"] = param val = self.cached_setup(lambda: funcargfactory(request=self), - scope=scope) + scope=scope, **kwargs) else: val = funcargfactory(request=self) mp.undo() diff --git a/doc/en/conf.py b/doc/en/conf.py index 59fc8089e..46235ca5d 100644 --- a/doc/en/conf.py +++ b/doc/en/conf.py @@ -17,7 +17,7 @@ # # The full version, including alpha/beta/rc tags. # The short X.Y version. -version = release = "2.3.0.dev5" +version = release = "2.3.0.dev3" import sys, os diff --git a/doc/en/example/index.txt b/doc/en/example/index.txt index e0b628316..3a31c5611 100644 --- a/doc/en/example/index.txt +++ b/doc/en/example/index.txt @@ -21,3 +21,4 @@ need more examples or have questions. Also take a look at the :ref:`comprehensiv markers.txt pythoncollection.txt nonpython.txt + newexamples.txt diff --git a/doc/en/example/newexamples.txt b/doc/en/example/newexamples.txt new file mode 100644 index 000000000..8736cd3ea --- /dev/null +++ b/doc/en/example/newexamples.txt @@ -0,0 +1,183 @@ + +Scoping and parametrizing Funcarg factories +--------------------------------------------------- + +.. regendoc:wipe + +.. versionadded:: 2.3 + +The ``@pytest.mark.funcarg`` marker allows + +* to mark a function without a ``pytest_funcarg__`` as a factory +* to cause parametrization and run all tests multiple times + with the multiple created resources +* to set a scope which determines the level of caching + +Here is a simple example for defining a SMTPServer server +object with a session scope:: + + # content of conftest.py + import pytest + import smtplib + + @pytest.mark.funcarg(scope="session") + def smtp(request): + smtp = smtplib.SMTP("merlinux.eu") + request.addfinalizer(smtp.close) + return smtp + +You can now use this server connection from your tests:: + + # content of test_module.py + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + assert "merlinux" in response[1] + assert 0 # for demo purposes + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + assert 0 # for demo purposes + +If you run the tests:: + + $ py.test -q + collecting ... collected 2 items + FF + ================================= FAILURES ================================= + ________________________________ test_ehlo _________________________________ + + smtp = + + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + assert "merlinux" in response[1] + > assert 0 # for demo purposes + E assert 0 + + test_module.py:5: AssertionError + ________________________________ test_noop _________________________________ + + smtp = + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:10: AssertionError + 2 failed in 0.14 seconds + +you will see the two ``assert 0`` failing and can see that +the same (session-scoped) object was passed into the two test functions. + +If you now want to test multiple servers you can simply parametrize +the ``smtp`` factory:: + + # content of conftest.py + import pytest + import smtplib + + @pytest.mark.funcarg(scope="session", + params=["merlinux.eu", "mail.python.org"]) + def smtp(request): + smtp = smtplib.SMTP(request.param) + def fin(): + print "closing", smtp + smtp.close() + request.addfinalizer(fin) + return smtp + +Only two lines changed and no test code needs to change. Let's do +another run:: + + $ py.test -q + collecting ... collected 4 items + FFFF + ================================= FAILURES ================================= + __________________________ test_ehlo[merlinux.eu] __________________________ + + smtp = + + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + assert "merlinux" in response[1] + > assert 0 # for demo purposes + E assert 0 + + test_module.py:5: AssertionError + ________________________ test_ehlo[mail.python.org] ________________________ + + smtp = + + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + > assert "merlinux" in response[1] + E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' + + test_module.py:4: AssertionError + __________________________ test_noop[merlinux.eu] __________________________ + + smtp = + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:10: AssertionError + ________________________ test_noop[mail.python.org] ________________________ + + smtp = + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:10: AssertionError + 4 failed in 5.70 seconds + closing + closing + +We get four failures because we are running the two tests twice with +different ``smtp`` instantiations as defined on the factory. +Note that with the ``mail.python.org`` connection the second tests +fails already in ``test_ehlo`` because it wrongly expects a specific +server string. + +You can look at what tests pytest collects without running them:: + + $ py.test --collectonly + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3 + plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov + collecting ... collected 4 items + + + + + + + ============================= in 0.02 seconds ============================= + +And you can run without output capturing and minimized failure reporting to check that the ``smtp`` objects are finalized at session end:: + + $ py.test --tb=line -q -s + collecting ... collected 4 items + FFFF + ================================= FAILURES ================================= + /home/hpk/tmp/doc-exec-330/test_module.py:5: assert 0 + /home/hpk/tmp/doc-exec-330/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' + /home/hpk/tmp/doc-exec-330/test_module.py:10: assert 0 + /home/hpk/tmp/doc-exec-330/test_module.py:10: assert 0 + 4 failed in 6.02 seconds + closing + closing diff --git a/doc/en/resources.txt b/doc/en/resources.txt index cd0554997..9a43341e2 100644 --- a/doc/en/resources.txt +++ b/doc/en/resources.txt @@ -94,6 +94,8 @@ each of these problems. Direct scoping of funcarg factories -------------------------------------------------------- +.. note:: Implemented + Instead of calling cached_setup(), you can decorate your factory to state its scope:: @@ -116,6 +118,8 @@ still want to have your factory get called on a per-item basis. Direct parametrization of funcarg resource factories ---------------------------------------------------------- +.. note:: Implemented + Previously, funcarg factories could not directly cause parametrization. You needed to specify a ``@parametrize`` or implement a ``pytest_generate_tests`` hook to perform parametrization, i.e. calling a test multiple times with different value sets. pytest-2.X introduces a decorator for use @@ -154,6 +158,8 @@ factory function. Direct usage of funcargs with funcargs factories ---------------------------------------------------------- +.. note:: Not Implemented - unclear if to. + You can now directly use funcargs in funcarg factories. Example:: @pytest.mark.funcarg(scope="session") @@ -169,6 +175,8 @@ actually parametrized. The "pytest_funcarg__" prefix becomes optional ----------------------------------------------------- +.. note:: Implemented + When using the ``@funcarg`` decorator you do not need to use the ``pytest_funcarg__`` prefix any more:: @@ -186,6 +194,8 @@ that the funcarg factory will be called for each test function invocation. support for a new @setup marker ------------------------------------------------------ +.. note:: Not-Implemented, still under consideration if to. + pytest for a long time offered a pytest_configure and a pytest_sessionstart hook which are often used to setup global resources. This suffers from several problems: @@ -252,6 +262,8 @@ funcarg which represents a takes on each of the values in the Using funcarg resources in xUnit setup methods ------------------------------------------------------------ +.. note:: Not implemented. Not clear if to. + XXX Consider this feature in contrast to the @setup feature - probably introducing one of them is better and the @setup decorator is more flexible. @@ -296,6 +308,8 @@ resource, collection would early on report a ScopingMismatch error. the "directory" caching scope -------------------------------------------- +.. note:: Not implemented. + All API accepting a scope (:py:func:`cached_setup()` and the new funcarg/setup decorators) now also accept a "directory" specification. This allows to restrict/cache resource values on a @@ -304,6 +318,8 @@ per-directory level. funcarg and setup discovery now happens at collection time --------------------------------------------------------------------- +.. note:: Partially implemented - collectonly shows no extra information + pytest-2.X takes care to discover funcarg factories and setup_X methods at collection time. This is more efficient especially for large test suites. Moreover, a call to "py.test --collectonly" should be able to show @@ -404,7 +420,7 @@ through the ``node.session`` attribute:: ISSUES -------------------------- -decorating a parametrized funcarg factory: +decorating a parametrized funcarg factory:: @pytest.mark.funcarg(scope="session", params=["mysql", "pg"]) def db(request): diff --git a/setup.py b/setup.py index d25b3f533..164f80351 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def main(): name='pytest', description='py.test: simple powerful testing with Python', long_description = long_description, - version='2.3.0.dev2', + version='2.3.0.dev3', url='http://pytest.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], diff --git a/testing/test_python.py b/testing/test_python.py index 10d5dcbb8..2a5eb8a98 100644 --- a/testing/test_python.py +++ b/testing/test_python.py @@ -1826,4 +1826,17 @@ class TestFuncargMarker: reprec = testdir.inline_run() reprec.assertoutcome(passed=1) - + def test_parametrize_and_scope(self, testdir): + testdir.makepyfile(""" + import pytest + @pytest.mark.funcarg(scope="module", params=["a", "b", "c"]) + def pytest_funcarg__arg(request): + return request.param + l = [] + def test_param(arg): + l.append(arg) + def test_result(): + assert l == list("abc") + """) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=4)