From 17719b99a1ee638e38d4a34e270b746e1981cfd0 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 13 Oct 2010 12:26:14 +0200 Subject: [PATCH] select tests by call-id, add and refine documentation around it --HG-- branch : trunk --- doc/debugging.txt | 2 +- doc/doctest.txt | 2 +- doc/funcargs.txt | 35 ++++++++++++--- doc/mark.txt | 63 ++++++++++++++++++--------- doc/recwarn.txt | 3 +- doc/reporting.txt | 2 +- doc/skipping.txt | 4 +- doc/tmpdir.txt | 2 +- doc/unittest.txt | 2 +- pytest/plugin/mark.py | 87 +------------------------------------- pytest/plugin/python.py | 7 +-- testing/acceptance_test.py | 16 +++++++ 12 files changed, 101 insertions(+), 124 deletions(-) diff --git a/doc/debugging.txt b/doc/debugging.txt index 102489022..894aedf8b 100644 --- a/doc/debugging.txt +++ b/doc/debugging.txt @@ -1,5 +1,5 @@ -Tracebacks and debugging Python failures +debugging Python failures ================================================================= Stopping after the first (or N) failures diff --git a/doc/doctest.txt b/doc/doctest.txt index e8d7d8986..63f8c45f0 100644 --- a/doc/doctest.txt +++ b/doc/doctest.txt @@ -1,5 +1,5 @@ -collect and run doctests from modules and test files. +doctest integration for modules and test files. ========================================================= By default all files matching the ``test*.txt`` pattern will diff --git a/doc/funcargs.txt b/doc/funcargs.txt index 5033a11d4..e0cef3d6f 100644 --- a/doc/funcargs.txt +++ b/doc/funcargs.txt @@ -74,7 +74,6 @@ to see available function arguments (which you can also think of as "resources"). -.. _`contact possibilities`: ../contact.html .. _`parametrizing tests, generalized`: http://tetamap.wordpress.com/2009/05/13/parametrizing-python-tests-generalized/ .. _`blog post about the monkeypatch funcarg`: http://tetamap.wordpress.com/2009/03/03/monkeypatching-in-unit-tests-done-right/ @@ -157,15 +156,37 @@ Running this:: E assert 9 < 9 test_example.py:7: AssertionError - ==================== 1 failed, 9 passed in 0.04 seconds ==================== + ==================== 1 failed, 9 passed in 0.03 seconds ==================== -Here is what happens in detail: +Note that the ``pytest_generate_tests(metafunc)`` hook is called during +the test collection phase. You can have a look at it with this:: -1. ``pytest_generate_tests(metafunc)`` hook is called once for each test - function. It adds ten new function calls with explicit function arguments. + $ py.test --collectonly test_example.py + + + + + + + + + + + + + +If you want to select only the run with the value ``7`` you could do:: + + $ py.test -v -k 7 test_example.py # or -k test_func[7] + =========================== test session starts ============================ + platform linux2 -- Python 2.6.5 -- pytest-2.0.0dev0 -- /home/hpk/venv/0/bin/python + test path 1: test_example.py + + test_example.py:6: test_func[7] PASSED + + ======================== 9 tests deselected by '7' ========================= + ================== 1 passed, 9 deselected in 0.01 seconds ================== -2. **execute tests**: ``test_func(numiter)`` is called ten times with - ten different arguments. .. _`metafunc object`: diff --git a/doc/mark.txt b/doc/mark.txt index a4a3b07f3..bc7ba4064 100644 --- a/doc/mark.txt +++ b/doc/mark.txt @@ -1,8 +1,8 @@ .. _mark: -generic mechanism for marking python functions. -=============================================== +mark (attribute) python functions +================================================================= By using the ``py.test.mark`` helper you can instantiate decorators that will set named meta data on test functions. @@ -12,28 +12,29 @@ Marking a single function You can "mark" a test function with meta data like this:: + import py @py.test.mark.webtest def test_send_http(): ... -This will set a "Marker" instance as a function attribute named "webtest". -You can also specify parametrized meta data like this:: +This will set the function attribute ``webtest`` to a :py:meth:`MarkInfo` +instance. You can also specify parametrized meta data like this:: + # content of test_mark.py + + import py @py.test.mark.webtest(firefox=30) def test_receive(): - ... + pass -The named marker can be accessed like this later:: + @py.test.mark.webtest("functional", firefox=30) + def test_run_and_look(): + pass + +and access it from other places like this:: test_receive.webtest.kwargs['firefox'] == 30 - -In addition to set key-value pairs you can also use positional arguments:: - - @py.test.mark.webtest("triangular") - def test_receive(): - ... - -and later access it with ``test_receive.webtest.args[0] == 'triangular``. + test_run_and_look.webtest.args[0] == "functional" .. _`scoped-marking`: @@ -43,12 +44,14 @@ Marking whole classes or modules If you are programming with Python2.6 you may use ``py.test.mark`` decorators with classes to apply markers to all its test methods:: + # content of test_mark_classlevel.py + import py @py.test.mark.webtest class TestClass: def test_startup(self): - ... + pass def test_startup_and_more(self): - ... + pass This is equivalent to directly applying the decorator to the two test functions. @@ -79,8 +82,30 @@ methods defined in the module. Using "-k MARKNAME" to select tests ---------------------------------------------------- -You can use the ``-k`` command line option to select -tests:: +You can use the ``-k`` command line option to select tests:: - py.test -k webtest # will only run tests marked as webtest + $ py.test -k webtest # will only run tests marked as webtest + =========================== test session starts ============================ + platform linux2 -- Python 2.6.5 -- pytest-2.0.0dev0 + test path 1: /tmp/doc-exec-335 + + test_mark.py .. + test_mark_classlevel.py .. + + ========================= 4 passed in 0.01 seconds ========================= + +And you can also run all tests except the ones that match the keyword:: + + $ py.test -k-webtest # "-" negates but be careful to have no space before + =========================== test session starts ============================ + platform linux2 -- Python 2.6.5 -- pytest-2.0.0dev0 + test path 1: /tmp/doc-exec-335 + + ===================== 4 tests deselected by '-webtest' ===================== + ======================= 4 deselected in 0.01 seconds ======================= + +API reference for mark related objects +------------------------------------------------ + +.. autoclass:: pytest.plugin.mark.MarkInfo diff --git a/doc/recwarn.txt b/doc/recwarn.txt index 0fecd2f8e..94431db32 100644 --- a/doc/recwarn.txt +++ b/doc/recwarn.txt @@ -1,8 +1,7 @@ -helpers for asserting deprecation and other warnings. +asserting deprecation and other warnings. ===================================================== - recwarn function argument ------------------------------------ diff --git a/doc/reporting.txt b/doc/reporting.txt index ad7d0d285..36d91b740 100644 --- a/doc/reporting.txt +++ b/doc/reporting.txt @@ -1,4 +1,4 @@ -Sending test report output to XML files or to remote services +XML, pastebin services and other reporting ================================================================= creating JUnitXML format files diff --git a/doc/skipping.txt b/doc/skipping.txt index 95a9a445d..324fea6df 100644 --- a/doc/skipping.txt +++ b/doc/skipping.txt @@ -1,6 +1,6 @@ -advanced skipping for python test functions, classes or modules. -================================================================ +skip and xfail mechanisms +===================================================================== You can mark test functions for a conditional *skip* or as *xfail*, expected-to-fail. Skipping a test avoids running a test. diff --git a/doc/tmpdir.txt b/doc/tmpdir.txt index 68b8162fb..8876a9fe2 100644 --- a/doc/tmpdir.txt +++ b/doc/tmpdir.txt @@ -1,5 +1,5 @@ -working with temporary directories and files +temporary directories and files ================================================ the 'tmpdir' test function argument diff --git a/doc/unittest.txt b/doc/unittest.txt index 8e4566188..dc2a88aeb 100644 --- a/doc/unittest.txt +++ b/doc/unittest.txt @@ -1,4 +1,4 @@ -automatically discover and run traditional "unittest.py" style tests. +unittest.py style testing integration ===================================================================== py.test has limited support for running Python `unittest.py style`_ tests. diff --git a/pytest/plugin/mark.py b/pytest/plugin/mark.py index f49c74786..ae7470d34 100644 --- a/pytest/plugin/mark.py +++ b/pytest/plugin/mark.py @@ -1,87 +1,4 @@ -""" -generic mechanism for marking python functions. - -By using the ``py.test.mark`` helper you can instantiate -decorators that will set named meta data on test functions. - -Marking a single function ----------------------------------------------------- - -You can "mark" a test function with meta data like this:: - - @py.test.mark.webtest - def test_send_http(): - ... - -This will set a "Marker" instance as a function attribute named "webtest". -You can also specify parametrized meta data like this:: - - @py.test.mark.webtest(firefox=30) - def test_receive(): - ... - -The named marker can be accessed like this later:: - - test_receive.webtest.kwargs['firefox'] == 30 - -In addition to set key-value pairs you can also use positional arguments:: - - @py.test.mark.webtest("triangular") - def test_receive(): - ... - -and later access it with ``test_receive.webtest.args[0] == 'triangular``. - -.. _`scoped-marking`: - -Marking whole classes or modules ----------------------------------------------------- - -If you are programming with Python2.6 you may use ``py.test.mark`` decorators -with classes to apply markers to all its test methods:: - - @py.test.mark.webtest - class TestClass: - def test_startup(self): - ... - def test_startup_and_more(self): - ... - -This is equivalent to directly applying the decorator to the -two test functions. - -To remain compatible with Python2.5 you can also set a -``pytestmark`` attribute on a TestClass like this:: - - import py - - class TestClass: - pytestmark = py.test.mark.webtest - -or if you need to use multiple markers you can use a list:: - - import py - - class TestClass: - pytestmark = [py.test.mark.webtest, pytest.mark.slowtest] - -You can also set a module level marker:: - - import py - pytestmark = py.test.mark.webtest - -in which case it will be applied to all functions and -methods defined in the module. - -Using "-k MARKNAME" to select tests ----------------------------------------------------- - -You can use the ``-k`` command line option to select -tests:: - - py.test -k webtest # will only run tests marked as webtest - -""" +""" generic mechanism for marking and selecting python functions. """ import py def pytest_namespace(): @@ -219,7 +136,6 @@ class MarkInfo: return "" % ( self._name, self.args, self.kwargs) - def pytest_pycollect_makeitem(__multicall__, collector, name, obj): item = __multicall__.execute() if isinstance(item, py.test.collect.Function): @@ -237,5 +153,4 @@ def pytest_pycollect_makeitem(__multicall__, collector, name, obj): if isinstance(mark, MarkDecorator): mark(func) item.keywords.update(py.builtin._getfuncdict(func) or {}) - return item diff --git a/pytest/plugin/python.py b/pytest/plugin/python.py index a316fce46..88af38cf3 100644 --- a/pytest/plugin/python.py +++ b/pytest/plugin/python.py @@ -54,7 +54,6 @@ def pytest_collect_file(path, parent): def pytest_pycollect_makemodule(path, parent): return parent.Module(path, parent) - def pytest_pycollect_makeitem(__multicall__, collector, name, obj): res = __multicall__.execute() if res is not None: @@ -204,7 +203,7 @@ class PyCollectorMixin(PyobjMixin, pytest.collect.Collector): for callspec in metafunc._calls: subname = "%s[%s]" %(name, callspec.id) function = self.Function(name=subname, parent=self, - callspec=callspec, callobj=funcobj) + callspec=callspec, callobj=funcobj, keywords={callspec.id:True}) l.append(function) return l @@ -414,7 +413,7 @@ class Function(FunctionMixin, pytest.collect.Item): """ _genid = None def __init__(self, name, parent=None, args=None, config=None, - callspec=None, callobj=_dummy, collection=None): + callspec=None, callobj=_dummy, keywords=None, collection=None): super(Function, self).__init__(name, parent, config=config, collection=collection) self._args = args @@ -432,6 +431,8 @@ class Function(FunctionMixin, pytest.collect.Item): self._obj = callobj self.function = getattr(self.obj, 'im_func', self.obj) self.keywords.update(py.builtin._getfuncdict(self.obj) or {}) + if keywords: + self.keywords.update(keywords) def _getobj(self): name = self.name diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 6314c1041..9e04ff7bf 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -220,3 +220,19 @@ class TestGeneralUsage: p1 = testdir.makepyfile("def test_fail(): 0/0") res = testdir.run(py.std.sys.executable, "-m", "py.test", str(p1)) assert res.ret == 1 + + def test_skip_on_generated_funcarg_id(self, testdir): + testdir.makeconftest(""" + import py + def pytest_generate_tests(metafunc): + metafunc.addcall({'x': 3}, id='hello-123') + def pytest_runtest_setup(item): + print (item.keywords) + if 'hello-123' in item.keywords: + py.test.skip("hello") + assert 0 + """) + p = testdir.makepyfile("""def test_func(x): pass""") + res = testdir.runpytest(p) + assert res.ret == 0 + res.stdout.fnmatch_lines(["*1 skipped*"])