From 7b273b8577f4d13338721bad3dfbee109e89cb15 Mon Sep 17 00:00:00 2001 From: "david@mcbf.net" Date: Sat, 26 Jul 2014 15:11:05 +0200 Subject: [PATCH 1/5] Add mark.xfail argument raises so that unexpected exceptions show up as test failures. --HG-- branch : xfail-cause --- _pytest/skipping.py | 23 ++++++++++++++---- testing/test_skipping.py | 50 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/_pytest/skipping.py b/_pytest/skipping.py index 6ddbb6553..e65ac577f 100644 --- a/_pytest/skipping.py +++ b/_pytest/skipping.py @@ -26,11 +26,13 @@ def pytest_configure(config): "http://pytest.org/latest/skipping.html" ) config.addinivalue_line("markers", - "xfail(condition, reason=None, run=True): mark the the test function " + "xfail(condition, reason=None, run=True, raises=None): mark the the test function " "as an expected failure if eval(condition) has a True value. " "Optionally specify a reason for better reporting and run=False if " - "you don't even want to execute the test function. See " - "http://pytest.org/latest/skipping.html" + "you don't even want to execute the test function. If only specific " + "exception(s) are expected, you can list them in raises, and if the test fails " + "in other ways, it will be reported as a true failure. " + "See http://pytest.org/latest/skipping.html" ) def pytest_namespace(): @@ -60,6 +62,15 @@ class MarkEvaluator: def wasvalid(self): return not hasattr(self, 'exc') + def invalidraise(self, exctype): + raises = self.get('raises') + if not raises: + return + if isinstance(raises, tuple): + return exctype not in raises + else: + return raises != exctype + def istrue(self): try: return self._istrue() @@ -171,7 +182,11 @@ def pytest_runtest_makereport(__multicall__, item, call): if not item.config.option.runxfail: if evalxfail.wasvalid() and evalxfail.istrue(): if call.excinfo: - rep.outcome = "skipped" + if evalxfail.invalidraise(call.excinfo.type): + rep.outcome = "failed" + return rep + else: + rep.outcome = "skipped" elif call.when == "call": rep.outcome = "failed" else: diff --git a/testing/test_skipping.py b/testing/test_skipping.py index d85a5d635..fdf73efde 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -330,6 +330,53 @@ class TestXFail: "*1 xfailed*", ]) + def test_xfail_raises_match(self, testdir): + p = testdir.makepyfile(""" + import pytest + @pytest.mark.xfail(raises=TypeError) + def test_raises(): + raise TypeError() + """) + result = testdir.runpytest(p) + result.stdout.fnmatch_lines([ + "*1 xfailed*", + ]) + + def test_xfail_raises_mismatch(self, testdir): + p = testdir.makepyfile(""" + import pytest + @pytest.mark.xfail(raises=IndexError) + def test_raises(): + raise TypeError() + """) + result = testdir.runpytest(p) + result.stdout.fnmatch_lines([ + "*1 failed*", + ]) + def test_xfail_raises_tuple_match(self, testdir): + p = testdir.makepyfile(""" + import pytest + @pytest.mark.xfail(raises=(AttributeError, TypeError)) + def test_raises(): + raise TypeError() + """) + result = testdir.runpytest(p) + result.stdout.fnmatch_lines([ + "*1 xfailed*", + ]) + + def test_xfail_raises_tuple_mismatch(self, testdir): + p = testdir.makepyfile(""" + import pytest + @pytest.mark.xfail(raises=(AttributeError, IndexError)) + def test_raises(): + raise TypeError() + """) + result = testdir.runpytest(p) + result.stdout.fnmatch_lines([ + "*1 failed*", + ]) + class TestXFailwithSetupTeardown: def test_failing_setup_issue9(self, testdir): testdir.makepyfile(""" @@ -575,7 +622,7 @@ def test_default_markers(testdir): result = testdir.runpytest("--markers") result.stdout.fnmatch_lines([ "*skipif(*condition)*skip*", - "*xfail(*condition, reason=None, run=True)*expected failure*", + "*xfail(*condition, reason=None, run=True, raises=None)*expected failure*", ]) def test_xfail_test_setup_exception(testdir): @@ -617,7 +664,6 @@ def test_imperativeskip_on_xfail_test(testdir): *2 skipped* """) - class TestBooleanCondition: def test_skipif(self, testdir): testdir.makepyfile(""" From 6a4492a22dac45fd7e3cb1c7adc987328bfc662e Mon Sep 17 00:00:00 2001 From: "david@mcbf.net" Date: Sat, 26 Jul 2014 17:46:50 +0200 Subject: [PATCH 2/5] isinstance() on exception value instead of comparing types, consolidate tests --HG-- branch : xfail-cause --- _pytest/skipping.py | 9 +++---- testing/test_skipping.py | 53 +++++++++++----------------------------- 2 files changed, 17 insertions(+), 45 deletions(-) diff --git a/_pytest/skipping.py b/_pytest/skipping.py index e65ac577f..aee98ffa8 100644 --- a/_pytest/skipping.py +++ b/_pytest/skipping.py @@ -62,14 +62,11 @@ class MarkEvaluator: def wasvalid(self): return not hasattr(self, 'exc') - def invalidraise(self, exctype): + def invalidraise(self, exc): raises = self.get('raises') if not raises: return - if isinstance(raises, tuple): - return exctype not in raises - else: - return raises != exctype + return not isinstance(exc, raises) def istrue(self): try: @@ -182,7 +179,7 @@ def pytest_runtest_makereport(__multicall__, item, call): if not item.config.option.runxfail: if evalxfail.wasvalid() and evalxfail.istrue(): if call.excinfo: - if evalxfail.invalidraise(call.excinfo.type): + if evalxfail.invalidraise(call.excinfo.value): rep.outcome = "failed" return rep else: diff --git a/testing/test_skipping.py b/testing/test_skipping.py index fdf73efde..cc5c29c0e 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -330,52 +330,27 @@ class TestXFail: "*1 xfailed*", ]) - def test_xfail_raises_match(self, testdir): + + @pytest.mark.parametrize('params', [('TypeError', 'TypeError', "*1 xfailed*"), + ('(AttributeError, TypeError)', 'TypeError', + "*1 xfailed*"), + ('TypeError', 'IndexError', "*1 failed*"), + ('(AttributeError, TypeError)', 'IndexError', + "*1 failed*"), + ]) + def test_xfail_raises(self, params, testdir): + expected, actual, matchline = params p = testdir.makepyfile(""" import pytest - @pytest.mark.xfail(raises=TypeError) + @pytest.mark.xfail(raises=%s) def test_raises(): - raise TypeError() - """) + raise %s() + """ % (expected, actual)) result = testdir.runpytest(p) result.stdout.fnmatch_lines([ - "*1 xfailed*", + matchline, ]) - def test_xfail_raises_mismatch(self, testdir): - p = testdir.makepyfile(""" - import pytest - @pytest.mark.xfail(raises=IndexError) - def test_raises(): - raise TypeError() - """) - result = testdir.runpytest(p) - result.stdout.fnmatch_lines([ - "*1 failed*", - ]) - def test_xfail_raises_tuple_match(self, testdir): - p = testdir.makepyfile(""" - import pytest - @pytest.mark.xfail(raises=(AttributeError, TypeError)) - def test_raises(): - raise TypeError() - """) - result = testdir.runpytest(p) - result.stdout.fnmatch_lines([ - "*1 xfailed*", - ]) - - def test_xfail_raises_tuple_mismatch(self, testdir): - p = testdir.makepyfile(""" - import pytest - @pytest.mark.xfail(raises=(AttributeError, IndexError)) - def test_raises(): - raise TypeError() - """) - result = testdir.runpytest(p) - result.stdout.fnmatch_lines([ - "*1 failed*", - ]) class TestXFailwithSetupTeardown: def test_failing_setup_issue9(self, testdir): From 91e2b232588c5ff1c3dea550b9649df523e66728 Mon Sep 17 00:00:00 2001 From: "david@mcbf.net" Date: Sat, 26 Jul 2014 18:10:32 +0200 Subject: [PATCH 3/5] Update documentation --HG-- branch : xfail-cause --- doc/en/assert.txt | 16 ++++++++++++++++ doc/en/example/xfail_demo.py | 5 +++++ doc/en/skipping.txt | 5 +++++ 3 files changed, 26 insertions(+) diff --git a/doc/en/assert.txt b/doc/en/assert.txt index 6d9f60448..1dd7ba0c0 100644 --- a/doc/en/assert.txt +++ b/doc/en/assert.txt @@ -95,6 +95,22 @@ asserts that the given ``ExpectedException`` is raised. The reporter will provide you with helpful output in case of failures such as *no exception* or *wrong exception*. +Note that it is also possible to specify a "raises" argument to +``pytest.mark.xfail``, which checks that the test is failing in a more +specific way than just having any exception raised:: + + @pytest.mark.xfail(raises=IndexError) + def test_f(): + f() + +Using ``pytest.raises`` is likely to be better for cases where you are testing +exceptions your own code is deliberately raising, whereas using +``@pytest.mark.xfail`` with a check function is probably better for something +like documenting unfixed bugs (where the test describes what "should" happen) +or bugs in dependencies. + + + .. _newreport: Making use of context-sensitive comparisons diff --git a/doc/en/example/xfail_demo.py b/doc/en/example/xfail_demo.py index 75cb7bea1..79752680a 100644 --- a/doc/en/example/xfail_demo.py +++ b/doc/en/example/xfail_demo.py @@ -23,3 +23,8 @@ def test_hello5(): def test_hello6(): pytest.xfail("reason") + +@xfail(raises=IndexError) +def test_hello7() + x = [] + assert x[1] == 1 diff --git a/doc/en/skipping.txt b/doc/en/skipping.txt index c896d404d..ab6c838ba 100644 --- a/doc/en/skipping.txt +++ b/doc/en/skipping.txt @@ -149,6 +149,11 @@ on a particular platform:: def test_function(): ... +If you want to be more specific as to why the test is failing, you can specify +a single exception, or a list of exceptions, in the ``raises`` argument. Then +the test will be reported as a regular failure if it fails with an +exception not mentioned in ``raises``. + You can furthermore prevent the running of an "xfail" test or specify a reason such as a bug ID or similar. Here is a simple test file with the several usages: From 309e3d38a0f45253413a5a803b5e090b6e2f0079 Mon Sep 17 00:00:00 2001 From: "david@mcbf.net" Date: Sat, 26 Jul 2014 18:19:27 +0200 Subject: [PATCH 4/5] Directly pass multiple parameters with mark.parametrize() --HG-- branch : xfail-cause --- testing/test_skipping.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/testing/test_skipping.py b/testing/test_skipping.py index cc5c29c0e..45652703a 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -331,15 +331,13 @@ class TestXFail: ]) - @pytest.mark.parametrize('params', [('TypeError', 'TypeError', "*1 xfailed*"), - ('(AttributeError, TypeError)', 'TypeError', - "*1 xfailed*"), - ('TypeError', 'IndexError', "*1 failed*"), - ('(AttributeError, TypeError)', 'IndexError', - "*1 failed*"), - ]) - def test_xfail_raises(self, params, testdir): - expected, actual, matchline = params + @pytest.mark.parametrize('expected, actual, matchline', + [('TypeError', 'TypeError', "*1 xfailed*"), + ('(AttributeError, TypeError)', 'TypeError', "*1 xfailed*"), + ('TypeError', 'IndexError', "*1 failed*"), + ('(AttributeError, TypeError)', 'IndexError', "*1 failed*"), + ]) + def test_xfail_raises(self, expected, actual, matchline, testdir): p = testdir.makepyfile(""" import pytest @pytest.mark.xfail(raises=%s) From d08c4ce0ad59dd8a3c428d22477391807b783509 Mon Sep 17 00:00:00 2001 From: "david@mcbf.net" Date: Sat, 26 Jul 2014 18:24:55 +0200 Subject: [PATCH 5/5] Tiny example update for clarification --HG-- branch : xfail-cause --- doc/en/example/xfail_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/en/example/xfail_demo.py b/doc/en/example/xfail_demo.py index 79752680a..c6e147e4d 100644 --- a/doc/en/example/xfail_demo.py +++ b/doc/en/example/xfail_demo.py @@ -27,4 +27,4 @@ def test_hello6(): @xfail(raises=IndexError) def test_hello7() x = [] - assert x[1] == 1 + x[1] = 1