From 4fc20d09feb39d31d6c13d48deae717c2d816cf3 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Fri, 5 Aug 2016 19:25:55 +0100 Subject: [PATCH 01/21] Change outcome to 'passed' for xfail unless it's strict --- _pytest/skipping.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/_pytest/skipping.py b/_pytest/skipping.py index 18e038d2c..66964f945 100644 --- a/_pytest/skipping.py +++ b/_pytest/skipping.py @@ -230,7 +230,8 @@ def pytest_runtest_makereport(item, call): if hasattr(item, '_unexpectedsuccess') and rep.when == "call": # we need to translate into how pytest encodes xpass rep.wasxfail = "reason: " + repr(item._unexpectedsuccess) - rep.outcome = "failed" + # TODO: Do we need to check for strict xfail here as well? + rep.outcome = "passed" elif item.config.option.runxfail: pass # don't interefere elif call.excinfo and call.excinfo.errisinstance(pytest.xfail.Exception): @@ -245,7 +246,12 @@ def pytest_runtest_makereport(item, call): rep.outcome = "skipped" rep.wasxfail = evalxfail.getexplanation() elif call.when == "call": - rep.outcome = "failed" # xpass outcome + strict_default = item.config.getini('xfail_strict') + is_strict_xfail = evalxfail.get('strict', strict_default) + if is_strict_xfail: + rep.outcome = "failed" + else: + rep.outcome = "passed" rep.wasxfail = evalxfail.getexplanation() elif evalskip is not None and rep.skipped and type(rep.longrepr) is tuple: # skipped by mark.skipif; change the location of the failure @@ -260,7 +266,7 @@ def pytest_report_teststatus(report): if hasattr(report, "wasxfail"): if report.skipped: return "xfailed", "x", "xfail" - elif report.failed: + elif report.passed: return "xpassed", "X", ("XPASS", {'yellow': True}) # called by the terminalreporter instance/plugin From e4028b4505148804fe30634a49980b4fc7de9a37 Mon Sep 17 00:00:00 2001 From: Christian Boelsen Date: Mon, 8 Aug 2016 13:29:20 +0100 Subject: [PATCH 02/21] Fix #1798 to include errors in total tests in junit xml output. --- AUTHORS | 1 + CHANGELOG.rst | 3 +++ _pytest/junitxml.py | 2 +- testing/test_junitxml.py | 27 ++++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0d5e1bb64..254207071 100644 --- a/AUTHORS +++ b/AUTHORS @@ -24,6 +24,7 @@ Carl Friedrich Bolz Charles Cloud Charnjit SiNGH (CCSJ) Chris Lamb +Christian Boelsen Christian Theunert Christian Tismer Christopher Gilling diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5cebf083d..ca2672f53 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -65,6 +65,8 @@ * Fixed scope overriding inside metafunc.parametrize (`#634`_). Thanks to `@Stranger6667`_ for the PR. +* Fixed the total tests tally in junit xml output (`#1798`_). + * * @@ -85,6 +87,7 @@ .. _#1597: https://github.com/pytest-dev/pytest/pull/1597 .. _#1605: https://github.com/pytest-dev/pytest/issues/1605 .. _#1626: https://github.com/pytest-dev/pytest/pull/1626 +.. _#1798: https://github.com/pytest-dev/pytest/pull/1798 .. _#460: https://github.com/pytest-dev/pytest/pull/460 .. _#634: https://github.com/pytest-dev/pytest/issues/634 .. _#717: https://github.com/pytest-dev/pytest/issues/717 diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py index 4c2b9d149..6041cb6f0 100644 --- a/_pytest/junitxml.py +++ b/_pytest/junitxml.py @@ -369,7 +369,7 @@ class LogXML(object): suite_stop_time = time.time() suite_time_delta = suite_stop_time - self.suite_start_time - numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped'] + numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped'] + self.stats['error'] logfile.write('') logfile.write(Junit.testsuite( diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index a4f10dec5..c29381a86 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -102,6 +102,27 @@ class TestPython: node = dom.find_first_by_tag("testsuite") node.assert_attr(name="pytest", errors=0, failures=1, skips=3, tests=5) + def test_summing_simple_with_errors(self, testdir): + testdir.makepyfile(""" + import pytest + @pytest.fixture + def fixture(): + raise Exception() + def test_pass(): + pass + def test_fail(): + assert 0 + def test_error(fixture): + pass + @pytest.mark.xfail + def test_xpass(): + assert 1 + """) + result, dom = runandparse(testdir) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(name="pytest", errors=1, failures=1, skips=1, tests=4) + def test_timing_function(self, testdir): testdir.makepyfile(""" import time, pytest @@ -128,7 +149,7 @@ class TestPython: result, dom = runandparse(testdir) assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(errors=1, tests=0) + node.assert_attr(errors=1, tests=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr( file="test_setup_error.py", @@ -195,7 +216,7 @@ class TestPython: result, dom = runandparse(testdir) assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(errors=1, tests=0) + node.assert_attr(errors=1, tests=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr(classname="pytest", name="internal") fnode = tnode.find_first_by_tag("error") @@ -341,7 +362,7 @@ class TestPython: result, dom = runandparse(testdir) assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(errors=1, tests=0) + node.assert_attr(errors=1, tests=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr( file="test_collect_error.py", From c4d9c7ea55db731b40764dcad7abbf0692690f8f Mon Sep 17 00:00:00 2001 From: Christian Boelsen Date: Mon, 8 Aug 2016 13:45:10 +0100 Subject: [PATCH 03/21] Add thanks as requested. --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ca2672f53..ed7adaf2e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -66,6 +66,7 @@ Thanks to `@Stranger6667`_ for the PR. * Fixed the total tests tally in junit xml output (`#1798`_). + Thanks to `@cryporchild`_ for the PR. * @@ -96,6 +97,7 @@ .. _@bagerard: https://github.com/bagerard .. _@BeyondEvil: https://github.com/BeyondEvil .. _@blueyed: https://github.com/blueyed +.. _@cryporchild: https://github.com/cryporchild .. _@davehunt: https://github.com/davehunt .. _@DRMacIver: https://github.com/DRMacIver .. _@eolo999: https://github.com/eolo999 From 10a6ed17071c2e416a917ecaac8f4b5cf246a4bc Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Fri, 12 Aug 2016 23:18:02 +0100 Subject: [PATCH 04/21] Update unittest test to expect failure for an unexpected success --- testing/test_unittest.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/testing/test_unittest.py b/testing/test_unittest.py index 144aad79b..39a31cfea 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -588,23 +588,37 @@ def test_unittest_typerror_traceback(testdir): assert result.ret == 1 @pytest.mark.skipif("sys.version_info < (2,7)") -def test_unittest_unexpected_failure(testdir): +def test_unittest_expected_failure_for_failing_test_is_xfail(testdir): testdir.makepyfile(""" import unittest class MyTestCase(unittest.TestCase): @unittest.expectedFailure - def test_func1(self): - assert 0 - @unittest.expectedFailure - def test_func2(self): - assert 1 + def test_failing_test_is_xfail(self): + assert False """) result = testdir.runpytest("-rxX") result.stdout.fnmatch_lines([ - "*XFAIL*MyTestCase*test_func1*", - "*XPASS*MyTestCase*test_func2*", - "*1 xfailed*1 xpass*", + "*XFAIL*MyTestCase*test_failing_test_is_xfail*", + "*1 xfailed*", ]) + assert result.ret == 0 + +@pytest.mark.skipif("sys.version_info < (2,7)") +def test_unittest_expected_failure_for_passing_test_is_fail(testdir): + testdir.makepyfile(""" + import unittest + class MyTestCase(unittest.TestCase): + @unittest.expectedFailure + def test_passing_test_is_fail(self): + assert True + """) + result = testdir.runpytest("-rxX") + result.stdout.fnmatch_lines([ + "*FAILURES*", + "*MyTestCase*test_passing_test_is_fail*", + "*1 failed*", + ]) + assert result.ret == 1 @pytest.mark.parametrize('fix_type, stmt', [ From 296f42a2c9f0870c793444837ae58c2da816f9e7 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Fri, 12 Aug 2016 23:18:36 +0100 Subject: [PATCH 05/21] Treat unittest.expectedFailure pass as a failure --- _pytest/skipping.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/_pytest/skipping.py b/_pytest/skipping.py index 66964f945..bd27870d6 100644 --- a/_pytest/skipping.py +++ b/_pytest/skipping.py @@ -228,10 +228,13 @@ def pytest_runtest_makereport(item, call): evalskip = getattr(item, '_evalskip', None) # unitttest special case, see setting of _unexpectedsuccess if hasattr(item, '_unexpectedsuccess') and rep.when == "call": - # we need to translate into how pytest encodes xpass - rep.wasxfail = "reason: " + repr(item._unexpectedsuccess) - # TODO: Do we need to check for strict xfail here as well? - rep.outcome = "passed" + # unittest treats an 'unexpected successes' as a failure + # which means pytest needs to handle it like a 'xfail(strict=True)' + rep.outcome = "failed" + if item._unexpectedsuccess: + rep.longrepr = "Unexpected success: {0}".format(item._unexpectedsuccess) + else: + rep.longrepr = "Unexpected success" elif item.config.option.runxfail: pass # don't interefere elif call.excinfo and call.excinfo.errisinstance(pytest.xfail.Exception): From 14a4dd0697db7be6043d0715c5b44e9d004dd73c Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Fri, 12 Aug 2016 23:31:38 +0100 Subject: [PATCH 06/21] Extend test to verify longrepr in stdout --- testing/test_unittest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/test_unittest.py b/testing/test_unittest.py index 39a31cfea..1390ed95c 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -616,6 +616,7 @@ def test_unittest_expected_failure_for_passing_test_is_fail(testdir): result.stdout.fnmatch_lines([ "*FAILURES*", "*MyTestCase*test_passing_test_is_fail*", + "*Unexpected success*", "*1 failed*", ]) assert result.ret == 1 From 225341cf2ca4733a739ec7ba2fe63cb834e0bb34 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Sat, 13 Aug 2016 00:00:51 +0100 Subject: [PATCH 07/21] Set wasxfail only for xpass w/o strict and else set longrepr --- _pytest/skipping.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_pytest/skipping.py b/_pytest/skipping.py index bd27870d6..651aa1a19 100644 --- a/_pytest/skipping.py +++ b/_pytest/skipping.py @@ -251,11 +251,13 @@ def pytest_runtest_makereport(item, call): elif call.when == "call": strict_default = item.config.getini('xfail_strict') is_strict_xfail = evalxfail.get('strict', strict_default) + explanation = evalxfail.getexplanation() if is_strict_xfail: rep.outcome = "failed" + rep.longrepr = "[XPASS(strict)] {0}".format(explanation) else: rep.outcome = "passed" - rep.wasxfail = evalxfail.getexplanation() + rep.wasxfail = explanation elif evalskip is not None and rep.skipped and type(rep.longrepr) is tuple: # skipped by mark.skipif; change the location of the failure # to point to the item definition, otherwise it will display From 16cb5d01b133c0b00526fce94f8eb6c405fe3c21 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 12 Aug 2016 00:29:03 +0200 Subject: [PATCH 08/21] Fix off-by-one error with lines from request.node.warn The line numbers in `node.location` seem to be zero-based?! --- CHANGELOG.rst | 3 ++- _pytest/main.py | 2 +- testing/test_config.py | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ed7adaf2e..6edb8c29f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -72,7 +72,8 @@ * -* +* Fixed off-by-one error with lines from ``request.node.warn``. + Thanks to `@blueyed`_ for the PR. * diff --git a/_pytest/main.py b/_pytest/main.py index 31b52c503..2ce9957af 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -267,7 +267,7 @@ class Node(object): if fslocation is None: fslocation = getattr(self, "fspath", None) else: - fslocation = "%s:%s" % fslocation[:2] + fslocation = "%s:%s" % (fslocation[0], fslocation[1] + 1) self.ihook.pytest_logwarning.call_historic(kwargs=dict( code=code, message=message, diff --git a/testing/test_config.py b/testing/test_config.py index f1c762554..1997ddacd 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -525,13 +525,14 @@ class TestWarning: reprec = testdir.inline_run() reprec.assertoutcome(passed=1) - def test_warn_on_test_item_from_request(self, testdir): + def test_warn_on_test_item_from_request(self, testdir, request): testdir.makepyfile(""" import pytest @pytest.fixture def fix(request): request.node.warn("T1", "hello") + def test_hello(fix): pass """) @@ -542,7 +543,7 @@ class TestWarning: result = testdir.runpytest("-rw") result.stdout.fnmatch_lines(""" ===*pytest-warning summary*=== - *WT1*test_warn_on_test_item*:5*hello* + *WT1*test_warn_on_test_item*:7 hello* """) class TestRootdir: From c163cc7937024309897c1b4d6526210657708c2b Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 12 Aug 2016 00:57:29 +0200 Subject: [PATCH 09/21] Improve display of continuation lines with multiline errors Fixes https://github.com/pytest-dev/pytest/issues/717. Follow-up to https://github.com/pytest-dev/pytest/pull/1762. --- CHANGELOG.rst | 7 ++++--- _pytest/python.py | 18 ++++++++++-------- testing/python/fixture.py | 9 +++++---- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ed7adaf2e..89de60ffb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,9 +3,10 @@ **Bug Fixes** -* Add an 'E' to the first line of error messages from FixtureLookupErrorRepr. - Fixes `#717`_. Thanks `@blueyed`_ for reporting, `@eolo999`_ for the PR - and `@tomviner`_ for his guidance during EuroPython2016 sprint. +* Improve error message with fixture lookup errors: add an 'E' to the first + line and '>' to the rest. Fixes `#717`_. Thanks `@blueyed`_ for reporting and + a PR, `@eolo999`_ for the initial PR and `@tomviner`_ for his guidance during + EuroPython2016 sprint. * Text documents without any doctests no longer appear as "skipped". Thanks `@graingert`_ for reporting and providing a full PR (`#1580`_). diff --git a/_pytest/python.py b/_pytest/python.py index dea7b58ce..2a23f477f 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -9,7 +9,7 @@ import warnings import py import pytest -from _pytest._code.code import TerminalRepr +from _pytest._code.code import FormattedExcinfo, TerminalRepr from _pytest.mark import MarkDecorator, MarkerError try: @@ -1836,6 +1836,7 @@ class FixtureLookupError(LookupError): return FixtureLookupErrorRepr(fspath, lineno, tblines, msg, self.argname) + class FixtureLookupErrorRepr(TerminalRepr): def __init__(self, filename, firstlineno, tblines, errorstring, argname): self.tblines = tblines @@ -1845,19 +1846,20 @@ class FixtureLookupErrorRepr(TerminalRepr): self.argname = argname def toterminal(self, tw): - #tw.line("FixtureLookupError: %s" %(self.argname), red=True) + # tw.line("FixtureLookupError: %s" %(self.argname), red=True) for tbline in self.tblines: tw.line(tbline.rstrip()) lines = self.errorstring.split("\n") - for line in lines: - if line == lines[0]: - prefix = 'E ' - else: - prefix = ' ' - tw.line(prefix + line.strip(), red=True) + if lines: + tw.line('{0} {1}'.format(FormattedExcinfo.fail_marker, + lines[0].strip()), red=True) + for line in lines[1:]: + tw.line('{0} {1}'.format(FormattedExcinfo.flow_marker, + line.strip()), red=True) tw.line() tw.line("%s:%d" % (self.filename, self.firstlineno+1)) + class FixtureManager: """ pytest fixtures definitions and information is stored and managed diff --git a/testing/python/fixture.py b/testing/python/fixture.py index 8b0a203bd..7264934a4 100644 --- a/testing/python/fixture.py +++ b/testing/python/fixture.py @@ -395,10 +395,11 @@ class TestFillFixtures: """) result = testdir.runpytest() result.stdout.fnmatch_lines([ - "*ERROR*test_lookup_error*", - "*def test_lookup_error(unknown):*", - "*fixture*unknown*not found*", - "*available fixtures*", + "*ERROR at setup of test_lookup_error*", + " def test_lookup_error(unknown):*", + "E fixture 'unknown' not found", + "> available fixtures:*", + "> use 'py*test --fixtures *' for help on them.", "*1 error*", ]) assert "INTERNAL" not in result.stdout.str() From 497152505ea6fd3362dd8ca95d187ddf8fdb6f8a Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 15 Aug 2016 19:31:00 -0300 Subject: [PATCH 10/21] Add CHANGELOG entry for #1809 --- CHANGELOG.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6b08341a8..e8aa1982a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -69,7 +69,8 @@ * Fixed the total tests tally in junit xml output (`#1798`_). Thanks to `@cryporchild`_ for the PR. -* +* ``pytest_terminal_summary`` hook now receives the ``exitstatus`` + of the test session as argument. Thanks `@blueyed`_ for the PR (`#1809`_). * @@ -91,6 +92,7 @@ .. _#1605: https://github.com/pytest-dev/pytest/issues/1605 .. _#1626: https://github.com/pytest-dev/pytest/pull/1626 .. _#1798: https://github.com/pytest-dev/pytest/pull/1798 +.. _#1809: https://github.com/pytest-dev/pytest/pull/1809 .. _#460: https://github.com/pytest-dev/pytest/pull/460 .. _#634: https://github.com/pytest-dev/pytest/issues/634 .. _#717: https://github.com/pytest-dev/pytest/issues/717 From 55ec1d7f56d56902e1a3fb4cbd1e9796f9420fe6 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Mon, 15 Aug 2016 23:58:16 +0100 Subject: [PATCH 11/21] Update test_junitxml.py to interpret XPASS as passed --- testing/test_junitxml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index a4f10dec5..1d2a5853b 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -100,7 +100,7 @@ class TestPython: result, dom = runandparse(testdir) assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(name="pytest", errors=0, failures=1, skips=3, tests=5) + node.assert_attr(name="pytest", errors=0, failures=1, skips=2, tests=5) def test_timing_function(self, testdir): testdir.makepyfile(""" From ea379e0e4f07182ebb10b1a4002f29f8ab36e5c2 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Wed, 17 Aug 2016 22:02:54 +0100 Subject: [PATCH 12/21] Fix test in test_junitxml and add one for strict --- testing/test_junitxml.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index 1d2a5853b..d5d6aa695 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -325,16 +325,33 @@ class TestPython: result, dom = runandparse(testdir) # assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(skips=1, tests=1) + node.assert_attr(skips=0, tests=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr( file="test_xfailure_xpass.py", line="1", classname="test_xfailure_xpass", name="test_xpass") - fnode = tnode.find_first_by_tag("skipped") - fnode.assert_attr(message="xfail-marked test passes unexpectedly") - # assert "ValueError" in fnode.toxml() + + def test_xfailure_xpass_strict(self, testdir): + testdir.makepyfile(""" + import pytest + @pytest.mark.xfail(strict=True, reason="This needs to fail!") + def test_xpass(): + pass + """) + result, dom = runandparse(testdir) + # assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(skips=0, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr( + file="test_xfailure_xpass_strict.py", + line="1", + classname="test_xfailure_xpass_strict", + name="test_xpass") + fnode = tnode.find_first_by_tag("failure") + fnode.assert_attr(message="[XPASS(strict)] This needs to fail!") def test_collect_error(self, testdir): testdir.makepyfile("syntax error") From 018197d72aa903c29cb874e155dbdff0a2ce5803 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Wed, 17 Aug 2016 22:14:51 +0100 Subject: [PATCH 13/21] Fix broken test in test_skipping and add one for strict xfail --- testing/test_skipping.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/testing/test_skipping.py b/testing/test_skipping.py index 2bfb6a8dc..4339da708 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -145,7 +145,20 @@ class TestXFail: def test_xfail_xpassed(self, testdir): item = testdir.getitem(""" import pytest - @pytest.mark.xfail + @pytest.mark.xfail(reason="nope") + def test_func(): + assert 1 + """) + reports = runtestprotocol(item, log=False) + assert len(reports) == 3 + callreport = reports[1] + assert callreport.passed + assert callreport.wasxfail == "nope" + + def test_xfail_xpassed_strict(self, testdir): + item = testdir.getitem(""" + import pytest + @pytest.mark.xfail(strict=True, reason="nope") def test_func(): assert 1 """) @@ -153,7 +166,8 @@ class TestXFail: assert len(reports) == 3 callreport = reports[1] assert callreport.failed - assert callreport.wasxfail == "" + assert callreport.longrepr == "[XPASS(strict)] nope" + assert not hasattr(callreport, "wasxfail") def test_xfail_run_anyway(self, testdir): testdir.makepyfile(""" From d1f2f779ee701838a2182d57e31535dc3c37d000 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Wed, 17 Aug 2016 22:31:56 +0100 Subject: [PATCH 14/21] Use a better xfail reason --- testing/test_skipping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/test_skipping.py b/testing/test_skipping.py index 4339da708..3b4bc7bd2 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -145,7 +145,7 @@ class TestXFail: def test_xfail_xpassed(self, testdir): item = testdir.getitem(""" import pytest - @pytest.mark.xfail(reason="nope") + @pytest.mark.xfail(reason="this is an xfail") def test_func(): assert 1 """) @@ -153,7 +153,7 @@ class TestXFail: assert len(reports) == 3 callreport = reports[1] assert callreport.passed - assert callreport.wasxfail == "nope" + assert callreport.wasxfail == "this is an xfail" def test_xfail_xpassed_strict(self, testdir): item = testdir.getitem(""" From 767c28d42257a3ddedd70dcfe429d9333eb5f9cb Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Wed, 17 Aug 2016 22:32:27 +0100 Subject: [PATCH 15/21] Fix broken test in test_junitxml --- testing/test_junitxml.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index f4bf2e631..899cc5880 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -115,13 +115,16 @@ class TestPython: def test_error(fixture): pass @pytest.mark.xfail + def test_xfail(): + assert False + @pytest.mark.xfail(strict=True) def test_xpass(): - assert 1 + assert True """) result, dom = runandparse(testdir) assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(name="pytest", errors=1, failures=1, skips=1, tests=4) + node.assert_attr(name="pytest", errors=1, failures=2, skips=1, tests=5) def test_timing_function(self, testdir): testdir.makepyfile(""" From 01739529611c433f7f3b91fb0734e428b84335b7 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Wed, 17 Aug 2016 23:15:14 +0100 Subject: [PATCH 16/21] Fix py3 xfail expression evaluation and parametrize strict --- testing/python/metafunc.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index d6e45384d..09d54e478 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -1080,22 +1080,23 @@ class TestMarkersWithParametrization: reprec = testdir.inline_run() reprec.assertoutcome(passed=2, skipped=1) - def test_xfail_passing_is_xpass(self, testdir): + @pytest.mark.parametrize('strict', [True, False]) + def test_xfail_passing_is_xpass(self, testdir, strict): s = """ import pytest @pytest.mark.parametrize(("n", "expected"), [ (1, 2), - pytest.mark.xfail("sys.version > 0", reason="some bug")((2, 3)), + pytest.mark.xfail("sys.version_info.major > 0", reason="some bug", strict={strict})((2, 3)), (3, 4), ]) def test_increment(n, expected): assert n + 1 == expected - """ + """.format(strict=strict) testdir.makepyfile(s) reprec = testdir.inline_run() - # xpass is fail, obviously :) - reprec.assertoutcome(passed=2, failed=1) + passed, failed = (2, 1) if strict else (3, 0) + reprec.assertoutcome(passed=passed, failed=failed) def test_parametrize_called_in_generate_tests(self, testdir): s = """ From dfc659f7810284a8607d02a1e0052ff4af002300 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Wed, 17 Aug 2016 23:32:56 +0100 Subject: [PATCH 17/21] Fix sys.version_info expression in xfail marker --- testing/python/metafunc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 09d54e478..249983ff5 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -1087,7 +1087,7 @@ class TestMarkersWithParametrization: @pytest.mark.parametrize(("n", "expected"), [ (1, 2), - pytest.mark.xfail("sys.version_info.major > 0", reason="some bug", strict={strict})((2, 3)), + pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})((2, 3)), (3, 4), ]) def test_increment(n, expected): From 92498109e48f2f174f9fca1e6511e688d65b1ad2 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 17 Aug 2016 21:20:12 -0300 Subject: [PATCH 18/21] Enable py35-trial testenv on Windows --- tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index b16a06887..31e1c0b41 100644 --- a/tox.ini +++ b/tox.ini @@ -78,13 +78,12 @@ commands= [testenv:py27-trial] deps=twisted commands= - py.test -rsxf {posargs:testing/test_unittest.py} + py.test -ra {posargs:testing/test_unittest.py} [testenv:py35-trial] -platform=linux|darwin deps={[testenv:py27-trial]deps} commands= - py.test -rsxf {posargs:testing/test_unittest.py} + py.test -ra {posargs:testing/test_unittest.py} [testenv:doctest] commands=py.test --doctest-modules _pytest From 4ed412eb5930455e0b14b012e7cd171f0c32752b Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 17 Aug 2016 20:29:26 -0300 Subject: [PATCH 19/21] unittest's unexpectedSuccess should work as non-strict xpass Make sure tests for that behavior obtain the same return code using either pytest or unittest to run the same file --- _pytest/skipping.py | 20 +++++++++++-- testing/test_unittest.py | 63 ++++++++++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/_pytest/skipping.py b/_pytest/skipping.py index 651aa1a19..7f4d927d9 100644 --- a/_pytest/skipping.py +++ b/_pytest/skipping.py @@ -220,6 +220,18 @@ def check_strict_xfail(pyfuncitem): pytest.fail('[XPASS(strict)] ' + explanation, pytrace=False) +def _is_unittest_unexpected_success_a_failure(): + """Return if the test suite should fail if a @expectedFailure unittest test PASSES. + + From https://docs.python.org/3/library/unittest.html?highlight=unittest#unittest.TestResult.wasSuccessful: + Changed in version 3.4: Returns False if there were any + unexpectedSuccesses from tests marked with the expectedFailure() decorator. + + TODO: this should be moved to the "compat" module. + """ + return sys.version_info >= (3, 4) + + @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): outcome = yield @@ -228,13 +240,15 @@ def pytest_runtest_makereport(item, call): evalskip = getattr(item, '_evalskip', None) # unitttest special case, see setting of _unexpectedsuccess if hasattr(item, '_unexpectedsuccess') and rep.when == "call": - # unittest treats an 'unexpected successes' as a failure - # which means pytest needs to handle it like a 'xfail(strict=True)' - rep.outcome = "failed" if item._unexpectedsuccess: rep.longrepr = "Unexpected success: {0}".format(item._unexpectedsuccess) else: rep.longrepr = "Unexpected success" + if _is_unittest_unexpected_success_a_failure(): + rep.outcome = "failed" + else: + rep.outcome = "passed" + rep.wasxfail = rep.longrepr elif item.config.option.runxfail: pass # don't interefere elif call.excinfo and call.excinfo.errisinstance(pytest.xfail.Exception): diff --git a/testing/test_unittest.py b/testing/test_unittest.py index 1390ed95c..9c35e4e3a 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -419,8 +419,9 @@ class TestTrialUnittest: def test_method(self): pass """) + from _pytest.skipping import _is_unittest_unexpected_success_a_failure + should_fail = _is_unittest_unexpected_success_a_failure() result = testdir.runpytest("-rxs") - assert result.ret == 0 result.stdout.fnmatch_lines_random([ "*XFAIL*test_trial_todo*", "*trialselfskip*", @@ -429,8 +430,9 @@ class TestTrialUnittest: "*i2wanto*", "*sys.version_info*", "*skip_in_method*", - "*4 skipped*3 xfail*1 xpass*", + "*1 failed*4 skipped*3 xfailed*" if should_fail else "*4 skipped*3 xfail*1 xpass*", ]) + assert result.ret == (1 if should_fail else 0) def test_trial_error(self, testdir): testdir.makepyfile(""" @@ -587,39 +589,62 @@ def test_unittest_typerror_traceback(testdir): assert "TypeError" in result.stdout.str() assert result.ret == 1 + @pytest.mark.skipif("sys.version_info < (2,7)") -def test_unittest_expected_failure_for_failing_test_is_xfail(testdir): - testdir.makepyfile(""" +@pytest.mark.parametrize('runner', ['pytest', 'unittest']) +def test_unittest_expected_failure_for_failing_test_is_xfail(testdir, runner): + script = testdir.makepyfile(""" import unittest class MyTestCase(unittest.TestCase): @unittest.expectedFailure def test_failing_test_is_xfail(self): assert False + if __name__ == '__main__': + unittest.main() """) - result = testdir.runpytest("-rxX") - result.stdout.fnmatch_lines([ - "*XFAIL*MyTestCase*test_failing_test_is_xfail*", - "*1 xfailed*", - ]) + if runner == 'pytest': + result = testdir.runpytest("-rxX") + result.stdout.fnmatch_lines([ + "*XFAIL*MyTestCase*test_failing_test_is_xfail*", + "*1 xfailed*", + ]) + else: + result = testdir.runpython(script) + result.stderr.fnmatch_lines([ + "*1 test in*", + "*OK*(expected failures=1)*", + ]) assert result.ret == 0 + @pytest.mark.skipif("sys.version_info < (2,7)") -def test_unittest_expected_failure_for_passing_test_is_fail(testdir): - testdir.makepyfile(""" +@pytest.mark.parametrize('runner', ['pytest', 'unittest']) +def test_unittest_expected_failure_for_passing_test_is_fail(testdir, runner): + script = testdir.makepyfile(""" import unittest class MyTestCase(unittest.TestCase): @unittest.expectedFailure def test_passing_test_is_fail(self): assert True + if __name__ == '__main__': + unittest.main() """) - result = testdir.runpytest("-rxX") - result.stdout.fnmatch_lines([ - "*FAILURES*", - "*MyTestCase*test_passing_test_is_fail*", - "*Unexpected success*", - "*1 failed*", - ]) - assert result.ret == 1 + from _pytest.skipping import _is_unittest_unexpected_success_a_failure + should_fail = _is_unittest_unexpected_success_a_failure() + if runner == 'pytest': + result = testdir.runpytest("-rxX") + result.stdout.fnmatch_lines([ + "*MyTestCase*test_passing_test_is_fail*", + "*1 failed*" if should_fail else "*1 xpassed*", + ]) + else: + result = testdir.runpython(script) + result.stderr.fnmatch_lines([ + "*1 test in*", + "*(unexpected successes=1)*", + ]) + + assert result.ret == (1 if should_fail else 0) @pytest.mark.parametrize('fix_type, stmt', [ From 224ef673740c519ebd1dcbc9f6688529c5ae0a9e Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 17 Aug 2016 20:32:25 -0300 Subject: [PATCH 20/21] Quick fix for tests in config depended on the current directory When running those tests from pytest's root folder, they would fail because they would end up picking pytest's own pytest.ini --- testing/test_config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/test_config.py b/testing/test_config.py index 1997ddacd..5a75f7d60 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -592,6 +592,7 @@ class TestRootdir: assert inicfg == {} def test_nothing(self, tmpdir): + tmpdir.chdir() rootdir, inifile, inicfg = determine_setup(None, [tmpdir]) assert rootdir == tmpdir assert inifile is None @@ -603,6 +604,7 @@ class TestRootdir: assert rootdir == tmpdir def test_with_arg_outside_cwd_without_inifile(self, tmpdir): + tmpdir.chdir() a = tmpdir.mkdir("a") b = tmpdir.mkdir("b") rootdir, inifile, inicfg = determine_setup(None, [a, b]) From 0fb34cd2a1bddf248c9e9c942e14c80e0991e2ea Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 17 Aug 2016 21:10:34 -0300 Subject: [PATCH 21/21] Update CHANGELOG entries --- CHANGELOG.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e8aa1982a..c26d071e5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -79,6 +79,20 @@ * +**Changes** + +* Change ``report.outcome`` for ``xpassed`` tests to ``"passed"`` in non-strict + mode and ``"failed"`` in strict mode. Thanks to `@hackebrot`_ for the PR + (`#1795`_) and `@gprasad84`_ for report (`#1546`_). + +* Tests marked with ``xfail(strict=False)`` (the default) now appear in + JUnitXML reports as passing tests instead of skipped. + Thanks to `@hackebrot`_ for the PR (`#1795`_). + +.. _#1795: https://github.com/pytest-dev/pytest/pull/1795 +.. _#1546: https://github.com/pytest-dev/pytest/issues/1546 +.. _@gprasad84: https://github.com/gprasad84 + .. _#1210: https://github.com/pytest-dev/pytest/issues/1210 .. _#1435: https://github.com/pytest-dev/pytest/issues/1435 .. _#1471: https://github.com/pytest-dev/pytest/issues/1471