From 37e415dbc12fc32ca92433a0d741491a7262f897 Mon Sep 17 00:00:00 2001 From: Brian Okken <1568356+okken@users.noreply.github.com> Date: Wed, 1 May 2024 20:30:25 -0700 Subject: [PATCH] implement #12231 new flag --xfail-tb --- changelog/12231.feature.rst | 13 ++++++ src/_pytest/terminal.py | 24 ++++++++--- testing/test_terminal.py | 79 +++++++++++++++++++++++-------------- 3 files changed, 82 insertions(+), 34 deletions(-) create mode 100644 changelog/12231.feature.rst diff --git a/changelog/12231.feature.rst b/changelog/12231.feature.rst new file mode 100644 index 000000000..268c4fe18 --- /dev/null +++ b/changelog/12231.feature.rst @@ -0,0 +1,13 @@ +Add `--xfail-tb` flag, which turns traceback output for XFAIL results. + +* If the `--xfail-tb` flag is not sent, tracebacks for XFAIL results are NOT shown. +* The style of traceback for XFAIL is set with `--tb`, and can be `auto|long|short|line|native|no`. +* Note: Even if you have `--xfail-tb` set, you won't see them if `--tb=no`. + +Some history: + +* This is a behavior break, but brings default behavior back to pre-8.0.0 behavior. +* With this change, default `-rx`/ `-ra` behavior is identical to pre-8.0 with respect to xfail tracebacks. +* With pytest 8.0, `-rx` or `-ra` would not only turn on summary reports for xfail, but also report the tracebacks for xfail results. +* This caused issues with some projects that utilize xfail, but don't want to see all of the xfail tracebacks. +* This change detaches xfail tracebacks from `-rx`, and now we turn on xfail tracebacks with `--xfail-tb`. \ No newline at end of file diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index d05eb7e3d..1438a9d32 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -216,6 +216,13 @@ def pytest_addoption(parser: Parser) -> None: choices=["auto", "long", "short", "no", "line", "native"], help="Traceback print mode (auto/long/short/line/native/no)", ) + group._addoption( + "--xfail-tb", + action="store_true", + dest="xfail_tb", + default=False, + help="Show tracebacks for xfail (as long as --tb != no)", + ) group._addoption( "--show-capture", action="store", @@ -1086,21 +1093,28 @@ class TerminalReporter: self._tw.line(content) def summary_failures(self) -> None: - self.summary_failures_combined("failed", "FAILURES") + style = self.config.option.tbstyle + self.summary_failures_combined("failed", "FAILURES", style=style) def summary_xfailures(self) -> None: - self.summary_failures_combined("xfailed", "XFAILURES", "x") + show_tb = self.config.option.xfail_tb + style = self.config.option.tbstyle if show_tb else "no" + self.summary_failures_combined("xfailed", "XFAILURES", style=style) def summary_failures_combined( - self, which_reports: str, sep_title: str, needed_opt: Optional[str] = None + self, + which_reports: str, + sep_title: str, + needed_opt: Optional[str] = None, + style: Optional[str] = None, ) -> None: - if self.config.option.tbstyle != "no": + if style != "no": if not needed_opt or self.hasopt(needed_opt): reports: List[BaseReport] = self.getreports(which_reports) if not reports: return self.write_sep("=", sep_title) - if self.config.option.tbstyle == "line": + if style == "line": for rep in reports: line = self._getcrashline(rep) self.write_line(line) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index dd5fa4bf6..6d1804a60 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2909,54 +2909,75 @@ def test_summary_xfail_reason(pytester: Pytester) -> None: assert result.stdout.lines.count(expect2) == 1 -def test_summary_xfail_tb(pytester: Pytester) -> None: - pytester.makepyfile( +@pytest.fixture() +def xfail_testfile(pytester: Pytester) -> Path: + return pytester.makepyfile( """ import pytest - @pytest.mark.xfail - def test_xfail(): + def test_fail(): a, b = 1, 2 assert a == b + + @pytest.mark.xfail + def test_xfail(): + c, d = 3, 4 + assert c == d """ ) - result = pytester.runpytest("-rx") + + +def test_xfail_tb_default(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile) + + # test_fail, show traceback result.stdout.fnmatch_lines( [ + "*= FAILURES =*", + "*_ test_fail _*", + "*def test_fail():*", + "* a, b = 1, 2*", + "*> assert a == b*", + "*E assert 1 == 2*", + ] + ) + + # test_xfail, don't show traceback + result.stdout.no_fnmatch_line("*= XFAILURES =*") + + +def test_xfail_tb_true(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile, "--xfail-tb") + + # both test_fail and test_xfail, show traceback + result.stdout.fnmatch_lines( + [ + "*= FAILURES =*", + "*_ test_fail _*", + "*def test_fail():*", + "* a, b = 1, 2*", + "*> assert a == b*", + "*E assert 1 == 2*", "*= XFAILURES =*", "*_ test_xfail _*", - "* @pytest.mark.xfail*", - "* def test_xfail():*", - "* a, b = 1, 2*", - "> *assert a == b*", - "E *assert 1 == 2*", - "test_summary_xfail_tb.py:6: AssertionError*", - "*= short test summary info =*", - "XFAIL test_summary_xfail_tb.py::test_xfail", - "*= 1 xfailed in * =*", + "*def test_xfail():*", + "* c, d = 3, 4*", + "*> assert c == d*", + "*E assert 3 == 4*", ] ) -def test_xfail_tb_line(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest +def test_xfail_tb_line(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile, "--xfail-tb", "--tb=line") - @pytest.mark.xfail - def test_xfail(): - a, b = 1, 2 - assert a == b - """ - ) - result = pytester.runpytest("-rx", "--tb=line") + # both test_fail and test_xfail, show line result.stdout.fnmatch_lines( [ + "*= FAILURES =*", + "*test_xfail_tb_line.py:5: assert 1 == 2", "*= XFAILURES =*", - "*test_xfail_tb_line.py:6: assert 1 == 2", - "*= short test summary info =*", - "XFAIL test_xfail_tb_line.py::test_xfail", - "*= 1 xfailed in * =*", + "*test_xfail_tb_line.py:10: assert 3 == 4", ] )