From a02f8c39d99fcabb8138597d99901f414c28330d Mon Sep 17 00:00:00 2001 From: Brian Okken <1568356+okken@users.noreply.github.com> Date: Sat, 27 Apr 2024 18:53:50 -0700 Subject: [PATCH 1/7] add --xfail-tb to be able to turn off or separately control xfail tb style --- src/_pytest/terminal.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index 724d5c54d..9e4d51bd2 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -216,6 +216,15 @@ def pytest_addoption(parser: Parser) -> None: choices=["auto", "long", "short", "no", "line", "native"], help="Traceback print mode (auto/long/short/line/native/no)", ) + group._addoption( + "--xfail-tb", + metavar="style", + action="store", + dest="xfail_tbstyle", + default="auto", + choices=["auto", "long", "short", "no", "line", "native"], + help="Traceback print mode for xfail(auto/long/short/line/native/no)", + ) group._addoption( "--show-capture", action="store", @@ -1074,18 +1083,22 @@ class TerminalReporter: self.summary_failures_combined("failed", "FAILURES") def summary_xfailures(self) -> None: - self.summary_failures_combined("xfailed", "XFAILURES", "x") + style = self.config.option.xfail_tbstyle + self.summary_failures_combined("xfailed", "XFAILURES", "x", style=style) def summary_failures_combined( - self, which_reports: str, sep_title: str, needed_opt: Optional[str] = None + self, which_reports: str, sep_title: str, needed_opt: Optional[str] = None, style = None ) -> None: - if self.config.option.tbstyle != "no": + if style is None: + style == self.config.option.tbstyle + + if style != "no": if not needed_opt or self.hasopt(needed_opt): reports: List[BaseReport] = self.getreports(which_reports) if not reports: return self.write_sep("=", sep_title) - if self.config.option.tbstyle == "line": + if style == "line": for rep in reports: line = self._getcrashline(rep) self.write_line(line) From d29558d40383321aad33be2388f08584551d9d9c Mon Sep 17 00:00:00 2001 From: Brian Okken <1568356+okken@users.noreply.github.com> Date: Sat, 27 Apr 2024 19:06:03 -0700 Subject: [PATCH 2/7] implement 12231, --xfail-tb --- src/_pytest/terminal.py | 3 +-- testing/test_terminal.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index 9e4d51bd2..113ff96c1 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -1090,8 +1090,7 @@ class TerminalReporter: self, which_reports: str, sep_title: str, needed_opt: Optional[str] = None, style = None ) -> None: if style is None: - style == self.config.option.tbstyle - + style = self.config.option.tbstyle if style != "no": if not needed_opt or self.hasopt(needed_opt): reports: List[BaseReport] = self.getreports(which_reports) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 170f1efcf..60c49c644 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2949,7 +2949,7 @@ def test_xfail_tb_line(pytester: Pytester) -> None: assert a == b """ ) - result = pytester.runpytest("-rx", "--tb=line") + result = pytester.runpytest("-rx", "--xfail-tb=line") result.stdout.fnmatch_lines( [ "*= XFAILURES =*", From 7320171f707fdeff29e8d27e8266bb30d12f071d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 28 Apr 2024 02:13:27 +0000 Subject: [PATCH 3/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/_pytest/terminal.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index 113ff96c1..441aa1c2b 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -1087,7 +1087,11 @@ class TerminalReporter: self.summary_failures_combined("xfailed", "XFAILURES", "x", style=style) def summary_failures_combined( - self, which_reports: str, sep_title: str, needed_opt: Optional[str] = None, style = None + self, + which_reports: str, + sep_title: str, + needed_opt: Optional[str] = None, + style=None, ) -> None: if style is None: style = self.config.option.tbstyle From a9e8ead563bbaeb27952f0ff3d156c0d5a8cf373 Mon Sep 17 00:00:00 2001 From: Brian Okken <1568356+okken@users.noreply.github.com> Date: Sun, 28 Apr 2024 12:56:28 -0700 Subject: [PATCH 4/7] Change default, add more tests, add changelog entry --- changelog/12231.feature.rst | 16 +++++++++++++++ src/_pytest/terminal.py | 6 +++--- testing/test_terminal.py | 41 +++++++++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 changelog/12231.feature.rst diff --git a/changelog/12231.feature.rst b/changelog/12231.feature.rst new file mode 100644 index 000000000..6369c92f7 --- /dev/null +++ b/changelog/12231.feature.rst @@ -0,0 +1,16 @@ +Add `--xfail-tb` flag, which acts a lot like `--tb`, but controls the traceback output for XFAIL results. + +* The `--xfail-tb` flag takes one argument, `auto|long|short|line|native|no`, and defaults to `no`. +* To turn on xfail tracebacks, pass in `-rx` or `-ra` along with `--xfail-tb=short` (or any of `auto`, `long`, `short`, `line`, or `native`). + +The `--tb` flag, that controls normal test failure tracebacks, defaults to `auto`. +This change really only separates the traceback behavior of normal vs xfail failures. + +Some history: + +* This is a behavior break, but brings default behavior back to pre-8.0.0 behavior. +* With this change, default `-rx`/ `-ra` behavior is identical to pre-8.0 with respect to xfail tracebacks. +* With pytest 8.0, `-rx` or `-ra` would not only turn on summary reports for xfail, but also report the tracebacks for xfail results. +* This caused issues with some projects that utilize xfail, but don't want to see all of the xfail tracebacks. + + diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index 441aa1c2b..bd12e56cf 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -221,9 +221,9 @@ def pytest_addoption(parser: Parser) -> None: metavar="style", action="store", dest="xfail_tbstyle", - default="auto", + default="no", choices=["auto", "long", "short", "no", "line", "native"], - help="Traceback print mode for xfail(auto/long/short/line/native/no)", + help="Traceback print mode for xfail (auto/long/short/line/native/no)", ) group._addoption( "--show-capture", @@ -1091,7 +1091,7 @@ class TerminalReporter: which_reports: str, sep_title: str, needed_opt: Optional[str] = None, - style=None, + style: Optional[str] = None, ) -> None: if style is None: style = self.config.option.tbstyle diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 60c49c644..4957c0e02 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2909,7 +2909,7 @@ def test_summary_xfail_reason(pytester: Pytester) -> None: assert result.stdout.lines.count(expect2) == 1 -def test_summary_xfail_tb(pytester: Pytester) -> None: +def test_xfail_tb_default(pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @@ -2920,19 +2920,44 @@ def test_summary_xfail_tb(pytester: Pytester) -> None: assert a == b """ ) + # defaults to "no", so this is the same as `runpytest("-rx", "--xfail-tb=no")` result = pytester.runpytest("-rx") + # Don't show traceback + result.stdout.no_fnmatch_line("*= XFAILURES =*") + result.stdout.no_fnmatch_line("*test_xfail_tb_line.py:6: assert 1 == 2") + # still print summary + result.stdout.fnmatch_lines( + [ + "*= short test summary info =*", + "XFAIL test_xfail_tb_default.py::test_xfail", + "*= 1 xfailed in * =*", + ] + ) + + +def test_xfail_tb_short(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.xfail + def test_xfail(): + a, b = 1, 2 + assert a == b + """ + ) + result = pytester.runpytest("-rx", "--xfail-tb=short") result.stdout.fnmatch_lines( [ "*= XFAILURES =*", "*_ test_xfail _*", - "* @pytest.mark.xfail*", - "* def test_xfail():*", - "* a, b = 1, 2*", - "> *assert a == b*", - "E *assert 1 == 2*", - "test_summary_xfail_tb.py:6: AssertionError*", + "*@pytest.mark.xfail*", + "*def test_xfail():*", + "* a, b = 1, 2*", + "*> *assert a == b*", + "*E *assert 1 == 2*", "*= short test summary info =*", - "XFAIL test_summary_xfail_tb.py::test_xfail", + "XFAIL test_xfail_tb_short.py::test_xfail", "*= 1 xfailed in * =*", ] ) From f138f1d288033c12dec0c0b932e8c112489dc751 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 28 Apr 2024 19:56:47 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- changelog/12231.feature.rst | 10 ++++------ testing/test_terminal.py | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/changelog/12231.feature.rst b/changelog/12231.feature.rst index 6369c92f7..cddb1e2a8 100644 --- a/changelog/12231.feature.rst +++ b/changelog/12231.feature.rst @@ -1,16 +1,14 @@ -Add `--xfail-tb` flag, which acts a lot like `--tb`, but controls the traceback output for XFAIL results. +Add `--xfail-tb` flag, which acts a lot like `--tb`, but controls the traceback output for XFAIL results. -* The `--xfail-tb` flag takes one argument, `auto|long|short|line|native|no`, and defaults to `no`. +* The `--xfail-tb` flag takes one argument, `auto|long|short|line|native|no`, and defaults to `no`. * To turn on xfail tracebacks, pass in `-rx` or `-ra` along with `--xfail-tb=short` (or any of `auto`, `long`, `short`, `line`, or `native`). -The `--tb` flag, that controls normal test failure tracebacks, defaults to `auto`. +The `--tb` flag, that controls normal test failure tracebacks, defaults to `auto`. This change really only separates the traceback behavior of normal vs xfail failures. -Some history: +Some history: * This is a behavior break, but brings default behavior back to pre-8.0.0 behavior. * With this change, default `-rx`/ `-ra` behavior is identical to pre-8.0 with respect to xfail tracebacks. * With pytest 8.0, `-rx` or `-ra` would not only turn on summary reports for xfail, but also report the tracebacks for xfail results. * This caused issues with some projects that utilize xfail, but don't want to see all of the xfail tracebacks. - - diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 4957c0e02..f58605be9 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2933,7 +2933,7 @@ def test_xfail_tb_default(pytester: Pytester) -> None: "*= 1 xfailed in * =*", ] ) - + def test_xfail_tb_short(pytester: Pytester) -> None: pytester.makepyfile( From 1a20198784c263424b2a12a49b7844a142d6df96 Mon Sep 17 00:00:00 2001 From: Brian Okken <1568356+okken@users.noreply.github.com> Date: Mon, 29 Apr 2024 11:36:40 -0700 Subject: [PATCH 6/7] Change --xfail-tb to a boolean flag --- changelog/12231.feature.rst | 13 ++--- src/_pytest/terminal.py | 20 +++---- testing/test_terminal.py | 107 +++++++++++++++++------------------- 3 files changed, 65 insertions(+), 75 deletions(-) diff --git a/changelog/12231.feature.rst b/changelog/12231.feature.rst index 6369c92f7..594cf063a 100644 --- a/changelog/12231.feature.rst +++ b/changelog/12231.feature.rst @@ -1,10 +1,8 @@ -Add `--xfail-tb` flag, which acts a lot like `--tb`, but controls the traceback output for XFAIL results. +Add `--xfail-tb` flag, which turns traceback output for XFAIL results. -* The `--xfail-tb` flag takes one argument, `auto|long|short|line|native|no`, and defaults to `no`. -* To turn on xfail tracebacks, pass in `-rx` or `-ra` along with `--xfail-tb=short` (or any of `auto`, `long`, `short`, `line`, or `native`). - -The `--tb` flag, that controls normal test failure tracebacks, defaults to `auto`. -This change really only separates the traceback behavior of normal vs xfail failures. +* If the `--xfail-tb` flag is not sent, tracebacks for XFAIL results are NOT shown. +* The style of traceback for XFAIL is set with `--tb`, and can be `auto|long|short|line|native|no`. +* Note: Even if you have `--xfail-tb` set, you won't see them if `--tb=no`. Some history: @@ -12,5 +10,4 @@ Some history: * With this change, default `-rx`/ `-ra` behavior is identical to pre-8.0 with respect to xfail tracebacks. * With pytest 8.0, `-rx` or `-ra` would not only turn on summary reports for xfail, but also report the tracebacks for xfail results. * This caused issues with some projects that utilize xfail, but don't want to see all of the xfail tracebacks. - - +* This change detaches xfail tracebacks from `-rx`, and now we turn on xfail tracebacks with `--xfail-tb`. \ No newline at end of file diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index bd12e56cf..5dfed31c3 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -218,12 +218,10 @@ def pytest_addoption(parser: Parser) -> None: ) group._addoption( "--xfail-tb", - metavar="style", - action="store", - dest="xfail_tbstyle", - default="no", - choices=["auto", "long", "short", "no", "line", "native"], - help="Traceback print mode for xfail (auto/long/short/line/native/no)", + action="store_true", + dest="xfail_tb", + default=False, + help="Show tracebacks for xfail (as long as --tb != no)", ) group._addoption( "--show-capture", @@ -1080,11 +1078,13 @@ class TerminalReporter: self._tw.line(content) def summary_failures(self) -> None: - self.summary_failures_combined("failed", "FAILURES") + style = self.config.option.tbstyle + self.summary_failures_combined("failed", "FAILURES", style=style) def summary_xfailures(self) -> None: - style = self.config.option.xfail_tbstyle - self.summary_failures_combined("xfailed", "XFAILURES", "x", style=style) + show_tb = self.config.option.xfail_tb + style = self.config.option.tbstyle if show_tb else "no" + self.summary_failures_combined("xfailed", "XFAILURES", style=style) def summary_failures_combined( self, @@ -1093,8 +1093,6 @@ class TerminalReporter: needed_opt: Optional[str] = None, style: Optional[str] = None, ) -> None: - if style is None: - style = self.config.option.tbstyle if style != "no": if not needed_opt or self.hasopt(needed_opt): reports: List[BaseReport] = self.getreports(which_reports) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 4957c0e02..bfe853858 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2909,79 +2909,74 @@ def test_summary_xfail_reason(pytester: Pytester) -> None: assert result.stdout.lines.count(expect2) == 1 -def test_xfail_tb_default(pytester: Pytester) -> None: - pytester.makepyfile( +@pytest.fixture() +def xfail_testfile(pytester: Pytester) -> Path: + return pytester.makepyfile( """ import pytest - @pytest.mark.xfail - def test_xfail(): + def test_fail(): a, b = 1, 2 assert a == b + + @pytest.mark.xfail + def test_xfail(): + c, d = 3, 4 + assert c == d """ ) - # defaults to "no", so this is the same as `runpytest("-rx", "--xfail-tb=no")` - result = pytester.runpytest("-rx") - # Don't show traceback - result.stdout.no_fnmatch_line("*= XFAILURES =*") - result.stdout.no_fnmatch_line("*test_xfail_tb_line.py:6: assert 1 == 2") - # still print summary + +def test_xfail_tb_default(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile) + + # test_fail, show traceback result.stdout.fnmatch_lines( [ - "*= short test summary info =*", - "XFAIL test_xfail_tb_default.py::test_xfail", - "*= 1 xfailed in * =*", + "*= FAILURES =*", + "*_ test_fail _*", + "*def test_fail():*", + "* a, b = 1, 2*", + "*> assert a == b*", + "*E assert 1 == 2*", + ] + ) + + # test_xfail, don't show traceback + result.stdout.no_fnmatch_line("*= XFAILURES =*") + + +def test_xfail_tb_true(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile, "--xfail-tb") + + # both test_fail and test_xfail, show traceback + result.stdout.fnmatch_lines( + [ + "*= FAILURES =*", + "*_ test_fail _*", + "*def test_fail():*", + "* a, b = 1, 2*", + "*> assert a == b*", + "*E assert 1 == 2*", + "*= XFAILURES =*", + "*_ test_xfail _*", + "*def test_xfail():*", + "* c, d = 3, 4*", + "*> assert c == d*", + "*E assert 3 == 4*", ] ) -def test_xfail_tb_short(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest +def test_xfail_tb_line(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile, "--xfail-tb", "--tb=line") - @pytest.mark.xfail - def test_xfail(): - a, b = 1, 2 - assert a == b - """ - ) - result = pytester.runpytest("-rx", "--xfail-tb=short") + # both test_fail and test_xfail, show line result.stdout.fnmatch_lines( [ + "*= FAILURES =*", + "*test_xfail_tb_line.py:5: assert 1 == 2", "*= XFAILURES =*", - "*_ test_xfail _*", - "*@pytest.mark.xfail*", - "*def test_xfail():*", - "* a, b = 1, 2*", - "*> *assert a == b*", - "*E *assert 1 == 2*", - "*= short test summary info =*", - "XFAIL test_xfail_tb_short.py::test_xfail", - "*= 1 xfailed in * =*", - ] - ) - - -def test_xfail_tb_line(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - - @pytest.mark.xfail - def test_xfail(): - a, b = 1, 2 - assert a == b - """ - ) - result = pytester.runpytest("-rx", "--xfail-tb=line") - result.stdout.fnmatch_lines( - [ - "*= XFAILURES =*", - "*test_xfail_tb_line.py:6: assert 1 == 2", - "*= short test summary info =*", - "XFAIL test_xfail_tb_line.py::test_xfail", - "*= 1 xfailed in * =*", + "*test_xfail_tb_line.py:10: assert 3 == 4", ] ) From aff8b059a659d27a91e11aada07a0bc60237196b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 18:43:02 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- changelog/12231.feature.rst | 4 ++-- testing/test_terminal.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/changelog/12231.feature.rst b/changelog/12231.feature.rst index 056eebb6b..82b6e5c0a 100644 --- a/changelog/12231.feature.rst +++ b/changelog/12231.feature.rst @@ -1,4 +1,4 @@ -Add `--xfail-tb` flag, which turns traceback output for XFAIL results. +Add `--xfail-tb` flag, which turns traceback output for XFAIL results. * If the `--xfail-tb` flag is not sent, tracebacks for XFAIL results are NOT shown. * The style of traceback for XFAIL is set with `--tb`, and can be `auto|long|short|line|native|no`. @@ -10,4 +10,4 @@ Some history: * With this change, default `-rx`/ `-ra` behavior is identical to pre-8.0 with respect to xfail tracebacks. * With pytest 8.0, `-rx` or `-ra` would not only turn on summary reports for xfail, but also report the tracebacks for xfail results. * This caused issues with some projects that utilize xfail, but don't want to see all of the xfail tracebacks. -* This change detaches xfail tracebacks from `-rx`, and now we turn on xfail tracebacks with `--xfail-tb`. \ No newline at end of file +* This change detaches xfail tracebacks from `-rx`, and now we turn on xfail tracebacks with `--xfail-tb`. diff --git a/testing/test_terminal.py b/testing/test_terminal.py index cb0f8124e..a067bc407 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2918,7 +2918,7 @@ def xfail_testfile(pytester: Pytester) -> Path: def test_fail(): a, b = 1, 2 assert a == b - + @pytest.mark.xfail def test_xfail(): c, d = 3, 4 @@ -2926,9 +2926,10 @@ def xfail_testfile(pytester: Pytester) -> Path: """ ) + def test_xfail_tb_default(xfail_testfile, pytester: Pytester) -> None: result = pytester.runpytest(xfail_testfile) - + # test_fail, show traceback result.stdout.fnmatch_lines( [ @@ -2947,7 +2948,7 @@ def test_xfail_tb_default(xfail_testfile, pytester: Pytester) -> None: def test_xfail_tb_true(xfail_testfile, pytester: Pytester) -> None: result = pytester.runpytest(xfail_testfile, "--xfail-tb") - + # both test_fail and test_xfail, show traceback result.stdout.fnmatch_lines( [