From b660596f614f7a67dcf5f6f863345600840df18e Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Mon, 29 Apr 2024 20:50:34 +0300 Subject: [PATCH 1/3] testing: restore integration testing with pytest-bdd The problem was fixed. --- testing/plugins_integration/requirements.txt | 4 +--- tox.ini | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/testing/plugins_integration/requirements.txt b/testing/plugins_integration/requirements.txt index 9e152f119..d60bc5d3c 100644 --- a/testing/plugins_integration/requirements.txt +++ b/testing/plugins_integration/requirements.txt @@ -1,9 +1,7 @@ anyio[curio,trio]==4.3.0 django==5.0.4 pytest-asyncio==0.23.6 -# Temporarily not installed until pytest-bdd is fixed: -# https://github.com/pytest-dev/pytest/pull/11785 -# pytest-bdd==7.0.1 +pytest-bdd==7.1.2 pytest-cov==5.0.0 pytest-django==4.8.0 pytest-flakes==4.0.5 diff --git a/tox.ini b/tox.ini index cb3ca4b83..4e1ff1119 100644 --- a/tox.ini +++ b/tox.ini @@ -134,11 +134,9 @@ changedir = testing/plugins_integration deps = -rtesting/plugins_integration/requirements.txt setenv = PYTHONPATH=. -# Command temporarily removed until pytest-bdd is fixed: -# https://github.com/pytest-dev/pytest/pull/11785 -# pytest bdd_wallet.py commands = pip check + pytest bdd_wallet.py pytest --cov=. simple_integration.py pytest --ds=django_settings simple_integration.py pytest --html=simple.html simple_integration.py From 4c5298c3954efa71436364efaf531d3851a6aeb4 Mon Sep 17 00:00:00 2001 From: Ben Brown Date: Thu, 2 May 2024 03:42:31 -0400 Subject: [PATCH 2/3] Add back "Fix teardown error reporting when --maxfail=1 (#11721)" (#12279) Closes #11706. Originally fixed in #11721, but then reverted in #12022 due to a regression in pytest-xdist. The regression was fixed on the pytest-xdist side in pytest-dev/pytest-xdist#1026. --- changelog/11706.bugfix.rst | 4 +++ src/_pytest/runner.py | 4 +++ testing/test_runner.py | 50 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 changelog/11706.bugfix.rst diff --git a/changelog/11706.bugfix.rst b/changelog/11706.bugfix.rst new file mode 100644 index 000000000..a86db5ef6 --- /dev/null +++ b/changelog/11706.bugfix.rst @@ -0,0 +1,4 @@ +Fix reporting of teardown errors in higher-scoped fixtures when using `--maxfail` or `--stepwise`. + +Originally added in pytest 8.0.0, but reverted in 8.0.2 due to a regression in pytest-xdist. +This regression was fixed in pytest-xdist 3.6.1. diff --git a/src/_pytest/runner.py b/src/_pytest/runner.py index 5b7cd3e1d..dfefa73b7 100644 --- a/src/_pytest/runner.py +++ b/src/_pytest/runner.py @@ -134,6 +134,10 @@ def runtestprotocol( show_test_item(item) if not item.config.getoption("setuponly", False): reports.append(call_and_report(item, "call", log)) + # If the session is about to fail or stop, teardown everything - this is + # necessary to correctly report fixture teardown errors (see #11706) + if item.session.shouldfail or item.session.shouldstop: + nextitem = None reports.append(call_and_report(item, "teardown", log, nextitem=nextitem)) # After all teardown hooks have been called # want funcargs and request info to go away. diff --git a/testing/test_runner.py b/testing/test_runner.py index 6bd4a045d..436ce2f10 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -1216,3 +1216,53 @@ def test_pytest_version_env_var(pytester: Pytester, monkeypatch: MonkeyPatch) -> result = pytester.runpytest_inprocess() assert result.ret == ExitCode.OK assert os.environ["PYTEST_VERSION"] == "old version" + + +def test_teardown_session_failed(pytester: Pytester) -> None: + """Test that higher-scoped fixture teardowns run in the context of the last + item after the test session bails early due to --maxfail. + + Regression test for #11706. + """ + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module") + def baz(): + yield + pytest.fail("This is a failing teardown") + + def test_foo(baz): + pytest.fail("This is a failing test") + + def test_bar(): pass + """ + ) + result = pytester.runpytest("--maxfail=1") + result.assert_outcomes(failed=1, errors=1) + + +def test_teardown_session_stopped(pytester: Pytester) -> None: + """Test that higher-scoped fixture teardowns run in the context of the last + item after the test session bails early due to --stepwise. + + Regression test for #11706. + """ + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module") + def baz(): + yield + pytest.fail("This is a failing teardown") + + def test_foo(baz): + pytest.fail("This is a failing test") + + def test_bar(): pass + """ + ) + result = pytester.runpytest("--stepwise") + result.assert_outcomes(failed=1, errors=1) From 97610067acf220b13cc4c6fca34e5aac4834b04d Mon Sep 17 00:00:00 2001 From: Anita Hammer <166057949+anitahammer@users.noreply.github.com> Date: Thu, 2 May 2024 12:59:09 +0100 Subject: [PATCH 3/3] Handle KeyboardInterrupt and SystemExit at collection time (#12191) --- AUTHORS | 1 + changelog/12191.bugfix.rst | 1 + src/_pytest/runner.py | 4 +++- testing/test_collection.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 changelog/12191.bugfix.rst diff --git a/AUTHORS b/AUTHORS index 4f61c0591..4619cf1bc 100644 --- a/AUTHORS +++ b/AUTHORS @@ -36,6 +36,7 @@ Andrey Paramonov Andrzej Klajnert Andrzej Ostrowski Andy Freeland +Anita Hammer Anthon van der Neut Anthony Shaw Anthony Sottile diff --git a/changelog/12191.bugfix.rst b/changelog/12191.bugfix.rst new file mode 100644 index 000000000..5102d4698 --- /dev/null +++ b/changelog/12191.bugfix.rst @@ -0,0 +1 @@ +Keyboard interrupts and system exits are now properly handled during the test collection. diff --git a/src/_pytest/runner.py b/src/_pytest/runner.py index dfefa73b7..e5af60e38 100644 --- a/src/_pytest/runner.py +++ b/src/_pytest/runner.py @@ -393,7 +393,9 @@ def pytest_make_collect_report(collector: Collector) -> CollectReport: return list(collector.collect()) - call = CallInfo.from_call(collect, "collect") + call = CallInfo.from_call( + collect, "collect", reraise=(KeyboardInterrupt, SystemExit) + ) longrepr: Union[None, Tuple[str, int, str], str, TerminalRepr] = None if not call.excinfo: outcome: Literal["passed", "skipped", "failed"] = "passed" diff --git a/testing/test_collection.py b/testing/test_collection.py index 9caca622c..995e2999b 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -7,6 +7,7 @@ import sys import tempfile import textwrap from typing import List +from typing import Type from _pytest.assertion.util import running_on_ci from _pytest.config import ExitCode @@ -1856,3 +1857,33 @@ def test_do_not_collect_symlink_siblings( # Ensure we collect it only once if we pass the symlinked directory. result = pytester.runpytest(symlink_path, "-sv") result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize( + "exception_class, msg", + [ + (KeyboardInterrupt, "*!!! KeyboardInterrupt !!!*"), + (SystemExit, "INTERNALERROR> SystemExit"), + ], +) +def test_respect_system_exceptions( + pytester: Pytester, + exception_class: Type[BaseException], + msg: str, +): + head = "Before exception" + tail = "After exception" + ensure_file(pytester.path / "test_eggs.py").write_text( + f"print('{head}')", encoding="UTF-8" + ) + ensure_file(pytester.path / "test_ham.py").write_text( + f"raise {exception_class.__name__}()", encoding="UTF-8" + ) + ensure_file(pytester.path / "test_spam.py").write_text( + f"print('{tail}')", encoding="UTF-8" + ) + + result = pytester.runpytest_subprocess("-s") + result.stdout.fnmatch_lines([f"*{head}*"]) + result.stdout.fnmatch_lines([msg]) + result.stdout.no_fnmatch_line(f"*{tail}*")