From 8efb4bb9c1339b5ae6403d56e533a14a428102cf Mon Sep 17 00:00:00 2001 From: s-padmanaban <73488630+s-padmanaban@users.noreply.github.com> Date: Fri, 13 Jan 2023 02:14:52 -0800 Subject: [PATCH 1/3] Do not update cache from xdist worker (#10641) --- AUTHORS | 1 + changelog/10641.bugfix.rst | 1 + src/_pytest/stepwise.py | 8 ++++ testing/test_stepwise.py | 77 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 changelog/10641.bugfix.rst diff --git a/AUTHORS b/AUTHORS index a4ca99267..7cf1748e6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -315,6 +315,7 @@ Samuel Searles-Bryant Samuele Pedroni Sanket Duthade Sankt Petersbug +Saravanan Padmanaban Segev Finer Serhii Mozghovyi Seth Junot diff --git a/changelog/10641.bugfix.rst b/changelog/10641.bugfix.rst new file mode 100644 index 000000000..ac293f6ae --- /dev/null +++ b/changelog/10641.bugfix.rst @@ -0,0 +1 @@ +Fix a race condition when creating or updating the stepwise plugin's cache, which could occur when multiple xdist worker nodes try to simultaneously update the stepwise plugin's cache. diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index 84f1a6ce8..74ad9dbd4 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -48,6 +48,10 @@ def pytest_configure(config: Config) -> None: def pytest_sessionfinish(session: Session) -> None: if not session.config.getoption("stepwise"): assert session.config.cache is not None + if hasattr(session.config, "workerinput"): + # Do not update cache if this process is a xdist worker to prevent + # race conditions (#10641). + return # Clear the list of failing tests if the plugin is not active. session.config.cache.set(STEPWISE_CACHE_DIR, []) @@ -119,4 +123,8 @@ class StepwisePlugin: return None def pytest_sessionfinish(self) -> None: + if hasattr(self.config, "workerinput"): + # Do not update cache if this process is a xdist worker to prevent + # race conditions (#10641). + return self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed) diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index 20781d42c..2094abc4e 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -1,6 +1,10 @@ +from pathlib import Path + import pytest +from _pytest.cacheprovider import Cache from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester +from _pytest.stepwise import STEPWISE_CACHE_DIR @pytest.fixture @@ -278,3 +282,76 @@ def test_stepwise_skip_is_independent(pytester: Pytester) -> None: def test_sw_skip_help(pytester: Pytester) -> None: result = pytester.runpytest("-h") result.stdout.fnmatch_lines("*Implicitly enables --stepwise.") + + +def test_stepwise_xdist_dont_store_lastfailed(pytester: Pytester) -> None: + pytester.makefile( + ext=".ini", + pytest=f"[pytest]\ncache_dir = {pytester.path}\n", + ) + + pytester.makepyfile( + conftest=""" +import pytest + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config) -> None: + config.workerinput = True +""" + ) + pytester.makepyfile( + test_one=""" +def test_one(): + assert False +""" + ) + result = pytester.runpytest("--stepwise") + assert result.ret == pytest.ExitCode.INTERRUPTED + + stepwise_cache_file = ( + pytester.path / Cache._CACHE_PREFIX_VALUES / STEPWISE_CACHE_DIR + ) + assert not Path(stepwise_cache_file).exists() + + +def test_disabled_stepwise_xdist_dont_clear_cache(pytester: Pytester) -> None: + pytester.makefile( + ext=".ini", + pytest=f"[pytest]\ncache_dir = {pytester.path}\n", + ) + + stepwise_cache_file = ( + pytester.path / Cache._CACHE_PREFIX_VALUES / STEPWISE_CACHE_DIR + ) + stepwise_cache_dir = stepwise_cache_file.parent + stepwise_cache_dir.mkdir(exist_ok=True, parents=True) + + stepwise_cache_file_relative = f"{Cache._CACHE_PREFIX_VALUES}/{STEPWISE_CACHE_DIR}" + + expected_value = '"test_one.py::test_one"' + content = {f"{stepwise_cache_file_relative}": expected_value} + + pytester.makefile(ext="", **content) + + pytester.makepyfile( + conftest=""" +import pytest + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config) -> None: + config.workerinput = True +""" + ) + pytester.makepyfile( + test_one=""" +def test_one(): + assert True +""" + ) + result = pytester.runpytest() + assert result.ret == 0 + + assert Path(stepwise_cache_file).exists() + with stepwise_cache_file.open() as file_handle: + observed_value = file_handle.readlines() + assert [expected_value] == observed_value From 5e0583f4b9b9026185dc47865b7db60780437b0a Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 13 Jan 2023 07:15:41 -0300 Subject: [PATCH 2/3] Fix regen tox environment (#10640) Since tox 4.0, the whitelist_external option has been renamed to allowlist_externals. Co-authored-by: pytest bot --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f619287fb..663ada60c 100644 --- a/tox.ini +++ b/tox.ini @@ -104,7 +104,7 @@ deps = PyYAML regendoc>=0.8.1 sphinx -whitelist_externals = +allowlist_externals = make commands = make regen From 7421f3bb94df80ff2d131e932223b190f9b6d7b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Jan 2023 07:16:43 -0300 Subject: [PATCH 3/3] build(deps): Bump django in /testing/plugins_integration (#10643) Bumps [django](https://github.com/django/django) from 4.1.3 to 4.1.5. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/4.1.3...4.1.5) --- updated-dependencies: - dependency-name: django dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- testing/plugins_integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/plugins_integration/requirements.txt b/testing/plugins_integration/requirements.txt index 237e43ac0..7e7a5d633 100644 --- a/testing/plugins_integration/requirements.txt +++ b/testing/plugins_integration/requirements.txt @@ -1,5 +1,5 @@ anyio[curio,trio]==3.6.2 -django==4.1.3 +django==4.1.5 pytest-asyncio==0.20.2 pytest-bdd==6.1.1 pytest-cov==4.0.0