make_xpass_failure_again_#11498

changelog/11498.improvement.rst

@@ -0,0 +1,4 @@
+In order to ensure backward compatibility we cannot go straight from non-strict to strict instead
+we have to start by warning if strict was not set to true or false. The warning should indicate that a
+future major release of pytest will change the default from False to True and recommend to use
+strict=True as default and a plugin for actually flaky tests.

src/_pytest/skipping.py

@@ -4,6 +4,7 @@ import os
 import platform
 import sys
 import traceback
+import warnings
 from collections.abc import Mapping
 from typing import Generator
 from typing import Optional
@@ -80,6 +81,14 @@ def pytest_configure(config: Config) -> None:
         "raises, and if the test fails in other ways, it will be reported as "
         "a true failure. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail",
     )
+    if not config.getini("xfail_strict"):
+        warnings.warn(
+            "In a future major release of pytest, the default 'strict' parameter behavior"
+            "for xfail markers will change from False to True. "
+            "Consider setting 'xfail_strict = True' in your pytest configuration"
+            "or use a plugin for handling flaky tests.",
+            FutureWarning,
+        )
 
 
 def evaluate_condition(item: Item, mark: Mark, condition: object) -> Tuple[bool, str]:

testing/test_skipping.py

@@ -326,6 +326,19 @@ class TestXFail:
         assert callreport.passed
         assert callreport.wasxfail == "this is an xfail"
 
+    def test_xfail_xpassed_warning(self, pytester: Pytester) -> None:
+        p = pytester.makepyfile(
+            test_one="""
+            import pytest
+            @pytest.mark.xfail
+            def test_that():
+                assert 1
+        """
+        )
+        result = pytester.runpytest(p, "-rX")
+        result.stdout.fnmatch_lines(["*FutureWarning*", "*1 xpassed*"])
+        assert result.ret == 0
+
     def test_xfail_using_platform(self, pytester: Pytester) -> None:
         """Verify that platform can be used with xfail statements."""
         item = pytester.getitem(