diff --git a/testing/test_recwarn.py b/testing/test_recwarn.py
index 8f39818db..c7eac6c40 100644
--- a/testing/test_recwarn.py
+++ b/testing/test_recwarn.py
@@ -479,15 +479,42 @@ class TestWarns:
                 warnings.warn("some warning", category=FutureWarning)
                 raise ValueError("some exception")
 
-    def test_multiple_arg_custom_warning(self) -> None:
-        """Test for issue #11906."""
 
-        class CustomWarning(UserWarning):
-            def __init__(self, a, b):
-                pass
+def test_raise_type_error_on_non_string_warning() -> None:
+    """Check pytest.warns validates warning messages are strings (#10865)."""
+    with pytest.raises(TypeError, match="Warning message must be str"):
+        with pytest.warns(UserWarning):
+            warnings.warn(1)  # type: ignore
 
-        with pytest.warns(CustomWarning):
-            with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"):
-                with pytest.warns(CustomWarning, match="not gonna match"):
-                    a, b = 1, 2
-                    warnings.warn(CustomWarning(a, b))
+
+def test_no_raise_type_error_on_string_warning() -> None:
+    """Check pytest.warns validates warning messages are strings (#10865)."""
+    with pytest.warns(UserWarning):
+        warnings.warn("Warning")
+
+
+@pytest.mark.skipif(
+    hasattr(sys, "pypy_version_info"),
+    reason="Not for pypy",
+)
+def test_raise_type_error_on_non_string_warning_cpython() -> None:
+    # Check that we get the same behavior with the stdlib, at least if filtering
+    # (see https://github.com/python/cpython/issues/103577 for details)
+    with pytest.raises(TypeError):
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "test")
+            warnings.warn(1)  # type: ignore
+
+
+def test_multiple_arg_custom_warning(self) -> None:
+    """Test for issue #11906."""
+
+    class CustomWarning(UserWarning):
+        def __init__(self, a, b):
+            pass
+
+    with pytest.warns(CustomWarning):
+        with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"):
+            with pytest.warns(CustomWarning, match="not gonna match"):
+                a, b = 1, 2
+                warnings.warn(CustomWarning(a, b))