Merge pull request #4935 from Zac-HD/warn-unknown-marks

Emit warning on unknown marks via decorator

changelog/4826.feature.rst

@@ -0,0 +1,2 @@
+A warning is now emitted when unknown marks are used as a decorator.
+This is often due to a typo, which can lead to silently broken tests.

doc/en/mark.rst

@@ -31,7 +31,10 @@ which also serve as documentation.
 Raising errors on unknown marks
 -------------------------------
 
-Marks can be registered in ``pytest.ini`` like this:
+Unknown marks applied with the ``@pytest.mark.name_of_the_mark`` decorator
+will always emit a warning, in order to avoid silently doing something
+surprising due to mis-typed names.  You can disable the warning for custom
+marks by registering them in ``pytest.ini`` like this:
 
 .. code-block:: ini
 

src/_pytest/mark/__init__.py

@@ -147,7 +147,6 @@ def pytest_collection_modifyitems(items, config):
 
 def pytest_configure(config):
     config._old_mark_config = MARK_GEN._config
-    if config.option.strict:
     MARK_GEN._config = config
 
     empty_parameterset = config.getini(EMPTY_PARAMETERSET_OPTION)

src/_pytest/mark/structures.py

@@ -11,6 +11,7 @@ from ..compat import getfslineno
 from ..compat import MappingMixin
 from ..compat import NOTSET
 from _pytest.outcomes import fail
+from _pytest.warning_types import UnknownMarkWarning
 
 EMPTY_PARAMETERSET_OPTION = "empty_parameter_set_mark"
 
@@ -283,28 +284,38 @@ class MarkGenerator(object):
     on the ``test_function`` object. """
 
     _config = None
+    _markers = set()
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError("Marker name must NOT start with underscore")
-        if self._config is not None:
-            self._check(name)
-        return MarkDecorator(Mark(name, (), {}))
 
-    def _check(self, name):
-        try:
-            if name in self._markers:
-                return
-        except AttributeError:
-            pass
-        self._markers = values = set()
-        for line in self._config.getini("markers"):
-            marker = line.split(":", 1)[0]
-            marker = marker.rstrip()
-            x = marker.split("(", 1)[0]
-            values.add(x)
+        if self._config is not None:
+            # We store a set of markers as a performance optimisation - if a mark
+            # name is in the set we definitely know it, but a mark may be known and
+            # not in the set.  We therefore start by updating the set!
             if name not in self._markers:
+                for line in self._config.getini("markers"):
+                    # example lines: "skipif(condition): skip the given test if..."
+                    # or "hypothesis: tests which use Hypothesis", so to get the
+                    # marker name we we split on both `:` and `(`.
+                    marker = line.split(":")[0].split("(")[0].strip()
+                    self._markers.add(marker)
+
+            # If the name is not in the set of known marks after updating,
+            # then it really is time to issue a warning or an error.
+            if name not in self._markers:
+                if self._config.option.strict:
                     fail("{!r} not a registered marker".format(name), pytrace=False)
+                else:
+                    warnings.warn(
+                        "Unknown pytest.mark.%s - is this a typo?  You can register "
+                        "custom marks to avoid this warning - for details, see "
+                        "https://docs.pytest.org/en/latest/mark.html" % name,
+                        UnknownMarkWarning,
+                    )
+
+        return MarkDecorator(Mark(name, (), {}))
 
 
 MARK_GEN = MarkGenerator()

src/_pytest/warning_types.py

@@ -9,6 +9,15 @@ class PytestWarning(UserWarning):
     """
 
 
+class UnknownMarkWarning(PytestWarning):
+    """
+    Bases: :class:`PytestWarning`.
+
+    Warning emitted on use of unknown markers.
+    See https://docs.pytest.org/en/latest/mark.html for details.
+    """
+
+
 class PytestDeprecationWarning(PytestWarning, DeprecationWarning):
     """
     Bases: :class:`pytest.PytestWarning`, :class:`DeprecationWarning`.

tox.ini

@@ -165,6 +165,8 @@ filterwarnings =
     ignore::pytest.PytestExperimentalApiWarning
     # Do not cause SyntaxError for invalid escape sequences in py37.
     default:invalid escape sequence:DeprecationWarning
+    # ignore use of unregistered marks, because we use many to test the implementation
+    ignore::_pytest.warning_types.UnknownMarkWarning
 pytester_example_dir = testing/example_scripts
 markers =
     issue