Merge pull request #12500 from lovetheguitar/feat/support_marker_kwarg_in_marker_expressions

AUTHORS

@@ -245,6 +245,7 @@ Levon Saldamli
 Lewis Cowles
 Llandy Riveron Del Risco
 Loic Esteve
+lovetheguitar
 Lukas Bednar
 Luke Murphy
 Maciek Fijalkowski

changelog/12281.feature.rst

@@ -0,0 +1,8 @@
+Added support for keyword matching in marker expressions.
+
+Now tests can be selected by marker keyword arguments.
+Supported values are :class:`int`, (unescaped) :class:`str`, :class:`bool` & :data:`None`.
+
+See :ref:`marker examples <marker_keyword_expression_example>` for more information.
+
+-- by :user:`lovetheguitar`

doc/en/example/markers.rst

@@ -25,10 +25,12 @@ You can "mark" a test function with custom metadata like this:
         pass  # perform some webtest test for your app
 
 
+    @pytest.mark.device(serial="123")
     def test_something_quick():
         pass
 
 
+    @pytest.mark.device(serial="abc")
     def test_another():
         pass
 
@@ -71,6 +73,28 @@ Or the inverse, running all tests except the webtest ones:
 
     ===================== 3 passed, 1 deselected in 0.12s ======================
 
+.. _`marker_keyword_expression_example`:
+
+Additionally, you can restrict a test run to only run tests matching one or multiple marker
+keyword arguments, e.g. to run only tests marked with ``device`` and the specific ``serial="123"``:
+
+.. code-block:: pytest
+
+    $ pytest -v -m 'device(serial="123")'
+    =========================== test session starts ============================
+    platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python
+    cachedir: .pytest_cache
+    rootdir: /home/sweet/project
+    collecting ... collected 4 items / 3 deselected / 1 selected
+
+    test_server.py::test_something_quick PASSED                          [100%]
+
+    ===================== 1 passed, 3 deselected in 0.12s ======================
+
+.. note:: Only keyword argument matching is supported in marker expressions.
+
+.. note:: Only :class:`int`, (unescaped) :class:`str`, :class:`bool` & :data:`None` values are supported in marker expressions.
+
 Selecting tests based on their node ID
 --------------------------------------
 

doc/en/how-to/usage.rst

@@ -76,11 +76,19 @@ Specifying a specific parametrization of a test:
 
 **Run tests by marker expressions**
 
+To run all tests which are decorated with the ``@pytest.mark.slow`` decorator:
+
 .. code-block:: bash
 
     pytest -m slow
 
-Will run all tests which are decorated with the ``@pytest.mark.slow`` decorator.
+
+To run all tests which are decorated with the annotated ``@pytest.mark.slow(phase=1)`` decorator,
+with the ``phase`` keyword argument set to ``1``:
+
+.. code-block:: bash
+
+    pytest -m slow(phase=1)
 
 For more information see :ref:`marks <mark>`.
 

pyproject.toml

@@ -358,6 +358,9 @@ markers = [
     "foo",
     "bar",
     "baz",
+    "number_mark",
+    "builtin_matchers_mark",
+    "str_mark",
     # conftest.py reorders tests moving slow ones to the end of the list
     "slow",
     # experimental mark for all tests using pexpect

src/_pytest/mark/__init__.py

@@ -2,9 +2,11 @@
 
 from __future__ import annotations
 
+import collections
 import dataclasses
 from typing import AbstractSet
 from typing import Collection
+from typing import Iterable
 from typing import Optional
 from typing import TYPE_CHECKING
 
@@ -21,6 +23,7 @@ from _pytest.config import Config
 from _pytest.config import ExitCode
 from _pytest.config import hookimpl
 from _pytest.config import UsageError
+from _pytest.config.argparsing import NOT_SET
 from _pytest.config.argparsing import Parser
 from _pytest.stash import StashKey
 
@@ -181,7 +184,9 @@ class KeywordMatcher:
 
         return cls(mapped_names)
 
-    def __call__(self, subname: str) -> bool:
+    def __call__(self, subname: str, /, **kwargs: str | int | bool | None) -> bool:
+        if kwargs:
+            raise UsageError("Keyword expressions do not support call parameters.")
         subname = subname.lower()
         names = (name.lower() for name in self._names)
 
@@ -218,17 +223,26 @@ class MarkMatcher:
     Tries to match on any marker names, attached to the given colitem.
     """
 
-    __slots__ = ("own_mark_names",)
+    __slots__ = ("own_mark_name_mapping",)
 
-    own_mark_names: AbstractSet[str]
+    own_mark_name_mapping: dict[str, list[Mark]]
 
     @classmethod
-    def from_item(cls, item: Item) -> MarkMatcher:
+    def from_markers(cls, markers: Iterable[Mark]) -> MarkMatcher:
-        mark_names = {mark.name for mark in item.iter_markers()}
+        mark_name_mapping = collections.defaultdict(list)
-        return cls(mark_names)
+        for mark in markers:
+            mark_name_mapping[mark.name].append(mark)
+        return cls(mark_name_mapping)
 
-    def __call__(self, name: str) -> bool:
+    def __call__(self, name: str, /, **kwargs: str | int | bool | None) -> bool:
-        return name in self.own_mark_names
+        if not (matches := self.own_mark_name_mapping.get(name, [])):
+            return False
+
+        for mark in matches:
+            if all(mark.kwargs.get(k, NOT_SET) == v for k, v in kwargs.items()):
+                return True
+
+        return False
 
 
 def deselect_by_mark(items: list[Item], config: Config) -> None:
@@ -240,7 +254,7 @@ def deselect_by_mark(items: list[Item], config: Config) -> None:
     remaining: list[Item] = []
     deselected: list[Item] = []
     for item in items:
-        if expr.evaluate(MarkMatcher.from_item(item)):
+        if expr.evaluate(MarkMatcher.from_markers(item.iter_markers())):
             remaining.append(item)
         else:
             deselected.append(item)

src/_pytest/mark/expression.py

@@ -5,7 +5,8 @@ The grammar is:
 expression: expr? EOF
 expr:       and_expr ('or' and_expr)*
 and_expr:   not_expr ('and' not_expr)*
-not_expr:   'not' not_expr | '(' expr ')' | ident
+not_expr:   'not' not_expr | '(' expr ')' | ident ( '(' name '=' value ( ', ' name '=' value )*  ')')*
+
 ident:      (\w|:|\+|-|\.|\[|\]|\\|/)+
 
 The semantics are:
@@ -20,12 +21,15 @@ from __future__ import annotations
 import ast
 import dataclasses
 import enum
+import keyword
 import re
 import types
-from typing import Callable
 from typing import Iterator
+from typing import Literal
 from typing import Mapping
 from typing import NoReturn
+from typing import overload
+from typing import Protocol
 from typing import Sequence
 
 
@@ -43,6 +47,9 @@ class TokenType(enum.Enum):
     NOT = "not"
     IDENT = "identifier"
     EOF = "end of input"
+    EQUAL = "="
+    STRING = "str"
+    COMMA = ","
 
 
 @dataclasses.dataclass(frozen=True)
@@ -86,6 +93,27 @@ class Scanner:
             elif input[pos] == ")":
                 yield Token(TokenType.RPAREN, ")", pos)
                 pos += 1
+            elif input[pos] == "=":
+                yield Token(TokenType.EQUAL, "=", pos)
+                pos += 1
+            elif input[pos] == ",":
+                yield Token(TokenType.COMMA, ",", pos)
+                pos += 1
+            elif (quote_char := input[pos]) in ("'", '"'):
+                end_quote_pos = input.find(quote_char, pos + 1)
+                if end_quote_pos == -1:
+                    raise ParseError(
+                        pos + 1,
+                        f'closing quote "{quote_char}" is missing',
+                    )
+                value = input[pos : end_quote_pos + 1]
+                if (backslash_pos := input.find("\\")) != -1:
+                    raise ParseError(
+                        backslash_pos + 1,
+                        r'escaping with "\" not supported in marker expression',
+                    )
+                yield Token(TokenType.STRING, value, pos)
+                pos += len(value)
             else:
                 match = re.match(r"(:?\w|:|\+|-|\.|\[|\]|\\|/)+", input[pos:])
                 if match:
@@ -106,6 +134,14 @@ class Scanner:
                     )
         yield Token(TokenType.EOF, "", pos)
 
+    @overload
+    def accept(self, type: TokenType, *, reject: Literal[True]) -> Token: ...
+
+    @overload
+    def accept(
+        self, type: TokenType, *, reject: Literal[False] = False
+    ) -> Token | None: ...
+
     def accept(self, type: TokenType, *, reject: bool = False) -> Token | None:
         if self.current.type is type:
             token = self.current
@@ -166,18 +202,87 @@ def not_expr(s: Scanner) -> ast.expr:
         return ret
     ident = s.accept(TokenType.IDENT)
     if ident:
-        return ast.Name(IDENT_PREFIX + ident.value, ast.Load())
+        name = ast.Name(IDENT_PREFIX + ident.value, ast.Load())
+        if s.accept(TokenType.LPAREN):
+            ret = ast.Call(func=name, args=[], keywords=all_kwargs(s))
+            s.accept(TokenType.RPAREN, reject=True)
+        else:
+            ret = name
+        return ret
+
     s.reject((TokenType.NOT, TokenType.LPAREN, TokenType.IDENT))
 
 
-class MatcherAdapter(Mapping[str, bool]):
+BUILTIN_MATCHERS = {"True": True, "False": False, "None": None}
+
+
+def single_kwarg(s: Scanner) -> ast.keyword:
+    keyword_name = s.accept(TokenType.IDENT, reject=True)
+    if not keyword_name.value.isidentifier():
+        raise ParseError(
+            keyword_name.pos + 1,
+            f"not a valid python identifier {keyword_name.value}",
+        )
+    if keyword.iskeyword(keyword_name.value):
+        raise ParseError(
+            keyword_name.pos + 1,
+            f"unexpected reserved python keyword `{keyword_name.value}`",
+        )
+    s.accept(TokenType.EQUAL, reject=True)
+
+    if value_token := s.accept(TokenType.STRING):
+        value: str | int | bool | None = value_token.value[1:-1]  # strip quotes
+    else:
+        value_token = s.accept(TokenType.IDENT, reject=True)
+        if (
+            (number := value_token.value).isdigit()
+            or number.startswith("-")
+            and number[1:].isdigit()
+        ):
+            value = int(number)
+        elif value_token.value in BUILTIN_MATCHERS:
+            value = BUILTIN_MATCHERS[value_token.value]
+        else:
+            raise ParseError(
+                value_token.pos + 1,
+                f'unexpected character/s "{value_token.value}"',
+            )
+
+    ret = ast.keyword(keyword_name.value, ast.Constant(value))
+    return ret
+
+
+def all_kwargs(s: Scanner) -> list[ast.keyword]:
+    ret = [single_kwarg(s)]
+    while s.accept(TokenType.COMMA):
+        ret.append(single_kwarg(s))
+    return ret
+
+
+class MatcherCall(Protocol):
+    def __call__(self, name: str, /, **kwargs: str | int | bool | None) -> bool: ...
+
+
+@dataclasses.dataclass
+class MatcherNameAdapter:
+    matcher: MatcherCall
+    name: str
+
+    def __bool__(self) -> bool:
+        return self.matcher(self.name)
+
+    def __call__(self, **kwargs: str | int | bool | None) -> bool:
+        return self.matcher(self.name, **kwargs)
+
+
+class MatcherAdapter(Mapping[str, MatcherNameAdapter]):
     """Adapts a matcher function to a locals mapping as required by eval()."""
 
-    def __init__(self, matcher: Callable[[str], bool]) -> None:
+    def __init__(self, matcher: MatcherCall) -> None:
         self.matcher = matcher
 
-    def __getitem__(self, key: str) -> bool:
+    def __getitem__(self, key: str) -> MatcherNameAdapter:
-        return self.matcher(key[len(IDENT_PREFIX) :])
+        return MatcherNameAdapter(matcher=self.matcher, name=key[len(IDENT_PREFIX) :])
 
     def __iter__(self) -> Iterator[str]:
         raise NotImplementedError()
@@ -211,7 +316,7 @@ class Expression:
         )
         return Expression(code)
 
-    def evaluate(self, matcher: Callable[[str], bool]) -> bool:
+    def evaluate(self, matcher: MatcherCall) -> bool:
         """Evaluate the match expression.
 
         :param matcher:
@@ -220,5 +325,5 @@ class Expression:
 
         :returns: Whether the expression matches or not.
         """
-        ret: bool = eval(self.code, {"__builtins__": {}}, MatcherAdapter(matcher))
+        ret: bool = bool(eval(self.code, {"__builtins__": {}}, MatcherAdapter(matcher)))
         return ret

testing/test_mark.py

@@ -233,6 +233,54 @@ def test_mark_option(
     assert passed_str == expected_passed
 
 
+@pytest.mark.parametrize(
+    ("expr", "expected_passed"),
+    [  # TODO: improve/sort out
+        ("car(color='red')", ["test_one"]),
+        ("car(color='red') or car(color='blue')", ["test_one", "test_two"]),
+        ("car and not car(temp=5)", ["test_one", "test_three"]),
+        ("car(temp=4)", ["test_one"]),
+        ("car(temp=4) or car(temp=5)", ["test_one", "test_two"]),
+        ("car(temp=4) and car(temp=5)", []),
+        ("car(temp=-5)", ["test_three"]),
+        ("car(ac=True)", ["test_one"]),
+        ("car(ac=False)", ["test_two"]),
+        ("car(ac=None)", ["test_three"]),  # test NOT_NONE_SENTINEL
+    ],
+    ids=str,
+)
+def test_mark_option_with_kwargs(
+    expr: str, expected_passed: list[str | None], pytester: Pytester
+) -> None:
+    pytester.makepyfile(
+        """
+        import pytest
+        @pytest.mark.car
+        @pytest.mark.car(ac=True)
+        @pytest.mark.car(temp=4)
+        @pytest.mark.car(color="red")
+        def test_one():
+            pass
+        @pytest.mark.car
+        @pytest.mark.car(ac=False)
+        @pytest.mark.car(temp=5)
+        @pytest.mark.car(color="blue")
+        def test_two():
+            pass
+        @pytest.mark.car
+        @pytest.mark.car(ac=None)
+        @pytest.mark.car(temp=-5)
+        def test_three():
+            pass
+
+    """
+    )
+    rec = pytester.inline_run("-m", expr)
+    passed, skipped, fail = rec.listoutcomes()
+    passed_str = [x.nodeid.split("::")[-1] for x in passed]
+    assert passed_str == expected_passed
+
+
 @pytest.mark.parametrize(
     ("expr", "expected_passed"),
     [("interface", ["test_interface"]), ("not interface", ["test_nointer"])],
@@ -372,6 +420,10 @@ def test_parametrize_with_module(pytester: Pytester) -> None:
             "not or",
             "at column 5: expected not OR left parenthesis OR identifier; got or",
         ),
+        (
+            "nonexistent_mark(non_supported='kwarg')",
+            "Keyword expressions do not support call parameters",
+        ),
     ],
 )
 def test_keyword_option_wrong_arguments(

testing/test_mark_expression.py

@@ -1,14 +1,17 @@
 from __future__ import annotations
 
 from typing import Callable
+from typing import cast
 
+from _pytest.mark import MarkMatcher
 from _pytest.mark.expression import Expression
+from _pytest.mark.expression import MatcherCall
 from _pytest.mark.expression import ParseError
 import pytest
 
 
 def evaluate(input: str, matcher: Callable[[str], bool]) -> bool:
-    return Expression.compile(input).evaluate(matcher)
+    return Expression.compile(input).evaluate(cast(MatcherCall, matcher))
 
 
 def test_empty_is_false() -> None:
@@ -153,6 +156,8 @@ def test_syntax_errors(expr: str, column: int, message: str) -> None:
         "1234",
         "1234abcd",
         "1234and",
+        "1234or",
+        "1234not",
         "notandor",
         "not_and_or",
         "not[and]or",
@@ -195,3 +200,119 @@ def test_valid_idents(ident: str) -> None:
 def test_invalid_idents(ident: str) -> None:
     with pytest.raises(ParseError):
         evaluate(ident, lambda ident: True)
+
+
+@pytest.mark.parametrize(
+    "expr, expected_error_msg",
+    (
+        ("mark(True=False)", "unexpected reserved python keyword `True`"),
+        ("mark(def=False)", "unexpected reserved python keyword `def`"),
+        ("mark(class=False)", "unexpected reserved python keyword `class`"),
+        ("mark(if=False)", "unexpected reserved python keyword `if`"),
+        ("mark(else=False)", "unexpected reserved python keyword `else`"),
+        ("mark(valid=False, def=1)", "unexpected reserved python keyword `def`"),
+        ("mark(1)", "not a valid python identifier 1"),
+        ("mark(var:=False", "not a valid python identifier var:"),
+        ("mark(1=2)", "not a valid python identifier 1"),
+        ("mark(/=2)", "not a valid python identifier /"),
+        ("mark(var==", "expected identifier; got ="),
+        ("mark(var)", "expected =; got right parenthesis"),
+        ("mark(var=none)", 'unexpected character/s "none"'),
+        ("mark(var=1.1)", 'unexpected character/s "1.1"'),
+        ("mark(var=')", """closing quote "'" is missing"""),
+        ('mark(var=")', 'closing quote """ is missing'),
+        ("""mark(var="')""", 'closing quote """ is missing'),
+        ("""mark(var='")""", """closing quote "'" is missing"""),
+        (
+            r"mark(var='\hugo')",
+            r'escaping with "\\" not supported in marker expression',
+        ),
+        ("mark(empty_list=[])", r'unexpected character/s "\[\]"'),
+    ),
+)
+def test_invalid_kwarg_name_or_value(  # TODO: move to `test_syntax_errors` ?
+    expr: str, expected_error_msg: str, mark_matcher: MarkMatcher
+) -> None:
+    with pytest.raises(ParseError, match=expected_error_msg):
+        assert evaluate(expr, mark_matcher)
+
+
+@pytest.fixture(scope="session")
+def mark_matcher() -> MarkMatcher:
+    markers = [
+        pytest.mark.number_mark(a=1, b=2, c=3, d=999_999).mark,
+        pytest.mark.builtin_matchers_mark(x=True, y=False, z=None).mark,
+        pytest.mark.str_mark(
+            m="M", space="with space", empty="", aaאבגדcc="aaאבגדcc", אבגד="אבגד"
+        ).mark,
+    ]
+
+    return MarkMatcher.from_markers(markers)
+
+
+@pytest.mark.parametrize(
+    "expr, expected",
+    (
+        # happy cases
+        ("number_mark(a=1)", True),
+        ("number_mark(b=2)", True),
+        ("number_mark(a=1,b=2)", True),
+        ("number_mark(a=1,     b=2)", True),
+        ("number_mark(d=999999)", True),
+        ("number_mark(a   =   1,b= 2,     c = 3)", True),
+        # sad cases
+        ("number_mark(a=6)", False),
+        ("number_mark(b=6)", False),
+        ("number_mark(a=1,b=6)", False),
+        ("number_mark(a=6,b=2)", False),
+        ("number_mark(a   =   1,b= 2,     c = 6)", False),
+        ("number_mark(a='1')", False),
+    ),
+)
+def test_keyword_expressions_with_numbers(
+    expr: str, expected: bool, mark_matcher: MarkMatcher
+) -> None:
+    assert evaluate(expr, mark_matcher) is expected
+
+
+@pytest.mark.parametrize(
+    "expr, expected",
+    (
+        ("builtin_matchers_mark(x=True)", True),
+        ("builtin_matchers_mark(x=False)", False),
+        ("builtin_matchers_mark(y=True)", False),
+        ("builtin_matchers_mark(y=False)", True),
+        ("builtin_matchers_mark(z=None)", True),
+        ("builtin_matchers_mark(z=False)", False),
+        ("builtin_matchers_mark(z=True)", False),
+        ("builtin_matchers_mark(z=0)", False),
+        ("builtin_matchers_mark(z=1)", False),
+    ),
+)
+def test_builtin_matchers_keyword_expressions(  # TODO: naming when decided
+    expr: str, expected: bool, mark_matcher: MarkMatcher
+) -> None:
+    assert evaluate(expr, mark_matcher) is expected
+
+
+@pytest.mark.parametrize(
+    "expr, expected",
+    (
+        ("str_mark(m='M')", True),
+        ('str_mark(m="M")', True),
+        ("str_mark(aaאבגדcc='aaאבגדcc')", True),
+        ("str_mark(אבגד='אבגד')", True),
+        ("str_mark(space='with space')", True),
+        ("str_mark(empty='')", True),
+        ('str_mark(empty="")', True),
+        ("str_mark(m='wrong')", False),
+        ("str_mark(aaאבגדcc='wrong')", False),
+        ("str_mark(אבגד='wrong')", False),
+        ("str_mark(m='')", False),
+        ('str_mark(m="")', False),
+    ),
+)
+def test_str_keyword_expressions(
+    expr: str, expected: bool, mark_matcher: MarkMatcher
+) -> None:
+    assert evaluate(expr, mark_matcher) is expected