Merge pull request #12500 from lovetheguitar/feat/support_marker_kwarg_in_marker_expressions

This commit is contained in:
Ronny Pfannschmidt 2024-06-21 22:24:34 +02:00 committed by GitHub
commit 9f134fc6ad
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 356 additions and 20 deletions

View File

@ -245,6 +245,7 @@ Levon Saldamli
Lewis Cowles
Llandy Riveron Del Risco
Loic Esteve
lovetheguitar
Lukas Bednar
Luke Murphy
Maciek Fijalkowski

View File

@ -0,0 +1,8 @@
Added support for keyword matching in marker expressions.
Now tests can be selected by marker keyword arguments.
Supported values are :class:`int`, (unescaped) :class:`str`, :class:`bool` & :data:`None`.
See :ref:`marker examples <marker_keyword_expression_example>` for more information.
-- by :user:`lovetheguitar`

View File

@ -25,10 +25,12 @@ You can "mark" a test function with custom metadata like this:
pass # perform some webtest test for your app
@pytest.mark.device(serial="123")
def test_something_quick():
pass
@pytest.mark.device(serial="abc")
def test_another():
pass
@ -71,6 +73,28 @@ Or the inverse, running all tests except the webtest ones:
===================== 3 passed, 1 deselected in 0.12s ======================
.. _`marker_keyword_expression_example`:
Additionally, you can restrict a test run to only run tests matching one or multiple marker
keyword arguments, e.g. to run only tests marked with ``device`` and the specific ``serial="123"``:
.. code-block:: pytest
$ pytest -v -m 'device(serial="123")'
=========================== test session starts ============================
platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python
cachedir: .pytest_cache
rootdir: /home/sweet/project
collecting ... collected 4 items / 3 deselected / 1 selected
test_server.py::test_something_quick PASSED [100%]
===================== 1 passed, 3 deselected in 0.12s ======================
.. note:: Only keyword argument matching is supported in marker expressions.
.. note:: Only :class:`int`, (unescaped) :class:`str`, :class:`bool` & :data:`None` values are supported in marker expressions.
Selecting tests based on their node ID
--------------------------------------

View File

@ -76,11 +76,19 @@ Specifying a specific parametrization of a test:
**Run tests by marker expressions**
To run all tests which are decorated with the ``@pytest.mark.slow`` decorator:
.. code-block:: bash
pytest -m slow
Will run all tests which are decorated with the ``@pytest.mark.slow`` decorator.
To run all tests which are decorated with the annotated ``@pytest.mark.slow(phase=1)`` decorator,
with the ``phase`` keyword argument set to ``1``:
.. code-block:: bash
pytest -m slow(phase=1)
For more information see :ref:`marks <mark>`.

View File

@ -358,6 +358,9 @@ markers = [
"foo",
"bar",
"baz",
"number_mark",
"builtin_matchers_mark",
"str_mark",
# conftest.py reorders tests moving slow ones to the end of the list
"slow",
# experimental mark for all tests using pexpect

View File

@ -2,9 +2,11 @@
from __future__ import annotations
import collections
import dataclasses
from typing import AbstractSet
from typing import Collection
from typing import Iterable
from typing import Optional
from typing import TYPE_CHECKING
@ -21,6 +23,7 @@ from _pytest.config import Config
from _pytest.config import ExitCode
from _pytest.config import hookimpl
from _pytest.config import UsageError
from _pytest.config.argparsing import NOT_SET
from _pytest.config.argparsing import Parser
from _pytest.stash import StashKey
@ -181,7 +184,9 @@ class KeywordMatcher:
return cls(mapped_names)
def __call__(self, subname: str) -> bool:
def __call__(self, subname: str, /, **kwargs: str | int | bool | None) -> bool:
if kwargs:
raise UsageError("Keyword expressions do not support call parameters.")
subname = subname.lower()
names = (name.lower() for name in self._names)
@ -218,17 +223,26 @@ class MarkMatcher:
Tries to match on any marker names, attached to the given colitem.
"""
__slots__ = ("own_mark_names",)
__slots__ = ("own_mark_name_mapping",)
own_mark_names: AbstractSet[str]
own_mark_name_mapping: dict[str, list[Mark]]
@classmethod
def from_item(cls, item: Item) -> MarkMatcher:
mark_names = {mark.name for mark in item.iter_markers()}
return cls(mark_names)
def from_markers(cls, markers: Iterable[Mark]) -> MarkMatcher:
mark_name_mapping = collections.defaultdict(list)
for mark in markers:
mark_name_mapping[mark.name].append(mark)
return cls(mark_name_mapping)
def __call__(self, name: str) -> bool:
return name in self.own_mark_names
def __call__(self, name: str, /, **kwargs: str | int | bool | None) -> bool:
if not (matches := self.own_mark_name_mapping.get(name, [])):
return False
for mark in matches:
if all(mark.kwargs.get(k, NOT_SET) == v for k, v in kwargs.items()):
return True
return False
def deselect_by_mark(items: list[Item], config: Config) -> None:
@ -240,7 +254,7 @@ def deselect_by_mark(items: list[Item], config: Config) -> None:
remaining: list[Item] = []
deselected: list[Item] = []
for item in items:
if expr.evaluate(MarkMatcher.from_item(item)):
if expr.evaluate(MarkMatcher.from_markers(item.iter_markers())):
remaining.append(item)
else:
deselected.append(item)

View File

@ -5,7 +5,8 @@ The grammar is:
expression: expr? EOF
expr: and_expr ('or' and_expr)*
and_expr: not_expr ('and' not_expr)*
not_expr: 'not' not_expr | '(' expr ')' | ident
not_expr: 'not' not_expr | '(' expr ')' | ident ( '(' name '=' value ( ', ' name '=' value )* ')')*
ident: (\w|:|\+|-|\.|\[|\]|\\|/)+
The semantics are:
@ -20,12 +21,15 @@ from __future__ import annotations
import ast
import dataclasses
import enum
import keyword
import re
import types
from typing import Callable
from typing import Iterator
from typing import Literal
from typing import Mapping
from typing import NoReturn
from typing import overload
from typing import Protocol
from typing import Sequence
@ -43,6 +47,9 @@ class TokenType(enum.Enum):
NOT = "not"
IDENT = "identifier"
EOF = "end of input"
EQUAL = "="
STRING = "str"
COMMA = ","
@dataclasses.dataclass(frozen=True)
@ -86,6 +93,27 @@ class Scanner:
elif input[pos] == ")":
yield Token(TokenType.RPAREN, ")", pos)
pos += 1
elif input[pos] == "=":
yield Token(TokenType.EQUAL, "=", pos)
pos += 1
elif input[pos] == ",":
yield Token(TokenType.COMMA, ",", pos)
pos += 1
elif (quote_char := input[pos]) in ("'", '"'):
end_quote_pos = input.find(quote_char, pos + 1)
if end_quote_pos == -1:
raise ParseError(
pos + 1,
f'closing quote "{quote_char}" is missing',
)
value = input[pos : end_quote_pos + 1]
if (backslash_pos := input.find("\\")) != -1:
raise ParseError(
backslash_pos + 1,
r'escaping with "\" not supported in marker expression',
)
yield Token(TokenType.STRING, value, pos)
pos += len(value)
else:
match = re.match(r"(:?\w|:|\+|-|\.|\[|\]|\\|/)+", input[pos:])
if match:
@ -106,6 +134,14 @@ class Scanner:
)
yield Token(TokenType.EOF, "", pos)
@overload
def accept(self, type: TokenType, *, reject: Literal[True]) -> Token: ...
@overload
def accept(
self, type: TokenType, *, reject: Literal[False] = False
) -> Token | None: ...
def accept(self, type: TokenType, *, reject: bool = False) -> Token | None:
if self.current.type is type:
token = self.current
@ -166,18 +202,87 @@ def not_expr(s: Scanner) -> ast.expr:
return ret
ident = s.accept(TokenType.IDENT)
if ident:
return ast.Name(IDENT_PREFIX + ident.value, ast.Load())
name = ast.Name(IDENT_PREFIX + ident.value, ast.Load())
if s.accept(TokenType.LPAREN):
ret = ast.Call(func=name, args=[], keywords=all_kwargs(s))
s.accept(TokenType.RPAREN, reject=True)
else:
ret = name
return ret
s.reject((TokenType.NOT, TokenType.LPAREN, TokenType.IDENT))
class MatcherAdapter(Mapping[str, bool]):
BUILTIN_MATCHERS = {"True": True, "False": False, "None": None}
def single_kwarg(s: Scanner) -> ast.keyword:
keyword_name = s.accept(TokenType.IDENT, reject=True)
if not keyword_name.value.isidentifier():
raise ParseError(
keyword_name.pos + 1,
f"not a valid python identifier {keyword_name.value}",
)
if keyword.iskeyword(keyword_name.value):
raise ParseError(
keyword_name.pos + 1,
f"unexpected reserved python keyword `{keyword_name.value}`",
)
s.accept(TokenType.EQUAL, reject=True)
if value_token := s.accept(TokenType.STRING):
value: str | int | bool | None = value_token.value[1:-1] # strip quotes
else:
value_token = s.accept(TokenType.IDENT, reject=True)
if (
(number := value_token.value).isdigit()
or number.startswith("-")
and number[1:].isdigit()
):
value = int(number)
elif value_token.value in BUILTIN_MATCHERS:
value = BUILTIN_MATCHERS[value_token.value]
else:
raise ParseError(
value_token.pos + 1,
f'unexpected character/s "{value_token.value}"',
)
ret = ast.keyword(keyword_name.value, ast.Constant(value))
return ret
def all_kwargs(s: Scanner) -> list[ast.keyword]:
ret = [single_kwarg(s)]
while s.accept(TokenType.COMMA):
ret.append(single_kwarg(s))
return ret
class MatcherCall(Protocol):
def __call__(self, name: str, /, **kwargs: str | int | bool | None) -> bool: ...
@dataclasses.dataclass
class MatcherNameAdapter:
matcher: MatcherCall
name: str
def __bool__(self) -> bool:
return self.matcher(self.name)
def __call__(self, **kwargs: str | int | bool | None) -> bool:
return self.matcher(self.name, **kwargs)
class MatcherAdapter(Mapping[str, MatcherNameAdapter]):
"""Adapts a matcher function to a locals mapping as required by eval()."""
def __init__(self, matcher: Callable[[str], bool]) -> None:
def __init__(self, matcher: MatcherCall) -> None:
self.matcher = matcher
def __getitem__(self, key: str) -> bool:
return self.matcher(key[len(IDENT_PREFIX) :])
def __getitem__(self, key: str) -> MatcherNameAdapter:
return MatcherNameAdapter(matcher=self.matcher, name=key[len(IDENT_PREFIX) :])
def __iter__(self) -> Iterator[str]:
raise NotImplementedError()
@ -211,7 +316,7 @@ class Expression:
)
return Expression(code)
def evaluate(self, matcher: Callable[[str], bool]) -> bool:
def evaluate(self, matcher: MatcherCall) -> bool:
"""Evaluate the match expression.
:param matcher:
@ -220,5 +325,5 @@ class Expression:
:returns: Whether the expression matches or not.
"""
ret: bool = eval(self.code, {"__builtins__": {}}, MatcherAdapter(matcher))
ret: bool = bool(eval(self.code, {"__builtins__": {}}, MatcherAdapter(matcher)))
return ret

View File

@ -233,6 +233,54 @@ def test_mark_option(
assert passed_str == expected_passed
@pytest.mark.parametrize(
("expr", "expected_passed"),
[ # TODO: improve/sort out
("car(color='red')", ["test_one"]),
("car(color='red') or car(color='blue')", ["test_one", "test_two"]),
("car and not car(temp=5)", ["test_one", "test_three"]),
("car(temp=4)", ["test_one"]),
("car(temp=4) or car(temp=5)", ["test_one", "test_two"]),
("car(temp=4) and car(temp=5)", []),
("car(temp=-5)", ["test_three"]),
("car(ac=True)", ["test_one"]),
("car(ac=False)", ["test_two"]),
("car(ac=None)", ["test_three"]), # test NOT_NONE_SENTINEL
],
ids=str,
)
def test_mark_option_with_kwargs(
expr: str, expected_passed: list[str | None], pytester: Pytester
) -> None:
pytester.makepyfile(
"""
import pytest
@pytest.mark.car
@pytest.mark.car(ac=True)
@pytest.mark.car(temp=4)
@pytest.mark.car(color="red")
def test_one():
pass
@pytest.mark.car
@pytest.mark.car(ac=False)
@pytest.mark.car(temp=5)
@pytest.mark.car(color="blue")
def test_two():
pass
@pytest.mark.car
@pytest.mark.car(ac=None)
@pytest.mark.car(temp=-5)
def test_three():
pass
"""
)
rec = pytester.inline_run("-m", expr)
passed, skipped, fail = rec.listoutcomes()
passed_str = [x.nodeid.split("::")[-1] for x in passed]
assert passed_str == expected_passed
@pytest.mark.parametrize(
("expr", "expected_passed"),
[("interface", ["test_interface"]), ("not interface", ["test_nointer"])],
@ -372,6 +420,10 @@ def test_parametrize_with_module(pytester: Pytester) -> None:
"not or",
"at column 5: expected not OR left parenthesis OR identifier; got or",
),
(
"nonexistent_mark(non_supported='kwarg')",
"Keyword expressions do not support call parameters",
),
],
)
def test_keyword_option_wrong_arguments(

View File

@ -1,14 +1,17 @@
from __future__ import annotations
from typing import Callable
from typing import cast
from _pytest.mark import MarkMatcher
from _pytest.mark.expression import Expression
from _pytest.mark.expression import MatcherCall
from _pytest.mark.expression import ParseError
import pytest
def evaluate(input: str, matcher: Callable[[str], bool]) -> bool:
return Expression.compile(input).evaluate(matcher)
return Expression.compile(input).evaluate(cast(MatcherCall, matcher))
def test_empty_is_false() -> None:
@ -153,6 +156,8 @@ def test_syntax_errors(expr: str, column: int, message: str) -> None:
"1234",
"1234abcd",
"1234and",
"1234or",
"1234not",
"notandor",
"not_and_or",
"not[and]or",
@ -195,3 +200,119 @@ def test_valid_idents(ident: str) -> None:
def test_invalid_idents(ident: str) -> None:
with pytest.raises(ParseError):
evaluate(ident, lambda ident: True)
@pytest.mark.parametrize(
"expr, expected_error_msg",
(
("mark(True=False)", "unexpected reserved python keyword `True`"),
("mark(def=False)", "unexpected reserved python keyword `def`"),
("mark(class=False)", "unexpected reserved python keyword `class`"),
("mark(if=False)", "unexpected reserved python keyword `if`"),
("mark(else=False)", "unexpected reserved python keyword `else`"),
("mark(valid=False, def=1)", "unexpected reserved python keyword `def`"),
("mark(1)", "not a valid python identifier 1"),
("mark(var:=False", "not a valid python identifier var:"),
("mark(1=2)", "not a valid python identifier 1"),
("mark(/=2)", "not a valid python identifier /"),
("mark(var==", "expected identifier; got ="),
("mark(var)", "expected =; got right parenthesis"),
("mark(var=none)", 'unexpected character/s "none"'),
("mark(var=1.1)", 'unexpected character/s "1.1"'),
("mark(var=')", """closing quote "'" is missing"""),
('mark(var=")', 'closing quote """ is missing'),
("""mark(var="')""", 'closing quote """ is missing'),
("""mark(var='")""", """closing quote "'" is missing"""),
(
r"mark(var='\hugo')",
r'escaping with "\\" not supported in marker expression',
),
("mark(empty_list=[])", r'unexpected character/s "\[\]"'),
),
)
def test_invalid_kwarg_name_or_value( # TODO: move to `test_syntax_errors` ?
expr: str, expected_error_msg: str, mark_matcher: MarkMatcher
) -> None:
with pytest.raises(ParseError, match=expected_error_msg):
assert evaluate(expr, mark_matcher)
@pytest.fixture(scope="session")
def mark_matcher() -> MarkMatcher:
markers = [
pytest.mark.number_mark(a=1, b=2, c=3, d=999_999).mark,
pytest.mark.builtin_matchers_mark(x=True, y=False, z=None).mark,
pytest.mark.str_mark(
m="M", space="with space", empty="", aaאבגדcc="aaאבגדcc", אבגד="אבגד"
).mark,
]
return MarkMatcher.from_markers(markers)
@pytest.mark.parametrize(
"expr, expected",
(
# happy cases
("number_mark(a=1)", True),
("number_mark(b=2)", True),
("number_mark(a=1,b=2)", True),
("number_mark(a=1, b=2)", True),
("number_mark(d=999999)", True),
("number_mark(a = 1,b= 2, c = 3)", True),
# sad cases
("number_mark(a=6)", False),
("number_mark(b=6)", False),
("number_mark(a=1,b=6)", False),
("number_mark(a=6,b=2)", False),
("number_mark(a = 1,b= 2, c = 6)", False),
("number_mark(a='1')", False),
),
)
def test_keyword_expressions_with_numbers(
expr: str, expected: bool, mark_matcher: MarkMatcher
) -> None:
assert evaluate(expr, mark_matcher) is expected
@pytest.mark.parametrize(
"expr, expected",
(
("builtin_matchers_mark(x=True)", True),
("builtin_matchers_mark(x=False)", False),
("builtin_matchers_mark(y=True)", False),
("builtin_matchers_mark(y=False)", True),
("builtin_matchers_mark(z=None)", True),
("builtin_matchers_mark(z=False)", False),
("builtin_matchers_mark(z=True)", False),
("builtin_matchers_mark(z=0)", False),
("builtin_matchers_mark(z=1)", False),
),
)
def test_builtin_matchers_keyword_expressions( # TODO: naming when decided
expr: str, expected: bool, mark_matcher: MarkMatcher
) -> None:
assert evaluate(expr, mark_matcher) is expected
@pytest.mark.parametrize(
"expr, expected",
(
("str_mark(m='M')", True),
('str_mark(m="M")', True),
("str_mark(aaאבגדcc='aaאבגדcc')", True),
("str_mark(אבגד='אבגד')", True),
("str_mark(space='with space')", True),
("str_mark(empty='')", True),
('str_mark(empty="")', True),
("str_mark(m='wrong')", False),
("str_mark(aaאבגדcc='wrong')", False),
("str_mark(אבגד='wrong')", False),
("str_mark(m='')", False),
('str_mark(m="")', False),
),
)
def test_str_keyword_expressions(
expr: str, expected: bool, mark_matcher: MarkMatcher
) -> None:
assert evaluate(expr, mark_matcher) is expected