From c93bc1e0cacb62a5fe51c0fcf8d346271b56171d Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sat, 23 Sep 2023 17:18:43 -0400 Subject: [PATCH] Add OuputVerbosity and use it in assertions --- AUTHORS | 1 + changelog/11387.feature.rst | 1 + doc/en/how-to/output.rst | 10 +++++ doc/en/reference/reference.rst | 13 ++++++ src/_pytest/assertion/__init__.py | 9 ++++ src/_pytest/assertion/truncate.py | 4 +- src/_pytest/assertion/util.py | 2 +- src/_pytest/config/__init__.py | 51 ++++++++++++++++++++- src/pytest/__init__.py | 2 + testing/test_assertion.py | 21 ++++++--- testing/test_config.py | 73 +++++++++++++++++++++++++++++++ 11 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 changelog/11387.feature.rst diff --git a/AUTHORS b/AUTHORS index f8c66cae9..c138afbd7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -291,6 +291,7 @@ Ondřej Súkup Oscar Benjamin Parth Patel Patrick Hayes +Patrick Lannigan Paul Müller Paul Reece Pauli Virtanen diff --git a/changelog/11387.feature.rst b/changelog/11387.feature.rst new file mode 100644 index 000000000..d8d9f6a6a --- /dev/null +++ b/changelog/11387.feature.rst @@ -0,0 +1 @@ +:confval:`verbosity_assertions` option added to be able to control assertion output independent of the application wide verbosity level. diff --git a/doc/en/how-to/output.rst b/doc/en/how-to/output.rst index e8e9af0c7..0c803f447 100644 --- a/doc/en/how-to/output.rst +++ b/doc/en/how-to/output.rst @@ -270,6 +270,16 @@ situations, for example you are shown even fixtures that start with ``_`` if you Using higher verbosity levels (``-vvv``, ``-vvvv``, ...) is supported, but has no effect in pytest itself at the moment, however some plugins might make use of higher verbosity. +Fine gain verbosity +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In addition to specifying the application wide verbosity level, it is possible to control specific aspects independently. +This is done by setting a verbosity level in the configuration file for the specific aspect of the output. + +``verbosity_assertions``: Controls how verbose the assertion output should be when pytest is executed. A value of ``2`` +would have the same output as the previous example, but each test inside the file is shown by a single character in the +output. + .. _`pytest.detailed_failed_tests_usage`: Producing a detailed summary report diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index d8efbbbaa..6605aafd2 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1819,6 +1819,19 @@ passed multiple times. The expected format is ``name=value``. For example:: clean_db +.. confval:: verbosity_assertions + + Set a verbosity level specifically for assertion related output, overriding the application wide level. + + + .. code-block:: ini + + [pytest] + verbosity_assertions = 2 + + Defaults to application wide verbosity level. + + .. confval:: xfail_strict If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the diff --git a/src/_pytest/assertion/__init__.py b/src/_pytest/assertion/__init__.py index 64ad4b0e6..5cf0cfc34 100644 --- a/src/_pytest/assertion/__init__.py +++ b/src/_pytest/assertion/__init__.py @@ -12,6 +12,7 @@ from _pytest.assertion import util from _pytest.assertion.rewrite import assertstate_key from _pytest.config import Config from _pytest.config import hookimpl +from _pytest.config import OutputVerbosity from _pytest.config.argparsing import Parser from _pytest.nodes import Item @@ -42,6 +43,14 @@ def pytest_addoption(parser: Parser) -> None: help="Enables the pytest_assertion_pass hook. " "Make sure to delete any previously generated pyc cache files.", ) + OutputVerbosity.add_ini( + parser, + "assertions", + help=( + "Specify a verbosity level for assertions, overriding the main level. " + "Higher levels will provide more a more detailed explanation when an assertion fails." + ), + ) def register_assert_rewrite(*names: str) -> None: diff --git a/src/_pytest/assertion/truncate.py b/src/_pytest/assertion/truncate.py index dfd6f65d2..8f4252d40 100644 --- a/src/_pytest/assertion/truncate.py +++ b/src/_pytest/assertion/truncate.py @@ -1,7 +1,7 @@ """Utilities for truncating assertion output. Current default behaviour is to truncate assertion explanations at -~8 terminal lines, unless running in "-vv" mode or running on CI. +terminal lines, unless running with a verbosity level of at least 2 or running on CI. """ from typing import List from typing import Optional @@ -26,7 +26,7 @@ def truncate_if_required( def _should_truncate_item(item: Item) -> bool: """Whether or not this test item is eligible for truncation.""" - verbose = item.config.option.verbose + verbose = item.config.output_verbosity.verbosity_for("assertions") return verbose < 2 and not util.running_on_ci() diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 01534797d..d7e9c6346 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -161,7 +161,7 @@ def assertrepr_compare( config, op: str, left: Any, right: Any, use_ascii: bool = False ) -> Optional[List[str]]: """Return specialised explanations for some operators/operands.""" - verbose = config.getoption("verbose") + verbose = config.output_verbosity.verbosity_for("assertions") # Strings which normalize equal are often hard to distinguish when printed; use ascii() to make this easier. # See issue #3246. diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 447ebc42a..7ff1dace6 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -69,7 +69,7 @@ from _pytest.warning_types import warn_explicit_for if TYPE_CHECKING: from _pytest._code.code import _TracebackStyle from _pytest.terminal import TerminalReporter - from .argparsing import Argument + from .argparsing import Argument, Parser _PluggyPlugin = object @@ -1020,6 +1020,7 @@ class Config: ) self.args_source = Config.ArgsSource.ARGS self.args: List[str] = [] + self.output_verbosity = OutputVerbosity(self) if TYPE_CHECKING: from _pytest.cacheprovider import Cache @@ -1662,6 +1663,54 @@ class Config: ) +class OutputVerbosity: + DEFAULT = "auto" + _option_name_fmt = "verbosity_{}" + + def __init__(self, config: Config) -> None: + self._config = config + + @property + def verbose(self) -> int: + """Application wide verbosity level.""" + return cast(int, self._config.option.verbose) + + def verbosity_for(self, output_type: str) -> int: + """Return verbosity level for the given output type. + + :param output_type: Name of the output type. + + If the level is not configured, the value of ``config.option.verbose``. + """ + level = self._config.getini(OutputVerbosity._ini_name(output_type)) + + if level == OutputVerbosity.DEFAULT: + return self.verbose + return int(level) + + @staticmethod + def _ini_name(output_type: str) -> str: + return f"verbosity_{output_type}" + + @staticmethod + def add_ini(parser: "Parser", output_type: str, help: str) -> None: + """Add a output verbosity configuration option for the given output type. + + :param parser: Parser for command line arguments and ini-file values. + :param output_type: Name of the output type. + :param help: Description of the output this type controls. + + The value should be retrieved via a call to + :py:func:`config.output_verbosity.verbosity_for(name) `. + """ + parser.addini( + OutputVerbosity._ini_name(output_type), + help=help, + type="string", + default=OutputVerbosity.DEFAULT, + ) + + def _assertion_supported() -> bool: try: assert False diff --git a/src/pytest/__init__.py b/src/pytest/__init__.py index 0aa496a2f..93d659a53 100644 --- a/src/pytest/__init__.py +++ b/src/pytest/__init__.py @@ -15,6 +15,7 @@ from _pytest.config import ExitCode from _pytest.config import hookimpl from _pytest.config import hookspec from _pytest.config import main +from _pytest.config import OutputVerbosity from _pytest.config import PytestPluginManager from _pytest.config import UsageError from _pytest.config.argparsing import OptionGroup @@ -126,6 +127,7 @@ __all__ = [ "Module", "MonkeyPatch", "OptionGroup", + "OutputVerbosity", "Package", "param", "Parser", diff --git a/testing/test_assertion.py b/testing/test_assertion.py index c04c31f31..67cef9735 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -17,12 +17,23 @@ from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester -def mock_config(verbose=0): - class Config: - def getoption(self, name): - if name == "verbose": +def mock_config(verbose: int = 0, assertion_override: Optional[int] = None): + class OutputVerbosity: + @property + def verbose(self) -> int: + return verbose + + def verbosity_for(self, output_type: str) -> int: + if output_type == "assertions": + if assertion_override is not None: + return assertion_override return verbose - raise KeyError("Not mocked out: %s" % name) + + raise KeyError("Not mocked out: %s" % output_type) + + class Config: + def __init__(self) -> None: + self.output_verbosity = OutputVerbosity() return Config() diff --git a/testing/test_config.py b/testing/test_config.py index ded307901..85d41e663 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -20,7 +20,9 @@ from _pytest.config import _strtobool from _pytest.config import Config from _pytest.config import ConftestImportFailure from _pytest.config import ExitCode +from _pytest.config import OutputVerbosity from _pytest.config import parse_warning_filter +from _pytest.config.argparsing import Parser from _pytest.config.exceptions import UsageError from _pytest.config.findpaths import determine_setup from _pytest.config.findpaths import get_common_ancestor @@ -2181,3 +2183,74 @@ class TestDebugOptions: "*Default: pytestdebug.log.", ] ) + + +class TestOutputVerbosity: + SOME_OUTPUT_TYPE = "foo" + SOME_OUTPUT_VERBOSITY_LEVEL = 5 + + class VerbosityIni: + def pytest_addoption(self, parser: Parser) -> None: + OutputVerbosity.add_ini( + parser, TestOutputVerbosity.SOME_OUTPUT_TYPE, help="some help text" + ) + + def test_verbose_matches_option_verbose( + self, pytester: Pytester, tmp_path: Path + ) -> None: + tmp_path.joinpath("pytest.ini").write_text( + textwrap.dedent( + """\ + [pytest] + addopts = --verbose + """ + ), + encoding="utf-8", + ) + + config = pytester.parseconfig(tmp_path) + + assert config.option.verbose == config.output_verbosity.verbose + + def test_level_matches_verbost_when_not_specified( + self, pytester: Pytester, tmp_path: Path + ) -> None: + tmp_path.joinpath("pytest.ini").write_text( + textwrap.dedent( + """\ + [pytest] + addopts = --verbose + """ + ), + encoding="utf-8", + ) + pytester.plugins = [TestOutputVerbosity.VerbosityIni()] + + config = pytester.parseconfig(tmp_path) + + assert ( + config.output_verbosity.verbosity_for(TestOutputVerbosity.SOME_OUTPUT_TYPE) + == config.output_verbosity.verbose + ) + + def test_level_matches_specified_override( + self, pytester: Pytester, tmp_path: Path + ) -> None: + tmp_path.joinpath("pytest.ini").write_text( + textwrap.dedent( + f"""\ + [pytest] + addopts = --verbose + verbosity_{TestOutputVerbosity.SOME_OUTPUT_TYPE} = {TestOutputVerbosity.SOME_OUTPUT_VERBOSITY_LEVEL} + """ + ), + encoding="utf-8", + ) + pytester.plugins = [TestOutputVerbosity.VerbosityIni()] + + config = pytester.parseconfig(tmp_path) + + assert ( + config.output_verbosity.verbosity_for(TestOutputVerbosity.SOME_OUTPUT_TYPE) + == TestOutputVerbosity.SOME_OUTPUT_VERBOSITY_LEVEL + )