From 9d9b84d175ec1b390829e527700fc7c7dbbea421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tarc=C3=ADsio=20Fischer?= Date: Fri, 30 Apr 2021 07:36:56 -0300 Subject: [PATCH] Improve pytest.approx error messages readability (Pull request) (#8429) Improve pytest.approx error messages readability (Pull request) --- changelog/8335.improvement.rst | 10 ++ src/_pytest/assertion/util.py | 12 +- src/_pytest/python_api.py | 180 +++++++++++++++++++++++++- testing/python/approx.py | 230 +++++++++++++++++++++++++++++++++ 4 files changed, 428 insertions(+), 4 deletions(-) create mode 100644 changelog/8335.improvement.rst diff --git a/changelog/8335.improvement.rst b/changelog/8335.improvement.rst new file mode 100644 index 000000000..f6c0e3343 --- /dev/null +++ b/changelog/8335.improvement.rst @@ -0,0 +1,10 @@ +Improved :func:`pytest.approx` assertion messages for sequences of numbers. + +The assertion messages now dumps a table with the index and the error of each diff. +Example:: + + > assert [1, 2, 3, 4] == pytest.approx([1, 3, 3, 5]) + E assert comparison failed for 2 values: + E Index | Obtained | Expected + E 1 | 2 | 3 +- 3.0e-06 + E 3 | 4 | 5 +- 5.0e-06 diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 0e54335ab..d29a2a010 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -180,7 +180,15 @@ def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]: if istext(left) and istext(right): explanation = _diff_text(left, right, verbose) else: - if type(left) == type(right) and ( + from _pytest.python_api import ApproxBase + + if isinstance(left, ApproxBase) or isinstance(right, ApproxBase): + # Although the common order should be obtained == expected, this ensures both ways + approx_side = left if isinstance(left, ApproxBase) else right + other_side = right if isinstance(left, ApproxBase) else left + + explanation = approx_side._repr_compare(other_side) + elif type(left) == type(right) and ( isdatacls(left) or isattrs(left) or isnamedtuple(left) ): # Note: unlike dataclasses/attrs, namedtuples compare only the @@ -196,9 +204,11 @@ def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]: explanation = _compare_eq_dict(left, right, verbose) elif verbose > 0: explanation = _compare_eq_verbose(left, right) + if isiterable(left) and isiterable(right): expl = _compare_eq_iterable(left, right, verbose) explanation.extend(expl) + return explanation diff --git a/src/_pytest/python_api.py b/src/_pytest/python_api.py index 69f01619a..946d15b31 100644 --- a/src/_pytest/python_api.py +++ b/src/_pytest/python_api.py @@ -1,7 +1,5 @@ import math import pprint -from collections.abc import Iterable -from collections.abc import Mapping from collections.abc import Sized from decimal import Decimal from numbers import Complex @@ -10,9 +8,13 @@ from typing import Any from typing import Callable from typing import cast from typing import Generic +from typing import Iterable +from typing import List +from typing import Mapping from typing import Optional from typing import overload from typing import Pattern +from typing import Sequence from typing import Tuple from typing import Type from typing import TYPE_CHECKING @@ -38,6 +40,32 @@ def _non_numeric_type_error(value, at: Optional[str]) -> TypeError: ) +def _compare_approx( + full_object: object, + message_data: Sequence[Tuple[str, str, str]], + number_of_elements: int, + different_ids: Sequence[object], + max_abs_diff: float, + max_rel_diff: float, +) -> List[str]: + message_list = list(message_data) + message_list.insert(0, ("Index", "Obtained", "Expected")) + max_sizes = [0, 0, 0] + for index, obtained, expected in message_list: + max_sizes[0] = max(max_sizes[0], len(index)) + max_sizes[1] = max(max_sizes[1], len(obtained)) + max_sizes[2] = max(max_sizes[2], len(expected)) + explanation = [ + f"comparison failed. Mismatched elements: {len(different_ids)} / {number_of_elements}:", + f"Max absolute difference: {max_abs_diff}", + f"Max relative difference: {max_rel_diff}", + ] + [ + f"{indexes:<{max_sizes[0]}} | {obtained:<{max_sizes[1]}} | {expected:<{max_sizes[2]}}" + for indexes, obtained, expected in message_list + ] + return explanation + + # builtin pytest.approx helper @@ -60,6 +88,13 @@ class ApproxBase: def __repr__(self) -> str: raise NotImplementedError + def _repr_compare(self, other_side: Any) -> List[str]: + return [ + "comparison failed", + f"Obtained: {other_side}", + f"Expected: {self}", + ] + def __eq__(self, actual) -> bool: return all( a == self._approx_scalar(x) for a, x in self._yield_comparisons(actual) @@ -107,6 +142,66 @@ class ApproxNumpy(ApproxBase): list_scalars = _recursive_list_map(self._approx_scalar, self.expected.tolist()) return f"approx({list_scalars!r})" + def _repr_compare(self, other_side: "ndarray") -> List[str]: + import itertools + import math + + def get_value_from_nested_list( + nested_list: List[Any], nd_index: Tuple[Any, ...] + ) -> Any: + """ + Helper function to get the value out of a nested list, given an n-dimensional index. + This mimics numpy's indexing, but for raw nested python lists. + """ + value: Any = nested_list + for i in nd_index: + value = value[i] + return value + + np_array_shape = self.expected.shape + approx_side_as_list = _recursive_list_map( + self._approx_scalar, self.expected.tolist() + ) + + if np_array_shape != other_side.shape: + return [ + "Impossible to compare arrays with different shapes.", + f"Shapes: {np_array_shape} and {other_side.shape}", + ] + + number_of_elements = self.expected.size + max_abs_diff = -math.inf + max_rel_diff = -math.inf + different_ids = [] + for index in itertools.product(*(range(i) for i in np_array_shape)): + approx_value = get_value_from_nested_list(approx_side_as_list, index) + other_value = get_value_from_nested_list(other_side, index) + if approx_value != other_value: + abs_diff = abs(approx_value.expected - other_value) + max_abs_diff = max(max_abs_diff, abs_diff) + if other_value == 0.0: + max_rel_diff = math.inf + else: + max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value)) + different_ids.append(index) + + message_data = [ + ( + str(index), + str(get_value_from_nested_list(other_side, index)), + str(get_value_from_nested_list(approx_side_as_list, index)), + ) + for index in different_ids + ] + return _compare_approx( + self.expected, + message_data, + number_of_elements, + different_ids, + max_abs_diff, + max_rel_diff, + ) + def __eq__(self, actual) -> bool: import numpy as np @@ -147,6 +242,44 @@ class ApproxMapping(ApproxBase): {k: self._approx_scalar(v) for k, v in self.expected.items()} ) + def _repr_compare(self, other_side: Mapping[object, float]) -> List[str]: + import math + + approx_side_as_map = { + k: self._approx_scalar(v) for k, v in self.expected.items() + } + + number_of_elements = len(approx_side_as_map) + max_abs_diff = -math.inf + max_rel_diff = -math.inf + different_ids = [] + for (approx_key, approx_value), other_value in zip( + approx_side_as_map.items(), other_side.values() + ): + if approx_value != other_value: + max_abs_diff = max( + max_abs_diff, abs(approx_value.expected - other_value) + ) + max_rel_diff = max( + max_rel_diff, + abs((approx_value.expected - other_value) / approx_value.expected), + ) + different_ids.append(approx_key) + + message_data = [ + (str(key), str(other_side[key]), str(approx_side_as_map[key])) + for key in different_ids + ] + + return _compare_approx( + self.expected, + message_data, + number_of_elements, + different_ids, + max_abs_diff, + max_rel_diff, + ) + def __eq__(self, actual) -> bool: try: if set(actual.keys()) != set(self.expected.keys()): @@ -179,6 +312,48 @@ class ApproxSequencelike(ApproxBase): seq_type(self._approx_scalar(x) for x in self.expected) ) + def _repr_compare(self, other_side: Sequence[float]) -> List[str]: + import math + import numpy as np + + if len(self.expected) != len(other_side): + return [ + "Impossible to compare lists with different sizes.", + f"Lengths: {len(self.expected)} and {len(other_side)}", + ] + + approx_side_as_map = _recursive_list_map(self._approx_scalar, self.expected) + + number_of_elements = len(approx_side_as_map) + max_abs_diff = -math.inf + max_rel_diff = -math.inf + different_ids = [] + for i, (approx_value, other_value) in enumerate( + zip(approx_side_as_map, other_side) + ): + if approx_value != other_value: + abs_diff = abs(approx_value.expected - other_value) + max_abs_diff = max(max_abs_diff, abs_diff) + if other_value == 0.0: + max_rel_diff = np.inf + else: + max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value)) + different_ids.append(i) + + message_data = [ + (str(i), str(other_side[i]), str(approx_side_as_map[i])) + for i in different_ids + ] + + return _compare_approx( + self.expected, + message_data, + number_of_elements, + different_ids, + max_abs_diff, + max_rel_diff, + ) + def __eq__(self, actual) -> bool: try: if len(actual) != len(self.expected): @@ -212,7 +387,6 @@ class ApproxScalar(ApproxBase): For example, ``1.0 ± 1e-6``, ``(3+4j) ± 5e-6 ∠ ±180°``. """ - # Don't show a tolerance for values that aren't compared using # tolerances, i.e. non-numerics and infinities. Need to call abs to # handle complex numbers, e.g. (inf + 1j). diff --git a/testing/python/approx.py b/testing/python/approx.py index cb1d9b7c8..d4a152f3e 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -1,5 +1,6 @@ import operator import sys +from contextlib import contextmanager from decimal import Decimal from fractions import Fraction from operator import eq @@ -43,7 +44,236 @@ def mocked_doctest_runner(monkeypatch): return MyDocTestRunner() +@contextmanager +def temporary_verbosity(config, verbosity=0): + original_verbosity = config.getoption("verbose") + config.option.verbose = verbosity + try: + yield + finally: + config.option.verbose = original_verbosity + + +@pytest.fixture +def assert_approx_raises_regex(pytestconfig): + def do_assert(lhs, rhs, expected_message, verbosity_level=0): + import re + + with temporary_verbosity(pytestconfig, verbosity_level): + with pytest.raises(AssertionError) as e: + assert lhs == approx(rhs) + + nl = "\n" + obtained_message = str(e.value).splitlines()[1:] + assert len(obtained_message) == len(expected_message), ( + "Regex message length doesn't match obtained.\n" + "Obtained:\n" + f"{nl.join(obtained_message)}\n\n" + "Expected regex:\n" + f"{nl.join(expected_message)}\n\n" + ) + + for i, (obtained_line, expected_line) in enumerate( + zip(obtained_message, expected_message) + ): + regex = re.compile(expected_line) + assert regex.match(obtained_line) is not None, ( + "Unexpected error message:\n" + f"{nl.join(obtained_message)}\n\n" + "Did not match regex:\n" + f"{nl.join(expected_message)}\n\n" + f"With verbosity level = {verbosity_level}, on line {i}" + ) + + return do_assert + + +SOME_FLOAT = r"[+-]?([0-9]*[.])?[0-9]+\s*" +SOME_INT = r"[0-9]+\s*" + + class TestApprox: + def test_error_messages(self, assert_approx_raises_regex): + np = pytest.importorskip("numpy") + + assert_approx_raises_regex( + 2.0, + 1.0, + [ + " comparison failed", + f" Obtained: {SOME_FLOAT}", + f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + assert_approx_raises_regex( + {"a": 1.0, "b": 1000.0, "c": 1000000.0}, + { + "a": 2.0, + "b": 1000.0, + "c": 3000000.0, + }, + [ + r" comparison failed. Mismatched elements: 2 / 3:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected ", + rf" a \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" c \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + assert_approx_raises_regex( + [1.0, 2.0, 3.0, 4.0], + [1.0, 3.0, 3.0, 5.0], + [ + r" comparison failed. Mismatched elements: 2 / 4:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected ", + rf" 1 \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" 3 \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + a = np.linspace(0, 100, 20) + b = np.linspace(0, 100, 20) + a[10] += 0.5 + assert_approx_raises_regex( + a, + b, + [ + r" comparison failed. Mismatched elements: 1 / 20:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected", + rf" \(10,\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + assert_approx_raises_regex( + np.array( + [ + [[1.1987311, 12412342.3], [3.214143244, 1423412423415.677]], + [[1, 2], [3, 219371297321973]], + ] + ), + np.array( + [ + [[1.12313, 12412342.3], [3.214143244, 534523542345.677]], + [[1, 2], [3, 7]], + ] + ), + [ + r" comparison failed. Mismatched elements: 3 / 8:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index\s+\| Obtained\s+\| Expected\s+", + rf" \(0, 0, 0\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" \(0, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" \(1, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + # Specific test for comparison with 0.0 (relative diff will be 'inf') + assert_approx_raises_regex( + [0.0], + [1.0], + [ + r" comparison failed. Mismatched elements: 1 / 1:", + rf" Max absolute difference: {SOME_FLOAT}", + r" Max relative difference: inf", + r" Index \| Obtained\s+\| Expected ", + rf"\s*0\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + assert_approx_raises_regex( + np.array([0.0]), + np.array([1.0]), + [ + r" comparison failed. Mismatched elements: 1 / 1:", + rf" Max absolute difference: {SOME_FLOAT}", + r" Max relative difference: inf", + r" Index \| Obtained\s+\| Expected ", + rf"\s*\(0,\)\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + def test_error_messages_invalid_args(self, assert_approx_raises_regex): + np = pytest.importorskip("numpy") + with pytest.raises(AssertionError) as e: + assert np.array([[1.2, 3.4], [4.0, 5.0]]) == pytest.approx( + np.array([[4.0], [5.0]]) + ) + message = "\n".join(str(e.value).split("\n")[1:]) + assert message == "\n".join( + [ + " Impossible to compare arrays with different shapes.", + " Shapes: (2, 1) and (2, 2)", + ] + ) + + with pytest.raises(AssertionError) as e: + assert [1.0, 2.0, 3.0] == pytest.approx([4.0, 5.0]) + message = "\n".join(str(e.value).split("\n")[1:]) + assert message == "\n".join( + [ + " Impossible to compare lists with different sizes.", + " Lengths: 2 and 3", + ] + ) + + def test_error_messages_with_different_verbosity(self, assert_approx_raises_regex): + np = pytest.importorskip("numpy") + for v in [0, 1, 2]: + # Verbosity level doesn't affect the error message for scalars + assert_approx_raises_regex( + 2.0, + 1.0, + [ + " comparison failed", + f" Obtained: {SOME_FLOAT}", + f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}", + ], + verbosity_level=v, + ) + + a = np.linspace(1, 101, 20) + b = np.linspace(2, 102, 20) + assert_approx_raises_regex( + a, + b, + [ + r" comparison failed. Mismatched elements: 20 / 20:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected", + rf" \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" \(2,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}...", + "", + rf"\s*...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show", + ], + verbosity_level=0, + ) + + assert_approx_raises_regex( + a, + b, + [ + r" comparison failed. Mismatched elements: 20 / 20:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected", + ] + + [ + rf" \({i},\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}" + for i in range(20) + ], + verbosity_level=2, + ) + def test_repr_string(self): assert repr(approx(1.0)) == "1.0 ± 1.0e-06" assert repr(approx([1.0, 2.0])) == "approx([1.0 ± 1.0e-06, 2.0 ± 2.0e-06])"