Improve pytest.approx error messages readability (Pull request) (#8429)

Improve pytest.approx error messages readability (Pull request)

changelog/8335.improvement.rst

@@ -0,0 +1,10 @@
+Improved :func:`pytest.approx` assertion messages for sequences of numbers.
+
+The assertion messages now dumps a table with the index and the error of each diff.
+Example::
+
+    >       assert [1, 2, 3, 4] == pytest.approx([1, 3, 3, 5])
+    E       assert comparison failed for 2 values:
+    E         Index | Obtained | Expected
+    E         1     | 2        | 3 +- 3.0e-06
+    E         3     | 4        | 5 +- 5.0e-06

src/_pytest/assertion/util.py

@@ -180,7 +180,15 @@ def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]:
     if istext(left) and istext(right):
         explanation = _diff_text(left, right, verbose)
     else:
-        if type(left) == type(right) and (
+        from _pytest.python_api import ApproxBase
+
+        if isinstance(left, ApproxBase) or isinstance(right, ApproxBase):
+            # Although the common order should be obtained == expected, this ensures both ways
+            approx_side = left if isinstance(left, ApproxBase) else right
+            other_side = right if isinstance(left, ApproxBase) else left
+
+            explanation = approx_side._repr_compare(other_side)
+        elif type(left) == type(right) and (
             isdatacls(left) or isattrs(left) or isnamedtuple(left)
         ):
             # Note: unlike dataclasses/attrs, namedtuples compare only the
@@ -196,9 +204,11 @@ def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]:
             explanation = _compare_eq_dict(left, right, verbose)
         elif verbose > 0:
             explanation = _compare_eq_verbose(left, right)
+
         if isiterable(left) and isiterable(right):
             expl = _compare_eq_iterable(left, right, verbose)
             explanation.extend(expl)
+
     return explanation
 
 

src/_pytest/python_api.py

@@ -1,7 +1,5 @@
 import math
 import pprint
-from collections.abc import Iterable
-from collections.abc import Mapping
 from collections.abc import Sized
 from decimal import Decimal
 from numbers import Complex
@@ -10,9 +8,13 @@ from typing import Any
 from typing import Callable
 from typing import cast
 from typing import Generic
+from typing import Iterable
+from typing import List
+from typing import Mapping
 from typing import Optional
 from typing import overload
 from typing import Pattern
+from typing import Sequence
 from typing import Tuple
 from typing import Type
 from typing import TYPE_CHECKING
@@ -38,6 +40,32 @@ def _non_numeric_type_error(value, at: Optional[str]) -> TypeError:
     )
 
 
+def _compare_approx(
+    full_object: object,
+    message_data: Sequence[Tuple[str, str, str]],
+    number_of_elements: int,
+    different_ids: Sequence[object],
+    max_abs_diff: float,
+    max_rel_diff: float,
+) -> List[str]:
+    message_list = list(message_data)
+    message_list.insert(0, ("Index", "Obtained", "Expected"))
+    max_sizes = [0, 0, 0]
+    for index, obtained, expected in message_list:
+        max_sizes[0] = max(max_sizes[0], len(index))
+        max_sizes[1] = max(max_sizes[1], len(obtained))
+        max_sizes[2] = max(max_sizes[2], len(expected))
+    explanation = [
+        f"comparison failed. Mismatched elements: {len(different_ids)} / {number_of_elements}:",
+        f"Max absolute difference: {max_abs_diff}",
+        f"Max relative difference: {max_rel_diff}",
+    ] + [
+        f"{indexes:<{max_sizes[0]}} | {obtained:<{max_sizes[1]}} | {expected:<{max_sizes[2]}}"
+        for indexes, obtained, expected in message_list
+    ]
+    return explanation
+
+
 # builtin pytest.approx helper
 
 
@@ -60,6 +88,13 @@ class ApproxBase:
     def __repr__(self) -> str:
         raise NotImplementedError
 
+    def _repr_compare(self, other_side: Any) -> List[str]:
+        return [
+            "comparison failed",
+            f"Obtained: {other_side}",
+            f"Expected: {self}",
+        ]
+
     def __eq__(self, actual) -> bool:
         return all(
             a == self._approx_scalar(x) for a, x in self._yield_comparisons(actual)
@@ -107,6 +142,66 @@ class ApproxNumpy(ApproxBase):
         list_scalars = _recursive_list_map(self._approx_scalar, self.expected.tolist())
         return f"approx({list_scalars!r})"
 
+    def _repr_compare(self, other_side: "ndarray") -> List[str]:
+        import itertools
+        import math
+
+        def get_value_from_nested_list(
+            nested_list: List[Any], nd_index: Tuple[Any, ...]
+        ) -> Any:
+            """
+            Helper function to get the value out of a nested list, given an n-dimensional index.
+            This mimics numpy's indexing, but for raw nested python lists.
+            """
+            value: Any = nested_list
+            for i in nd_index:
+                value = value[i]
+            return value
+
+        np_array_shape = self.expected.shape
+        approx_side_as_list = _recursive_list_map(
+            self._approx_scalar, self.expected.tolist()
+        )
+
+        if np_array_shape != other_side.shape:
+            return [
+                "Impossible to compare arrays with different shapes.",
+                f"Shapes: {np_array_shape} and {other_side.shape}",
+            ]
+
+        number_of_elements = self.expected.size
+        max_abs_diff = -math.inf
+        max_rel_diff = -math.inf
+        different_ids = []
+        for index in itertools.product(*(range(i) for i in np_array_shape)):
+            approx_value = get_value_from_nested_list(approx_side_as_list, index)
+            other_value = get_value_from_nested_list(other_side, index)
+            if approx_value != other_value:
+                abs_diff = abs(approx_value.expected - other_value)
+                max_abs_diff = max(max_abs_diff, abs_diff)
+                if other_value == 0.0:
+                    max_rel_diff = math.inf
+                else:
+                    max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value))
+                different_ids.append(index)
+
+        message_data = [
+            (
+                str(index),
+                str(get_value_from_nested_list(other_side, index)),
+                str(get_value_from_nested_list(approx_side_as_list, index)),
+            )
+            for index in different_ids
+        ]
+        return _compare_approx(
+            self.expected,
+            message_data,
+            number_of_elements,
+            different_ids,
+            max_abs_diff,
+            max_rel_diff,
+        )
+
     def __eq__(self, actual) -> bool:
         import numpy as np
 
@@ -147,6 +242,44 @@ class ApproxMapping(ApproxBase):
             {k: self._approx_scalar(v) for k, v in self.expected.items()}
         )
 
+    def _repr_compare(self, other_side: Mapping[object, float]) -> List[str]:
+        import math
+
+        approx_side_as_map = {
+            k: self._approx_scalar(v) for k, v in self.expected.items()
+        }
+
+        number_of_elements = len(approx_side_as_map)
+        max_abs_diff = -math.inf
+        max_rel_diff = -math.inf
+        different_ids = []
+        for (approx_key, approx_value), other_value in zip(
+            approx_side_as_map.items(), other_side.values()
+        ):
+            if approx_value != other_value:
+                max_abs_diff = max(
+                    max_abs_diff, abs(approx_value.expected - other_value)
+                )
+                max_rel_diff = max(
+                    max_rel_diff,
+                    abs((approx_value.expected - other_value) / approx_value.expected),
+                )
+                different_ids.append(approx_key)
+
+        message_data = [
+            (str(key), str(other_side[key]), str(approx_side_as_map[key]))
+            for key in different_ids
+        ]
+
+        return _compare_approx(
+            self.expected,
+            message_data,
+            number_of_elements,
+            different_ids,
+            max_abs_diff,
+            max_rel_diff,
+        )
+
     def __eq__(self, actual) -> bool:
         try:
             if set(actual.keys()) != set(self.expected.keys()):
@@ -179,6 +312,48 @@ class ApproxSequencelike(ApproxBase):
             seq_type(self._approx_scalar(x) for x in self.expected)
         )
 
+    def _repr_compare(self, other_side: Sequence[float]) -> List[str]:
+        import math
+        import numpy as np
+
+        if len(self.expected) != len(other_side):
+            return [
+                "Impossible to compare lists with different sizes.",
+                f"Lengths: {len(self.expected)} and {len(other_side)}",
+            ]
+
+        approx_side_as_map = _recursive_list_map(self._approx_scalar, self.expected)
+
+        number_of_elements = len(approx_side_as_map)
+        max_abs_diff = -math.inf
+        max_rel_diff = -math.inf
+        different_ids = []
+        for i, (approx_value, other_value) in enumerate(
+            zip(approx_side_as_map, other_side)
+        ):
+            if approx_value != other_value:
+                abs_diff = abs(approx_value.expected - other_value)
+                max_abs_diff = max(max_abs_diff, abs_diff)
+                if other_value == 0.0:
+                    max_rel_diff = np.inf
+                else:
+                    max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value))
+                different_ids.append(i)
+
+        message_data = [
+            (str(i), str(other_side[i]), str(approx_side_as_map[i]))
+            for i in different_ids
+        ]
+
+        return _compare_approx(
+            self.expected,
+            message_data,
+            number_of_elements,
+            different_ids,
+            max_abs_diff,
+            max_rel_diff,
+        )
+
     def __eq__(self, actual) -> bool:
         try:
             if len(actual) != len(self.expected):
@@ -212,7 +387,6 @@ class ApproxScalar(ApproxBase):
 
         For example, ``1.0 ± 1e-6``, ``(3+4j) ± 5e-6 ∠ ±180°``.
         """
-
         # Don't show a tolerance for values that aren't compared using
         # tolerances, i.e. non-numerics and infinities. Need to call abs to
         # handle complex numbers, e.g. (inf + 1j).

testing/python/approx.py

@@ -1,5 +1,6 @@
 import operator
 import sys
+from contextlib import contextmanager
 from decimal import Decimal
 from fractions import Fraction
 from operator import eq
@@ -43,7 +44,236 @@ def mocked_doctest_runner(monkeypatch):
     return MyDocTestRunner()
 
 
+@contextmanager
+def temporary_verbosity(config, verbosity=0):
+    original_verbosity = config.getoption("verbose")
+    config.option.verbose = verbosity
+    try:
+        yield
+    finally:
+        config.option.verbose = original_verbosity
+
+
+@pytest.fixture
+def assert_approx_raises_regex(pytestconfig):
+    def do_assert(lhs, rhs, expected_message, verbosity_level=0):
+        import re
+
+        with temporary_verbosity(pytestconfig, verbosity_level):
+            with pytest.raises(AssertionError) as e:
+                assert lhs == approx(rhs)
+
+        nl = "\n"
+        obtained_message = str(e.value).splitlines()[1:]
+        assert len(obtained_message) == len(expected_message), (
+            "Regex message length doesn't match obtained.\n"
+            "Obtained:\n"
+            f"{nl.join(obtained_message)}\n\n"
+            "Expected regex:\n"
+            f"{nl.join(expected_message)}\n\n"
+        )
+
+        for i, (obtained_line, expected_line) in enumerate(
+            zip(obtained_message, expected_message)
+        ):
+            regex = re.compile(expected_line)
+            assert regex.match(obtained_line) is not None, (
+                "Unexpected error message:\n"
+                f"{nl.join(obtained_message)}\n\n"
+                "Did not match regex:\n"
+                f"{nl.join(expected_message)}\n\n"
+                f"With verbosity level = {verbosity_level}, on line {i}"
+            )
+
+    return do_assert
+
+
+SOME_FLOAT = r"[+-]?([0-9]*[.])?[0-9]+\s*"
+SOME_INT = r"[0-9]+\s*"
+
+
 class TestApprox:
+    def test_error_messages(self, assert_approx_raises_regex):
+        np = pytest.importorskip("numpy")
+
+        assert_approx_raises_regex(
+            2.0,
+            1.0,
+            [
+                "  comparison failed",
+                f"  Obtained: {SOME_FLOAT}",
+                f"  Expected: {SOME_FLOAT} ± {SOME_FLOAT}",
+            ],
+        )
+
+        assert_approx_raises_regex(
+            {"a": 1.0, "b": 1000.0, "c": 1000000.0},
+            {
+                "a": 2.0,
+                "b": 1000.0,
+                "c": 3000000.0,
+            },
+            [
+                r"  comparison failed. Mismatched elements: 2 / 3:",
+                rf"  Max absolute difference: {SOME_FLOAT}",
+                rf"  Max relative difference: {SOME_FLOAT}",
+                r"  Index \| Obtained\s+\| Expected           ",
+                rf"  a     \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+                rf"  c     \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+            ],
+        )
+
+        assert_approx_raises_regex(
+            [1.0, 2.0, 3.0, 4.0],
+            [1.0, 3.0, 3.0, 5.0],
+            [
+                r"  comparison failed. Mismatched elements: 2 / 4:",
+                rf"  Max absolute difference: {SOME_FLOAT}",
+                rf"  Max relative difference: {SOME_FLOAT}",
+                r"  Index \| Obtained\s+\| Expected   ",
+                rf"  1     \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+                rf"  3     \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+            ],
+        )
+
+        a = np.linspace(0, 100, 20)
+        b = np.linspace(0, 100, 20)
+        a[10] += 0.5
+        assert_approx_raises_regex(
+            a,
+            b,
+            [
+                r"  comparison failed. Mismatched elements: 1 / 20:",
+                rf"  Max absolute difference: {SOME_FLOAT}",
+                rf"  Max relative difference: {SOME_FLOAT}",
+                r"  Index \| Obtained\s+\| Expected",
+                rf"  \(10,\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+            ],
+        )
+
+        assert_approx_raises_regex(
+            np.array(
+                [
+                    [[1.1987311, 12412342.3], [3.214143244, 1423412423415.677]],
+                    [[1, 2], [3, 219371297321973]],
+                ]
+            ),
+            np.array(
+                [
+                    [[1.12313, 12412342.3], [3.214143244, 534523542345.677]],
+                    [[1, 2], [3, 7]],
+                ]
+            ),
+            [
+                r"  comparison failed. Mismatched elements: 3 / 8:",
+                rf"  Max absolute difference: {SOME_FLOAT}",
+                rf"  Max relative difference: {SOME_FLOAT}",
+                r"  Index\s+\| Obtained\s+\| Expected\s+",
+                rf"  \(0, 0, 0\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+                rf"  \(0, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+                rf"  \(1, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+            ],
+        )
+
+        # Specific test for comparison with 0.0 (relative diff will be 'inf')
+        assert_approx_raises_regex(
+            [0.0],
+            [1.0],
+            [
+                r"  comparison failed. Mismatched elements: 1 / 1:",
+                rf"  Max absolute difference: {SOME_FLOAT}",
+                r"  Max relative difference: inf",
+                r"  Index \| Obtained\s+\| Expected   ",
+                rf"\s*0\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+            ],
+        )
+
+        assert_approx_raises_regex(
+            np.array([0.0]),
+            np.array([1.0]),
+            [
+                r"  comparison failed. Mismatched elements: 1 / 1:",
+                rf"  Max absolute difference: {SOME_FLOAT}",
+                r"  Max relative difference: inf",
+                r"  Index \| Obtained\s+\| Expected   ",
+                rf"\s*\(0,\)\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+            ],
+        )
+
+    def test_error_messages_invalid_args(self, assert_approx_raises_regex):
+        np = pytest.importorskip("numpy")
+        with pytest.raises(AssertionError) as e:
+            assert np.array([[1.2, 3.4], [4.0, 5.0]]) == pytest.approx(
+                np.array([[4.0], [5.0]])
+            )
+        message = "\n".join(str(e.value).split("\n")[1:])
+        assert message == "\n".join(
+            [
+                "  Impossible to compare arrays with different shapes.",
+                "  Shapes: (2, 1) and (2, 2)",
+            ]
+        )
+
+        with pytest.raises(AssertionError) as e:
+            assert [1.0, 2.0, 3.0] == pytest.approx([4.0, 5.0])
+        message = "\n".join(str(e.value).split("\n")[1:])
+        assert message == "\n".join(
+            [
+                "  Impossible to compare lists with different sizes.",
+                "  Lengths: 2 and 3",
+            ]
+        )
+
+    def test_error_messages_with_different_verbosity(self, assert_approx_raises_regex):
+        np = pytest.importorskip("numpy")
+        for v in [0, 1, 2]:
+            # Verbosity level doesn't affect the error message for scalars
+            assert_approx_raises_regex(
+                2.0,
+                1.0,
+                [
+                    "  comparison failed",
+                    f"  Obtained: {SOME_FLOAT}",
+                    f"  Expected: {SOME_FLOAT} ± {SOME_FLOAT}",
+                ],
+                verbosity_level=v,
+            )
+
+        a = np.linspace(1, 101, 20)
+        b = np.linspace(2, 102, 20)
+        assert_approx_raises_regex(
+            a,
+            b,
+            [
+                r"  comparison failed. Mismatched elements: 20 / 20:",
+                rf"  Max absolute difference: {SOME_FLOAT}",
+                rf"  Max relative difference: {SOME_FLOAT}",
+                r"  Index \| Obtained\s+\| Expected",
+                rf"  \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+                rf"  \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
+                rf"  \(2,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}...",
+                "",
+                rf"\s*...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show",
+            ],
+            verbosity_level=0,
+        )
+
+        assert_approx_raises_regex(
+            a,
+            b,
+            [
+                r"  comparison failed. Mismatched elements: 20 / 20:",
+                rf"  Max absolute difference: {SOME_FLOAT}",
+                rf"  Max relative difference: {SOME_FLOAT}",
+                r"  Index \| Obtained\s+\| Expected",
+            ]
+            + [
+                rf"  \({i},\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}"
+                for i in range(20)
+            ],
+            verbosity_level=2,
+        )
+
     def test_repr_string(self):
         assert repr(approx(1.0)) == "1.0 ± 1.0e-06"
         assert repr(approx([1.0, 2.0])) == "approx([1.0 ± 1.0e-06, 2.0 ± 2.0e-06])"