ApproxScalar: Reduce float rounding errors

This commit reduces float rounding errors when comparing numbers with
a high (and non-float representable) mantissa and a low tolerance.

This came up when comparing geo coordinates with pytest.approx, e.g.:
>>> tolerance = 5e-7
>>> expected = 12.3456785
>>> actual = 12.345678
>>> abs(expected - actual) <= tolerance
False
>>> expected - tolerance <= actual <= expected + tolerance
True

versus

>>> expected = 51.500744
>>> actual = 51.5007435
>>> abs(expected - actual) <= tolerance
True
>>> expected - tolerance <= actual <= expected + tolerance
True

src/_pytest/python_api.py

@@ -3,6 +3,7 @@ import pprint
 from collections.abc import Sized
 from decimal import Decimal
 from numbers import Complex
+from numbers import Real
 from types import TracebackType
 from typing import Any
 from typing import Callable
@@ -453,7 +454,16 @@ class ApproxScalar(ApproxBase):
             return False
 
         # Return true if the two numbers are within the tolerance.
-        result: bool = abs(self.expected - actual) <= self.tolerance
+        result: bool
+        if isinstance(self.expected, Real) and isinstance(actual, Real):
+            # Use three-way comparison instead of abs() to reduce float rounding errors.
+            result = (
+                self.expected - self.tolerance
+                <= actual
+                <= self.expected + self.tolerance
+            )
+        else:
+            result = abs(self.expected - actual) <= self.tolerance
         return result
 
     # Ignore type because of https://github.com/python/mypy/issues/4266.

testing/python/approx.py

@@ -414,6 +414,16 @@ class TestApprox:
         # than have a small amount of floating-point error.
         assert 0.1 + 0.2 == approx(0.3)
 
+    @pytest.mark.parametrize(
+        ("expected", "actual", "abs_tolerance"),
+        [
+            (12.3456785, 12.345678, 5e-7),
+            (51.500744, 51.5007435, 5e-7),
+        ],
+    )
+    def test_float_rounding(self, expected, actual, abs_tolerance):
+        assert actual == pytest.approx(expected, abs=abs_tolerance)
+
     def test_default_tolerances(self):
         # This tests the defaults as they are currently set.  If you change the
         # defaults, this test will fail but you should feel free to change it.