doctest: Add +NUMBER option to ignore irrelevant floating-point differences

When enabled, floating-point numbers only need to match as far as the
precision you have written in the expected doctest output. This avoids
false positives caused by limited floating-point precision, like this:

    Expected:
        0.233
    Got:
        0.23300000000000001

This is inspired by Sébastien Boisgérault's [numtest] but the
implementation is a bit different:

* This implementation edits the literals that are in the "got"
  string (the actual output from the expression being tested), and then
  proceeds to compare the strings literally. This is similar to pytest's
  existing ALLOW_UNICODE and ALLOW_BYTES implementation.

* This implementation only compares floats against floats, not ints
  against floats. That is, the following doctest will fail with pytest
  whereas it would pass with numtest:

      >>> math.py  # doctest: +NUMBER
      3

  This behaviour should be less surprising (less false negatives) when
  you enable NUMBER globally in pytest.ini.

Advantages of this implementation compared to numtest:

* Doesn't require `import numtest` at the top level of the file.
* Works with pytest (if you try to use pytest & numtest together, pytest
  raises "TypeError: unbound method check_output() must be called with
  NumTestOutputChecker instance as first argument (got
  LiteralsOutputChecker instance instead)").
* Works with Python 3.

[numtest]: https://github.com/boisgera/numtest

doc/en/doctest.rst

@@ -115,15 +115,36 @@ lengthy exception stack traces you can just write:
     [pytest]
     doctest_optionflags= NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL
 
-pytest also introduces new options to allow doctests to run in Python 2 and
+pytest also introduces new options:
-Python 3 unchanged:
 
 * ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode
-  strings in expected doctest output.
+  strings in expected doctest output. This allows doctests to run in Python 2
+  and Python 3 unchanged.
 
-* ``ALLOW_BYTES``: when enabled, the ``b`` prefix is stripped from byte strings
+* ``ALLOW_BYTES``: similarly, the ``b`` prefix is stripped from byte strings
   in expected doctest output.
 
+* ``NUMBER``: when enabled, floating-point numbers only need to match as far as
+  the precision you have written in the expected doctest output. For example,
+  the following output would only need to match to 2 decimal places::
+
+      >>> math.pi
+      3.14
+
+  If you wrote ``3.1416`` then the actual output would need to match to 4
+  decimal places; and so on.
+
+  This avoids false positives caused by limited floating-point precision, like
+  this::
+
+      Expected:
+          0.233
+      Got:
+          0.23300000000000001
+
+  ``NUMBER`` also supports lists of floating-point numbers -- in fact, it
+  supports floating-point numbers appearing anywhere in the output.
+
 Alternatively, options can be enabled by an inline comment in the doc test
 itself:
 

src/_pytest/doctest.py

@@ -13,6 +13,7 @@ from _pytest._code.code import TerminalRepr
 from _pytest.compat import safe_getattr
 from _pytest.fixtures import FixtureRequest
 from _pytest.outcomes import Skipped
+from _pytest.python_api import approx
 from _pytest.warning_types import PytestWarning
 
 DOCTEST_REPORT_CHOICE_NONE = "none"
@@ -286,6 +287,7 @@ def _get_flag_lookup():
         COMPARISON_FLAGS=doctest.COMPARISON_FLAGS,
         ALLOW_UNICODE=_get_allow_unicode_flag(),
         ALLOW_BYTES=_get_allow_bytes_flag(),
+        NUMBER=_get_number_flag(),
     )
 
 
@@ -453,10 +455,15 @@ def _setup_fixtures(doctest_item):
 
 def _get_checker():
     """
-    Returns a doctest.OutputChecker subclass that takes in account the
+    Returns a doctest.OutputChecker subclass that supports some
-    ALLOW_UNICODE option to ignore u'' prefixes in strings and ALLOW_BYTES
+    additional options:
-    to strip b'' prefixes.
+
-    Useful when the same doctest should run in Python 2 and Python 3.
+    * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b''
+      prefixes (respectively) in string literals. Useful when the same
+      doctest should run in Python 2 and Python 3.
+
+    * NUMBER to ignore floating-point differences smaller than the
+      precision of the literal number in the doctest.
 
     An inner class is used to avoid importing "doctest" at the module
     level.
@@ -469,26 +476,46 @@ def _get_checker():
 
     class LiteralsOutputChecker(doctest.OutputChecker):
         """
-        Copied from doctest_nose_plugin.py from the nltk project:
+        Based on doctest_nose_plugin.py from the nltk project
-            https://github.com/nltk/nltk
+        (https://github.com/nltk/nltk) and on the "numtest" doctest extension
-
+        by Sebastien Boisgerault (https://github.com/boisgera/numtest).
-        Further extended to also support byte literals.
         """
 
         _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE)
         _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE)
+        _number_re = re.compile(
+            r"""
+            (?P<number>
+              (?P<mantissa>
+                (?P<integer1> [+-]?\d*)\.(?P<fraction>\d+)
+                |
+                (?P<integer2> [+-]?\d+)\.
+              )
+              (?:
+                [Ee]
+                (?P<exponent1> [+-]?\d+)
+              )?
+              |
+              (?P<integer3> [+-]?\d+)
+              (?:
+                [Ee]
+                (?P<exponent2> [+-]?\d+)
+              )
+            )
+            """,
+            re.VERBOSE,
+        )
 
         def check_output(self, want, got, optionflags):
-            res = doctest.OutputChecker.check_output(self, want, got, optionflags)
+            if doctest.OutputChecker.check_output(self, want, got, optionflags):
-            if res:
                 return True
 
             allow_unicode = optionflags & _get_allow_unicode_flag()
             allow_bytes = optionflags & _get_allow_bytes_flag()
-            if not allow_unicode and not allow_bytes:
+            allow_number = optionflags & _get_number_flag()
-                return False
 
-            else:  # pragma: no cover
+            if not allow_unicode and not allow_bytes and not allow_number:
+                return False
 
             def remove_prefixes(regex, txt):
                 return re.sub(regex, r"\1\2", txt)
@@ -496,11 +523,42 @@ def _get_checker():
             if allow_unicode:
                 want = remove_prefixes(self._unicode_literal_re, want)
                 got = remove_prefixes(self._unicode_literal_re, got)
+
             if allow_bytes:
                 want = remove_prefixes(self._bytes_literal_re, want)
                 got = remove_prefixes(self._bytes_literal_re, got)
-                res = doctest.OutputChecker.check_output(self, want, got, optionflags)
+
-                return res
+            if allow_number:
+                got = self._remove_unwanted_precision(want, got)
+
+            return doctest.OutputChecker.check_output(self, want, got, optionflags)
+
+        def _remove_unwanted_precision(self, want, got):
+            wants = list(self._number_re.finditer(want))
+            gots = list(self._number_re.finditer(got))
+            if len(wants) != len(gots):
+                return got
+            offset = 0
+            for w, g in zip(wants, gots):
+                fraction = w.group("fraction")
+                exponent = w.group("exponent1")
+                if exponent is None:
+                    exponent = w.group("exponent2")
+                if fraction is None:
+                    precision = 0
+                else:
+                    precision = len(fraction)
+                if exponent is not None:
+                    precision -= int(exponent)
+                if float(w.group()) == approx(float(g.group()), abs=10 ** -precision):
+                    # They're close enough. Replace the text we actually
+                    # got with the text we want, so that it will match when we
+                    # check the string literally.
+                    got = (
+                        got[: g.start() + offset] + w.group() + got[g.end() + offset :]
+                    )
+                    offset += w.end() - w.start() - (g.end() - g.start())
+            return got
 
     _get_checker.LiteralsOutputChecker = LiteralsOutputChecker
     return _get_checker.LiteralsOutputChecker()
@@ -524,6 +582,15 @@ def _get_allow_bytes_flag():
     return doctest.register_optionflag("ALLOW_BYTES")
 
 
+def _get_number_flag():
+    """
+    Registers and returns the NUMBER flag.
+    """
+    import doctest
+
+    return doctest.register_optionflag("NUMBER")
+
+
 def _get_report_choice(key):
     """
     This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid

testing/test_doctest.py

@@ -3,6 +3,7 @@ import textwrap
 
 import pytest
 from _pytest.compat import MODULE_NOT_FOUND_ERROR
+from _pytest.doctest import _get_checker
 from _pytest.doctest import _is_mocked
 from _pytest.doctest import _patch_unwrap_mock_aware
 from _pytest.doctest import DoctestItem
@@ -838,6 +839,149 @@ class TestLiterals:
         reprec = testdir.inline_run()
         reprec.assertoutcome(failed=1)
 
+    def test_number_re(self):
+        for s in [
+            "1.",
+            "+1.",
+            "-1.",
+            ".1",
+            "+.1",
+            "-.1",
+            "0.1",
+            "+0.1",
+            "-0.1",
+            "1e5",
+            "+1e5",
+            "1e+5",
+            "+1e+5",
+            "1e-5",
+            "+1e-5",
+            "-1e-5",
+            "1.2e3",
+            "-1.2e-3",
+        ]:
+            print(s)
+            m = _get_checker()._number_re.match(s)
+            assert m is not None
+            assert float(m.group()) == pytest.approx(float(s))
+        for s in ["1", "abc"]:
+            print(s)
+            assert _get_checker()._number_re.match(s) is None
+
+    @pytest.mark.parametrize("config_mode", ["ini", "comment"])
+    def test_number_precision(self, testdir, config_mode):
+        """Test the NUMBER option."""
+        if config_mode == "ini":
+            testdir.makeini(
+                """
+                [pytest]
+                doctest_optionflags = NUMBER
+                """
+            )
+            comment = ""
+        else:
+            comment = "#doctest: +NUMBER"
+
+        testdir.maketxtfile(
+            test_doc="""
+
+            Scalars:
+
+            >>> import math
+            >>> math.pi {comment}
+            3.141592653589793
+            >>> math.pi {comment}
+            3.1416
+            >>> math.pi {comment}
+            3.14
+            >>> -math.pi {comment}
+            -3.14
+            >>> math.pi {comment}
+            3.
+            >>> 3. {comment}
+            3.0
+            >>> 3. {comment}
+            3.
+            >>> 3. {comment}
+            3.01
+            >>> 3. {comment}
+            2.99
+            >>> .299 {comment}
+            .3
+            >>> .301 {comment}
+            .3
+            >>> 951. {comment}
+            1e3
+            >>> 1049. {comment}
+            1e3
+            >>> -1049. {comment}
+            -1e3
+            >>> 1e3 {comment}
+            1e3
+            >>> 1e3 {comment}
+            1000.
+
+            Lists:
+
+            >>> [3.1415, 0.097, 13.1, 7, 8.22222e5, 0.598e-2] {comment}
+            [3.14, 0.1, 13., 7, 8.22e5, 6.0e-3]
+            >>> [[0.333, 0.667], [0.999, 1.333]] {comment}
+            [[0.33, 0.667], [0.999, 1.333]]
+            >>> [[[0.101]]] {comment}
+            [[[0.1]]]
+
+            Doesn't barf on non-numbers:
+
+            >>> 'abc' {comment}
+            'abc'
+            >>> None {comment}
+            """.format(
+                comment=comment
+            )
+        )
+        reprec = testdir.inline_run()
+        reprec.assertoutcome(passed=1)
+
+    @pytest.mark.parametrize(
+        "expression,output",
+        [
+            # ints shouldn't match floats:
+            ("3.0", "3"),
+            ("3e0", "3"),
+            ("1e3", "1000"),
+            ("3", "3.0"),
+            # Rounding:
+            ("3.1", "3.0"),
+            ("3.1", "3.2"),
+            ("3.1", "4.0"),
+            ("8.22e5", "810000.0"),
+            # Only the actual output is rounded up, not the expected output:
+            ("3.0", "2.99"),
+            ("1e3", "999"),
+        ],
+    )
+    def test_number_non_matches(self, testdir, expression, output):
+        testdir.maketxtfile(
+            test_doc="""
+            >>> {expression} #doctest: +NUMBER
+            {output}
+            """
+        )
+        reprec = testdir.inline_run()
+        reprec.assertoutcome(passed=0, failed=1)
+
+    def test_number_and_allow_unicode(self, testdir):
+        testdir.maketxtfile(
+            test_doc="""
+            >>> from collections import namedtuple
+            >>> T = namedtuple('T', 'a b c')
+            >>> T(a=0.2330000001, b=u'str', c=b'bytes') # doctest: +ALLOW_UNICODE, +ALLOW_BYTES, +NUMBER
+            T(a=0.233, b=u'str', c='bytes')
+            """
+        )
+        reprec = testdir.inline_run()
+        reprec.assertoutcome(passed=1)
+
 
 class TestDoctestSkips:
     """