doctest: Add +NUMBER option to ignore irrelevant floating-point… (#5576)

doctest: Add +NUMBER option to ignore irrelevant floating-point differences

AUTHORS

@@ -71,6 +71,7 @@ Danielle Jenkins
 Dave Hunt
 David Díaz-Barquero
 David Mohr
+David Paul Röthlisberger
 David Szotten
 David Vierra
 Daw-Ran Liou

changelog/5576.feature.rst

@@ -0,0 +1,4 @@
+New `NUMBER <https://docs.pytest.org/en/latest/doctest.html#using-doctest-options>`__
+option for doctests to ignore irrelevant differences in floating-point numbers.
+Inspired by Sébastien Boisgérault's `numtest <https://github.com/boisgera/numtest>`__
+extension for doctest.

doc/en/doctest.rst

@@ -103,7 +103,7 @@ that will be used for those doctest files using the
 Using 'doctest' options
 -----------------------
 
-The standard ``doctest`` module provides some `options <https://docs.python.org/3/library/doctest.html#option-flags>`__
+Python's standard ``doctest`` module provides some `options <https://docs.python.org/3/library/doctest.html#option-flags>`__
 to configure the strictness of doctest tests. In pytest, you can enable those flags using the
 configuration file.
 
@@ -115,23 +115,50 @@ lengthy exception stack traces you can just write:
     [pytest]
     doctest_optionflags= NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL
 
-pytest also introduces new options to allow doctests to run in Python 2 and
-Python 3 unchanged:
-
-* ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode
-  strings in expected doctest output.
-
-* ``ALLOW_BYTES``: when enabled, the ``b`` prefix is stripped from byte strings
-  in expected doctest output.
-
 Alternatively, options can be enabled by an inline comment in the doc test
 itself:
 
 .. code-block:: rst
 
-    # content of example.rst
+    >>> something_that_raises()  # doctest: +IGNORE_EXCEPTION_DETAIL
-    >>> get_unicode_greeting()  # doctest: +ALLOW_UNICODE
+    Traceback (most recent call last):
-    'Hello'
+    ValueError: ...
+
+pytest also introduces new options:
+
+* ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode
+  strings in expected doctest output. This allows doctests to run in Python 2
+  and Python 3 unchanged.
+
+* ``ALLOW_BYTES``: similarly, the ``b`` prefix is stripped from byte strings
+  in expected doctest output.
+
+* ``NUMBER``: when enabled, floating-point numbers only need to match as far as
+  the precision you have written in the expected doctest output. For example,
+  the following output would only need to match to 2 decimal places::
+
+      >>> math.pi
+      3.14
+
+  If you wrote ``3.1416`` then the actual output would need to match to 4
+  decimal places; and so on.
+
+  This avoids false positives caused by limited floating-point precision, like
+  this::
+
+      Expected:
+          0.233
+      Got:
+          0.23300000000000001
+
+  ``NUMBER`` also supports lists of floating-point numbers -- in fact, it
+  matches floating-point numbers appearing anywhere in the output, even inside
+  a string! This means that it may not be appropriate to enable globally in
+  ``doctest_optionflags`` in your configuration file.
+
+
+Continue on failure
+-------------------
 
 By default, pytest would report only the first failure for a given doctest. If
 you want to continue the test even when you have failures, do:

src/_pytest/doctest.py

@@ -13,6 +13,7 @@ from _pytest._code.code import TerminalRepr
 from _pytest.compat import safe_getattr
 from _pytest.fixtures import FixtureRequest
 from _pytest.outcomes import Skipped
+from _pytest.python_api import approx
 from _pytest.warning_types import PytestWarning
 
 DOCTEST_REPORT_CHOICE_NONE = "none"
@@ -286,6 +287,7 @@ def _get_flag_lookup():
         COMPARISON_FLAGS=doctest.COMPARISON_FLAGS,
         ALLOW_UNICODE=_get_allow_unicode_flag(),
         ALLOW_BYTES=_get_allow_bytes_flag(),
+        NUMBER=_get_number_flag(),
     )
 
 
@@ -453,10 +455,15 @@ def _setup_fixtures(doctest_item):
 
 def _get_checker():
     """
-    Returns a doctest.OutputChecker subclass that takes in account the
+    Returns a doctest.OutputChecker subclass that supports some
-    ALLOW_UNICODE option to ignore u'' prefixes in strings and ALLOW_BYTES
+    additional options:
-    to strip b'' prefixes.
+
-    Useful when the same doctest should run in Python 2 and Python 3.
+    * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b''
+      prefixes (respectively) in string literals. Useful when the same
+      doctest should run in Python 2 and Python 3.
+
+    * NUMBER to ignore floating-point differences smaller than the
+      precision of the literal number in the doctest.
 
     An inner class is used to avoid importing "doctest" at the module
     level.
@@ -469,38 +476,89 @@ def _get_checker():
 
     class LiteralsOutputChecker(doctest.OutputChecker):
         """
-        Copied from doctest_nose_plugin.py from the nltk project:
+        Based on doctest_nose_plugin.py from the nltk project
-            https://github.com/nltk/nltk
+        (https://github.com/nltk/nltk) and on the "numtest" doctest extension
-
+        by Sebastien Boisgerault (https://github.com/boisgera/numtest).
-        Further extended to also support byte literals.
         """
 
         _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE)
         _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE)
+        _number_re = re.compile(
+            r"""
+            (?P<number>
+              (?P<mantissa>
+                (?P<integer1> [+-]?\d*)\.(?P<fraction>\d+)
+                |
+                (?P<integer2> [+-]?\d+)\.
+              )
+              (?:
+                [Ee]
+                (?P<exponent1> [+-]?\d+)
+              )?
+              |
+              (?P<integer3> [+-]?\d+)
+              (?:
+                [Ee]
+                (?P<exponent2> [+-]?\d+)
+              )
+            )
+            """,
+            re.VERBOSE,
+        )
 
         def check_output(self, want, got, optionflags):
-            res = doctest.OutputChecker.check_output(self, want, got, optionflags)
+            if doctest.OutputChecker.check_output(self, want, got, optionflags):
-            if res:
                 return True
 
             allow_unicode = optionflags & _get_allow_unicode_flag()
             allow_bytes = optionflags & _get_allow_bytes_flag()
-            if not allow_unicode and not allow_bytes:
+            allow_number = optionflags & _get_number_flag()
+
+            if not allow_unicode and not allow_bytes and not allow_number:
                 return False
 
-            else:  # pragma: no cover
+            def remove_prefixes(regex, txt):
+                return re.sub(regex, r"\1\2", txt)
 
-                def remove_prefixes(regex, txt):
+            if allow_unicode:
-                    return re.sub(regex, r"\1\2", txt)
+                want = remove_prefixes(self._unicode_literal_re, want)
+                got = remove_prefixes(self._unicode_literal_re, got)
 
-                if allow_unicode:
+            if allow_bytes:
-                    want = remove_prefixes(self._unicode_literal_re, want)
+                want = remove_prefixes(self._bytes_literal_re, want)
-                    got = remove_prefixes(self._unicode_literal_re, got)
+                got = remove_prefixes(self._bytes_literal_re, got)
-                if allow_bytes:
+
-                    want = remove_prefixes(self._bytes_literal_re, want)
+            if allow_number:
-                    got = remove_prefixes(self._bytes_literal_re, got)
+                got = self._remove_unwanted_precision(want, got)
-                res = doctest.OutputChecker.check_output(self, want, got, optionflags)
+
-                return res
+            return doctest.OutputChecker.check_output(self, want, got, optionflags)
+
+        def _remove_unwanted_precision(self, want, got):
+            wants = list(self._number_re.finditer(want))
+            gots = list(self._number_re.finditer(got))
+            if len(wants) != len(gots):
+                return got
+            offset = 0
+            for w, g in zip(wants, gots):
+                fraction = w.group("fraction")
+                exponent = w.group("exponent1")
+                if exponent is None:
+                    exponent = w.group("exponent2")
+                if fraction is None:
+                    precision = 0
+                else:
+                    precision = len(fraction)
+                if exponent is not None:
+                    precision -= int(exponent)
+                if float(w.group()) == approx(float(g.group()), abs=10 ** -precision):
+                    # They're close enough. Replace the text we actually
+                    # got with the text we want, so that it will match when we
+                    # check the string literally.
+                    got = (
+                        got[: g.start() + offset] + w.group() + got[g.end() + offset :]
+                    )
+                    offset += w.end() - w.start() - (g.end() - g.start())
+            return got
 
     _get_checker.LiteralsOutputChecker = LiteralsOutputChecker
     return _get_checker.LiteralsOutputChecker()
@@ -524,6 +582,15 @@ def _get_allow_bytes_flag():
     return doctest.register_optionflag("ALLOW_BYTES")
 
 
+def _get_number_flag():
+    """
+    Registers and returns the NUMBER flag.
+    """
+    import doctest
+
+    return doctest.register_optionflag("NUMBER")
+
+
 def _get_report_choice(key):
     """
     This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid

testing/test_doctest.py

@@ -3,6 +3,7 @@ import textwrap
 
 import pytest
 from _pytest.compat import MODULE_NOT_FOUND_ERROR
+from _pytest.doctest import _get_checker
 from _pytest.doctest import _is_mocked
 from _pytest.doctest import _patch_unwrap_mock_aware
 from _pytest.doctest import DoctestItem
@@ -838,6 +839,154 @@ class TestLiterals:
         reprec = testdir.inline_run()
         reprec.assertoutcome(failed=1)
 
+    def test_number_re(self):
+        for s in [
+            "1.",
+            "+1.",
+            "-1.",
+            ".1",
+            "+.1",
+            "-.1",
+            "0.1",
+            "+0.1",
+            "-0.1",
+            "1e5",
+            "+1e5",
+            "1e+5",
+            "+1e+5",
+            "1e-5",
+            "+1e-5",
+            "-1e-5",
+            "1.2e3",
+            "-1.2e-3",
+        ]:
+            print(s)
+            m = _get_checker()._number_re.match(s)
+            assert m is not None
+            assert float(m.group()) == pytest.approx(float(s))
+        for s in ["1", "abc"]:
+            print(s)
+            assert _get_checker()._number_re.match(s) is None
+
+    @pytest.mark.parametrize("config_mode", ["ini", "comment"])
+    def test_number_precision(self, testdir, config_mode):
+        """Test the NUMBER option."""
+        if config_mode == "ini":
+            testdir.makeini(
+                """
+                [pytest]
+                doctest_optionflags = NUMBER
+                """
+            )
+            comment = ""
+        else:
+            comment = "#doctest: +NUMBER"
+
+        testdir.maketxtfile(
+            test_doc="""
+
+            Scalars:
+
+            >>> import math
+            >>> math.pi {comment}
+            3.141592653589793
+            >>> math.pi {comment}
+            3.1416
+            >>> math.pi {comment}
+            3.14
+            >>> -math.pi {comment}
+            -3.14
+            >>> math.pi {comment}
+            3.
+            >>> 3. {comment}
+            3.0
+            >>> 3. {comment}
+            3.
+            >>> 3. {comment}
+            3.01
+            >>> 3. {comment}
+            2.99
+            >>> .299 {comment}
+            .3
+            >>> .301 {comment}
+            .3
+            >>> 951. {comment}
+            1e3
+            >>> 1049. {comment}
+            1e3
+            >>> -1049. {comment}
+            -1e3
+            >>> 1e3 {comment}
+            1e3
+            >>> 1e3 {comment}
+            1000.
+
+            Lists:
+
+            >>> [3.1415, 0.097, 13.1, 7, 8.22222e5, 0.598e-2] {comment}
+            [3.14, 0.1, 13., 7, 8.22e5, 6.0e-3]
+            >>> [[0.333, 0.667], [0.999, 1.333]] {comment}
+            [[0.33, 0.667], [0.999, 1.333]]
+            >>> [[[0.101]]] {comment}
+            [[[0.1]]]
+
+            Doesn't barf on non-numbers:
+
+            >>> 'abc' {comment}
+            'abc'
+            >>> None {comment}
+            """.format(
+                comment=comment
+            )
+        )
+        reprec = testdir.inline_run()
+        reprec.assertoutcome(passed=1)
+
+    @pytest.mark.parametrize(
+        "expression,output",
+        [
+            # ints shouldn't match floats:
+            ("3.0", "3"),
+            ("3e0", "3"),
+            ("1e3", "1000"),
+            ("3", "3.0"),
+            # Rounding:
+            ("3.1", "3.0"),
+            ("3.1", "3.2"),
+            ("3.1", "4.0"),
+            ("8.22e5", "810000.0"),
+            # Only the actual output is rounded up, not the expected output:
+            ("3.0", "2.98"),
+            ("1e3", "999"),
+            # The current implementation doesn't understand that numbers inside
+            # strings shouldn't be treated as numbers:
+            pytest.param("'3.1416'", "'3.14'", marks=pytest.mark.xfail),
+        ],
+    )
+    def test_number_non_matches(self, testdir, expression, output):
+        testdir.maketxtfile(
+            test_doc="""
+            >>> {expression} #doctest: +NUMBER
+            {output}
+            """.format(
+                expression=expression, output=output
+            )
+        )
+        reprec = testdir.inline_run()
+        reprec.assertoutcome(passed=0, failed=1)
+
+    def test_number_and_allow_unicode(self, testdir):
+        testdir.maketxtfile(
+            test_doc="""
+            >>> from collections import namedtuple
+            >>> T = namedtuple('T', 'a b c')
+            >>> T(a=0.2330000001, b=u'str', c=b'bytes') # doctest: +ALLOW_UNICODE, +ALLOW_BYTES, +NUMBER
+            T(a=0.233, b=u'str', c='bytes')
+            """
+        )
+        reprec = testdir.inline_run()
+        reprec.assertoutcome(passed=1)
+
 
 class TestDoctestSkips:
     """