From aaa7e837cce7439fdd74746f5adfc5bfe02edb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20R=C3=B6thlisberger?= Date: Mon, 8 Jul 2019 15:41:33 +0100 Subject: [PATCH] doctest: Add +NUMBER option to ignore irrelevant floating-point differences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When enabled, floating-point numbers only need to match as far as the precision you have written in the expected doctest output. This avoids false positives caused by limited floating-point precision, like this: Expected: 0.233 Got: 0.23300000000000001 This is inspired by Sébastien Boisgérault's [numtest] but the implementation is a bit different: * This implementation edits the literals that are in the "got" string (the actual output from the expression being tested), and then proceeds to compare the strings literally. This is similar to pytest's existing ALLOW_UNICODE and ALLOW_BYTES implementation. * This implementation only compares floats against floats, not ints against floats. That is, the following doctest will fail with pytest whereas it would pass with numtest: >>> math.py # doctest: +NUMBER 3 This behaviour should be less surprising (less false negatives) when you enable NUMBER globally in pytest.ini. Advantages of this implementation compared to numtest: * Doesn't require `import numtest` at the top level of the file. * Works with pytest (if you try to use pytest & numtest together, pytest raises "TypeError: unbound method check_output() must be called with NumTestOutputChecker instance as first argument (got LiteralsOutputChecker instance instead)"). * Works with Python 3. [numtest]: https://github.com/boisgera/numtest --- doc/en/doctest.rst | 29 ++++++-- src/_pytest/doctest.py | 111 +++++++++++++++++++++++++------ testing/test_doctest.py | 144 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 26 deletions(-) diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 2cb70af72..8eca000fd 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -115,15 +115,36 @@ lengthy exception stack traces you can just write: [pytest] doctest_optionflags= NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL -pytest also introduces new options to allow doctests to run in Python 2 and -Python 3 unchanged: +pytest also introduces new options: * ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode - strings in expected doctest output. + strings in expected doctest output. This allows doctests to run in Python 2 + and Python 3 unchanged. -* ``ALLOW_BYTES``: when enabled, the ``b`` prefix is stripped from byte strings +* ``ALLOW_BYTES``: similarly, the ``b`` prefix is stripped from byte strings in expected doctest output. +* ``NUMBER``: when enabled, floating-point numbers only need to match as far as + the precision you have written in the expected doctest output. For example, + the following output would only need to match to 2 decimal places:: + + >>> math.pi + 3.14 + + If you wrote ``3.1416`` then the actual output would need to match to 4 + decimal places; and so on. + + This avoids false positives caused by limited floating-point precision, like + this:: + + Expected: + 0.233 + Got: + 0.23300000000000001 + + ``NUMBER`` also supports lists of floating-point numbers -- in fact, it + supports floating-point numbers appearing anywhere in the output. + Alternatively, options can be enabled by an inline comment in the doc test itself: diff --git a/src/_pytest/doctest.py b/src/_pytest/doctest.py index ca6e4675f..cf886f906 100644 --- a/src/_pytest/doctest.py +++ b/src/_pytest/doctest.py @@ -13,6 +13,7 @@ from _pytest._code.code import TerminalRepr from _pytest.compat import safe_getattr from _pytest.fixtures import FixtureRequest from _pytest.outcomes import Skipped +from _pytest.python_api import approx from _pytest.warning_types import PytestWarning DOCTEST_REPORT_CHOICE_NONE = "none" @@ -286,6 +287,7 @@ def _get_flag_lookup(): COMPARISON_FLAGS=doctest.COMPARISON_FLAGS, ALLOW_UNICODE=_get_allow_unicode_flag(), ALLOW_BYTES=_get_allow_bytes_flag(), + NUMBER=_get_number_flag(), ) @@ -453,10 +455,15 @@ def _setup_fixtures(doctest_item): def _get_checker(): """ - Returns a doctest.OutputChecker subclass that takes in account the - ALLOW_UNICODE option to ignore u'' prefixes in strings and ALLOW_BYTES - to strip b'' prefixes. - Useful when the same doctest should run in Python 2 and Python 3. + Returns a doctest.OutputChecker subclass that supports some + additional options: + + * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b'' + prefixes (respectively) in string literals. Useful when the same + doctest should run in Python 2 and Python 3. + + * NUMBER to ignore floating-point differences smaller than the + precision of the literal number in the doctest. An inner class is used to avoid importing "doctest" at the module level. @@ -469,38 +476,89 @@ def _get_checker(): class LiteralsOutputChecker(doctest.OutputChecker): """ - Copied from doctest_nose_plugin.py from the nltk project: - https://github.com/nltk/nltk - - Further extended to also support byte literals. + Based on doctest_nose_plugin.py from the nltk project + (https://github.com/nltk/nltk) and on the "numtest" doctest extension + by Sebastien Boisgerault (https://github.com/boisgera/numtest). """ _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE) _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE) + _number_re = re.compile( + r""" + (?P + (?P + (?P [+-]?\d*)\.(?P\d+) + | + (?P [+-]?\d+)\. + ) + (?: + [Ee] + (?P [+-]?\d+) + )? + | + (?P [+-]?\d+) + (?: + [Ee] + (?P [+-]?\d+) + ) + ) + """, + re.VERBOSE, + ) def check_output(self, want, got, optionflags): - res = doctest.OutputChecker.check_output(self, want, got, optionflags) - if res: + if doctest.OutputChecker.check_output(self, want, got, optionflags): return True allow_unicode = optionflags & _get_allow_unicode_flag() allow_bytes = optionflags & _get_allow_bytes_flag() - if not allow_unicode and not allow_bytes: + allow_number = optionflags & _get_number_flag() + + if not allow_unicode and not allow_bytes and not allow_number: return False - else: # pragma: no cover + def remove_prefixes(regex, txt): + return re.sub(regex, r"\1\2", txt) - def remove_prefixes(regex, txt): - return re.sub(regex, r"\1\2", txt) + if allow_unicode: + want = remove_prefixes(self._unicode_literal_re, want) + got = remove_prefixes(self._unicode_literal_re, got) - if allow_unicode: - want = remove_prefixes(self._unicode_literal_re, want) - got = remove_prefixes(self._unicode_literal_re, got) - if allow_bytes: - want = remove_prefixes(self._bytes_literal_re, want) - got = remove_prefixes(self._bytes_literal_re, got) - res = doctest.OutputChecker.check_output(self, want, got, optionflags) - return res + if allow_bytes: + want = remove_prefixes(self._bytes_literal_re, want) + got = remove_prefixes(self._bytes_literal_re, got) + + if allow_number: + got = self._remove_unwanted_precision(want, got) + + return doctest.OutputChecker.check_output(self, want, got, optionflags) + + def _remove_unwanted_precision(self, want, got): + wants = list(self._number_re.finditer(want)) + gots = list(self._number_re.finditer(got)) + if len(wants) != len(gots): + return got + offset = 0 + for w, g in zip(wants, gots): + fraction = w.group("fraction") + exponent = w.group("exponent1") + if exponent is None: + exponent = w.group("exponent2") + if fraction is None: + precision = 0 + else: + precision = len(fraction) + if exponent is not None: + precision -= int(exponent) + if float(w.group()) == approx(float(g.group()), abs=10 ** -precision): + # They're close enough. Replace the text we actually + # got with the text we want, so that it will match when we + # check the string literally. + got = ( + got[: g.start() + offset] + w.group() + got[g.end() + offset :] + ) + offset += w.end() - w.start() - (g.end() - g.start()) + return got _get_checker.LiteralsOutputChecker = LiteralsOutputChecker return _get_checker.LiteralsOutputChecker() @@ -524,6 +582,15 @@ def _get_allow_bytes_flag(): return doctest.register_optionflag("ALLOW_BYTES") +def _get_number_flag(): + """ + Registers and returns the NUMBER flag. + """ + import doctest + + return doctest.register_optionflag("NUMBER") + + def _get_report_choice(key): """ This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 236066673..678c61af1 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -3,6 +3,7 @@ import textwrap import pytest from _pytest.compat import MODULE_NOT_FOUND_ERROR +from _pytest.doctest import _get_checker from _pytest.doctest import _is_mocked from _pytest.doctest import _patch_unwrap_mock_aware from _pytest.doctest import DoctestItem @@ -838,6 +839,149 @@ class TestLiterals: reprec = testdir.inline_run() reprec.assertoutcome(failed=1) + def test_number_re(self): + for s in [ + "1.", + "+1.", + "-1.", + ".1", + "+.1", + "-.1", + "0.1", + "+0.1", + "-0.1", + "1e5", + "+1e5", + "1e+5", + "+1e+5", + "1e-5", + "+1e-5", + "-1e-5", + "1.2e3", + "-1.2e-3", + ]: + print(s) + m = _get_checker()._number_re.match(s) + assert m is not None + assert float(m.group()) == pytest.approx(float(s)) + for s in ["1", "abc"]: + print(s) + assert _get_checker()._number_re.match(s) is None + + @pytest.mark.parametrize("config_mode", ["ini", "comment"]) + def test_number_precision(self, testdir, config_mode): + """Test the NUMBER option.""" + if config_mode == "ini": + testdir.makeini( + """ + [pytest] + doctest_optionflags = NUMBER + """ + ) + comment = "" + else: + comment = "#doctest: +NUMBER" + + testdir.maketxtfile( + test_doc=""" + + Scalars: + + >>> import math + >>> math.pi {comment} + 3.141592653589793 + >>> math.pi {comment} + 3.1416 + >>> math.pi {comment} + 3.14 + >>> -math.pi {comment} + -3.14 + >>> math.pi {comment} + 3. + >>> 3. {comment} + 3.0 + >>> 3. {comment} + 3. + >>> 3. {comment} + 3.01 + >>> 3. {comment} + 2.99 + >>> .299 {comment} + .3 + >>> .301 {comment} + .3 + >>> 951. {comment} + 1e3 + >>> 1049. {comment} + 1e3 + >>> -1049. {comment} + -1e3 + >>> 1e3 {comment} + 1e3 + >>> 1e3 {comment} + 1000. + + Lists: + + >>> [3.1415, 0.097, 13.1, 7, 8.22222e5, 0.598e-2] {comment} + [3.14, 0.1, 13., 7, 8.22e5, 6.0e-3] + >>> [[0.333, 0.667], [0.999, 1.333]] {comment} + [[0.33, 0.667], [0.999, 1.333]] + >>> [[[0.101]]] {comment} + [[[0.1]]] + + Doesn't barf on non-numbers: + + >>> 'abc' {comment} + 'abc' + >>> None {comment} + """.format( + comment=comment + ) + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=1) + + @pytest.mark.parametrize( + "expression,output", + [ + # ints shouldn't match floats: + ("3.0", "3"), + ("3e0", "3"), + ("1e3", "1000"), + ("3", "3.0"), + # Rounding: + ("3.1", "3.0"), + ("3.1", "3.2"), + ("3.1", "4.0"), + ("8.22e5", "810000.0"), + # Only the actual output is rounded up, not the expected output: + ("3.0", "2.99"), + ("1e3", "999"), + ], + ) + def test_number_non_matches(self, testdir, expression, output): + testdir.maketxtfile( + test_doc=""" + >>> {expression} #doctest: +NUMBER + {output} + """ + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=0, failed=1) + + def test_number_and_allow_unicode(self, testdir): + testdir.maketxtfile( + test_doc=""" + >>> from collections import namedtuple + >>> T = namedtuple('T', 'a b c') + >>> T(a=0.2330000001, b=u'str', c=b'bytes') # doctest: +ALLOW_UNICODE, +ALLOW_BYTES, +NUMBER + T(a=0.233, b=u'str', c='bytes') + """ + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=1) + class TestDoctestSkips: """