From aaa7e837cce7439fdd74746f5adfc5bfe02edb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20R=C3=B6thlisberger?= Date: Mon, 8 Jul 2019 15:41:33 +0100 Subject: [PATCH 1/5] doctest: Add +NUMBER option to ignore irrelevant floating-point differences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When enabled, floating-point numbers only need to match as far as the precision you have written in the expected doctest output. This avoids false positives caused by limited floating-point precision, like this: Expected: 0.233 Got: 0.23300000000000001 This is inspired by Sébastien Boisgérault's [numtest] but the implementation is a bit different: * This implementation edits the literals that are in the "got" string (the actual output from the expression being tested), and then proceeds to compare the strings literally. This is similar to pytest's existing ALLOW_UNICODE and ALLOW_BYTES implementation. * This implementation only compares floats against floats, not ints against floats. That is, the following doctest will fail with pytest whereas it would pass with numtest: >>> math.py # doctest: +NUMBER 3 This behaviour should be less surprising (less false negatives) when you enable NUMBER globally in pytest.ini. Advantages of this implementation compared to numtest: * Doesn't require `import numtest` at the top level of the file. * Works with pytest (if you try to use pytest & numtest together, pytest raises "TypeError: unbound method check_output() must be called with NumTestOutputChecker instance as first argument (got LiteralsOutputChecker instance instead)"). * Works with Python 3. [numtest]: https://github.com/boisgera/numtest --- doc/en/doctest.rst | 29 ++++++-- src/_pytest/doctest.py | 111 +++++++++++++++++++++++++------ testing/test_doctest.py | 144 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 26 deletions(-) diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 2cb70af72..8eca000fd 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -115,15 +115,36 @@ lengthy exception stack traces you can just write: [pytest] doctest_optionflags= NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL -pytest also introduces new options to allow doctests to run in Python 2 and -Python 3 unchanged: +pytest also introduces new options: * ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode - strings in expected doctest output. + strings in expected doctest output. This allows doctests to run in Python 2 + and Python 3 unchanged. -* ``ALLOW_BYTES``: when enabled, the ``b`` prefix is stripped from byte strings +* ``ALLOW_BYTES``: similarly, the ``b`` prefix is stripped from byte strings in expected doctest output. +* ``NUMBER``: when enabled, floating-point numbers only need to match as far as + the precision you have written in the expected doctest output. For example, + the following output would only need to match to 2 decimal places:: + + >>> math.pi + 3.14 + + If you wrote ``3.1416`` then the actual output would need to match to 4 + decimal places; and so on. + + This avoids false positives caused by limited floating-point precision, like + this:: + + Expected: + 0.233 + Got: + 0.23300000000000001 + + ``NUMBER`` also supports lists of floating-point numbers -- in fact, it + supports floating-point numbers appearing anywhere in the output. + Alternatively, options can be enabled by an inline comment in the doc test itself: diff --git a/src/_pytest/doctest.py b/src/_pytest/doctest.py index ca6e4675f..cf886f906 100644 --- a/src/_pytest/doctest.py +++ b/src/_pytest/doctest.py @@ -13,6 +13,7 @@ from _pytest._code.code import TerminalRepr from _pytest.compat import safe_getattr from _pytest.fixtures import FixtureRequest from _pytest.outcomes import Skipped +from _pytest.python_api import approx from _pytest.warning_types import PytestWarning DOCTEST_REPORT_CHOICE_NONE = "none" @@ -286,6 +287,7 @@ def _get_flag_lookup(): COMPARISON_FLAGS=doctest.COMPARISON_FLAGS, ALLOW_UNICODE=_get_allow_unicode_flag(), ALLOW_BYTES=_get_allow_bytes_flag(), + NUMBER=_get_number_flag(), ) @@ -453,10 +455,15 @@ def _setup_fixtures(doctest_item): def _get_checker(): """ - Returns a doctest.OutputChecker subclass that takes in account the - ALLOW_UNICODE option to ignore u'' prefixes in strings and ALLOW_BYTES - to strip b'' prefixes. - Useful when the same doctest should run in Python 2 and Python 3. + Returns a doctest.OutputChecker subclass that supports some + additional options: + + * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b'' + prefixes (respectively) in string literals. Useful when the same + doctest should run in Python 2 and Python 3. + + * NUMBER to ignore floating-point differences smaller than the + precision of the literal number in the doctest. An inner class is used to avoid importing "doctest" at the module level. @@ -469,38 +476,89 @@ def _get_checker(): class LiteralsOutputChecker(doctest.OutputChecker): """ - Copied from doctest_nose_plugin.py from the nltk project: - https://github.com/nltk/nltk - - Further extended to also support byte literals. + Based on doctest_nose_plugin.py from the nltk project + (https://github.com/nltk/nltk) and on the "numtest" doctest extension + by Sebastien Boisgerault (https://github.com/boisgera/numtest). """ _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE) _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE) + _number_re = re.compile( + r""" + (?P + (?P + (?P [+-]?\d*)\.(?P\d+) + | + (?P [+-]?\d+)\. + ) + (?: + [Ee] + (?P [+-]?\d+) + )? + | + (?P [+-]?\d+) + (?: + [Ee] + (?P [+-]?\d+) + ) + ) + """, + re.VERBOSE, + ) def check_output(self, want, got, optionflags): - res = doctest.OutputChecker.check_output(self, want, got, optionflags) - if res: + if doctest.OutputChecker.check_output(self, want, got, optionflags): return True allow_unicode = optionflags & _get_allow_unicode_flag() allow_bytes = optionflags & _get_allow_bytes_flag() - if not allow_unicode and not allow_bytes: + allow_number = optionflags & _get_number_flag() + + if not allow_unicode and not allow_bytes and not allow_number: return False - else: # pragma: no cover + def remove_prefixes(regex, txt): + return re.sub(regex, r"\1\2", txt) - def remove_prefixes(regex, txt): - return re.sub(regex, r"\1\2", txt) + if allow_unicode: + want = remove_prefixes(self._unicode_literal_re, want) + got = remove_prefixes(self._unicode_literal_re, got) - if allow_unicode: - want = remove_prefixes(self._unicode_literal_re, want) - got = remove_prefixes(self._unicode_literal_re, got) - if allow_bytes: - want = remove_prefixes(self._bytes_literal_re, want) - got = remove_prefixes(self._bytes_literal_re, got) - res = doctest.OutputChecker.check_output(self, want, got, optionflags) - return res + if allow_bytes: + want = remove_prefixes(self._bytes_literal_re, want) + got = remove_prefixes(self._bytes_literal_re, got) + + if allow_number: + got = self._remove_unwanted_precision(want, got) + + return doctest.OutputChecker.check_output(self, want, got, optionflags) + + def _remove_unwanted_precision(self, want, got): + wants = list(self._number_re.finditer(want)) + gots = list(self._number_re.finditer(got)) + if len(wants) != len(gots): + return got + offset = 0 + for w, g in zip(wants, gots): + fraction = w.group("fraction") + exponent = w.group("exponent1") + if exponent is None: + exponent = w.group("exponent2") + if fraction is None: + precision = 0 + else: + precision = len(fraction) + if exponent is not None: + precision -= int(exponent) + if float(w.group()) == approx(float(g.group()), abs=10 ** -precision): + # They're close enough. Replace the text we actually + # got with the text we want, so that it will match when we + # check the string literally. + got = ( + got[: g.start() + offset] + w.group() + got[g.end() + offset :] + ) + offset += w.end() - w.start() - (g.end() - g.start()) + return got _get_checker.LiteralsOutputChecker = LiteralsOutputChecker return _get_checker.LiteralsOutputChecker() @@ -524,6 +582,15 @@ def _get_allow_bytes_flag(): return doctest.register_optionflag("ALLOW_BYTES") +def _get_number_flag(): + """ + Registers and returns the NUMBER flag. + """ + import doctest + + return doctest.register_optionflag("NUMBER") + + def _get_report_choice(key): """ This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 236066673..678c61af1 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -3,6 +3,7 @@ import textwrap import pytest from _pytest.compat import MODULE_NOT_FOUND_ERROR +from _pytest.doctest import _get_checker from _pytest.doctest import _is_mocked from _pytest.doctest import _patch_unwrap_mock_aware from _pytest.doctest import DoctestItem @@ -838,6 +839,149 @@ class TestLiterals: reprec = testdir.inline_run() reprec.assertoutcome(failed=1) + def test_number_re(self): + for s in [ + "1.", + "+1.", + "-1.", + ".1", + "+.1", + "-.1", + "0.1", + "+0.1", + "-0.1", + "1e5", + "+1e5", + "1e+5", + "+1e+5", + "1e-5", + "+1e-5", + "-1e-5", + "1.2e3", + "-1.2e-3", + ]: + print(s) + m = _get_checker()._number_re.match(s) + assert m is not None + assert float(m.group()) == pytest.approx(float(s)) + for s in ["1", "abc"]: + print(s) + assert _get_checker()._number_re.match(s) is None + + @pytest.mark.parametrize("config_mode", ["ini", "comment"]) + def test_number_precision(self, testdir, config_mode): + """Test the NUMBER option.""" + if config_mode == "ini": + testdir.makeini( + """ + [pytest] + doctest_optionflags = NUMBER + """ + ) + comment = "" + else: + comment = "#doctest: +NUMBER" + + testdir.maketxtfile( + test_doc=""" + + Scalars: + + >>> import math + >>> math.pi {comment} + 3.141592653589793 + >>> math.pi {comment} + 3.1416 + >>> math.pi {comment} + 3.14 + >>> -math.pi {comment} + -3.14 + >>> math.pi {comment} + 3. + >>> 3. {comment} + 3.0 + >>> 3. {comment} + 3. + >>> 3. {comment} + 3.01 + >>> 3. {comment} + 2.99 + >>> .299 {comment} + .3 + >>> .301 {comment} + .3 + >>> 951. {comment} + 1e3 + >>> 1049. {comment} + 1e3 + >>> -1049. {comment} + -1e3 + >>> 1e3 {comment} + 1e3 + >>> 1e3 {comment} + 1000. + + Lists: + + >>> [3.1415, 0.097, 13.1, 7, 8.22222e5, 0.598e-2] {comment} + [3.14, 0.1, 13., 7, 8.22e5, 6.0e-3] + >>> [[0.333, 0.667], [0.999, 1.333]] {comment} + [[0.33, 0.667], [0.999, 1.333]] + >>> [[[0.101]]] {comment} + [[[0.1]]] + + Doesn't barf on non-numbers: + + >>> 'abc' {comment} + 'abc' + >>> None {comment} + """.format( + comment=comment + ) + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=1) + + @pytest.mark.parametrize( + "expression,output", + [ + # ints shouldn't match floats: + ("3.0", "3"), + ("3e0", "3"), + ("1e3", "1000"), + ("3", "3.0"), + # Rounding: + ("3.1", "3.0"), + ("3.1", "3.2"), + ("3.1", "4.0"), + ("8.22e5", "810000.0"), + # Only the actual output is rounded up, not the expected output: + ("3.0", "2.99"), + ("1e3", "999"), + ], + ) + def test_number_non_matches(self, testdir, expression, output): + testdir.maketxtfile( + test_doc=""" + >>> {expression} #doctest: +NUMBER + {output} + """ + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=0, failed=1) + + def test_number_and_allow_unicode(self, testdir): + testdir.maketxtfile( + test_doc=""" + >>> from collections import namedtuple + >>> T = namedtuple('T', 'a b c') + >>> T(a=0.2330000001, b=u'str', c=b'bytes') # doctest: +ALLOW_UNICODE, +ALLOW_BYTES, +NUMBER + T(a=0.233, b=u'str', c='bytes') + """ + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=1) + class TestDoctestSkips: """ From 2a23fdab9f066e3ebf7f3bcffe6e40c136772a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20R=C3=B6thlisberger?= Date: Mon, 8 Jul 2019 17:00:32 +0100 Subject: [PATCH 2/5] docs: Tidy up doctest options section * Move the parts about "how to configure it" (pytest.ini vs. inline comment) together. * Move `--doctest-continue-on-failure` into its own sub-heading, as it isn't related to the doctest optionflags. --- doc/en/doctest.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 8eca000fd..9839c7ad4 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -103,7 +103,7 @@ that will be used for those doctest files using the Using 'doctest' options ----------------------- -The standard ``doctest`` module provides some `options `__ +Python's standard ``doctest`` module provides some `options `__ to configure the strictness of doctest tests. In pytest, you can enable those flags using the configuration file. @@ -115,6 +115,15 @@ lengthy exception stack traces you can just write: [pytest] doctest_optionflags= NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL +Alternatively, options can be enabled by an inline comment in the doc test +itself: + +.. code-block:: rst + + >>> something_that_raises() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: ... + pytest also introduces new options: * ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode @@ -145,14 +154,9 @@ pytest also introduces new options: ``NUMBER`` also supports lists of floating-point numbers -- in fact, it supports floating-point numbers appearing anywhere in the output. -Alternatively, options can be enabled by an inline comment in the doc test -itself: -.. code-block:: rst - - # content of example.rst - >>> get_unicode_greeting() # doctest: +ALLOW_UNICODE - 'Hello' +Continue on failure +------------------- By default, pytest would report only the first failure for a given doctest. If you want to continue the test even when you have failures, do: From d5cc0f2a622ad7ce182ed1f4dad0ac1641a68d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20R=C3=B6thlisberger?= Date: Mon, 8 Jul 2019 17:16:35 +0100 Subject: [PATCH 3/5] changelog for new NUMBERS doctest option --- AUTHORS | 1 + changelog/5576.feature.rst | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelog/5576.feature.rst diff --git a/AUTHORS b/AUTHORS index 087fce8d0..aacf16a36 100644 --- a/AUTHORS +++ b/AUTHORS @@ -70,6 +70,7 @@ Danielle Jenkins Dave Hunt David Díaz-Barquero David Mohr +David Paul Röthlisberger David Szotten David Vierra Daw-Ran Liou diff --git a/changelog/5576.feature.rst b/changelog/5576.feature.rst new file mode 100644 index 000000000..267a28292 --- /dev/null +++ b/changelog/5576.feature.rst @@ -0,0 +1,4 @@ +New `NUMBER `__ +option for doctests to ignore irrelevant differences in floating-point numbers. +Inspired by Sébastien Boisgérault's `numtest `__ +extension for doctest. From 4c590e002fc220ae27e9451a39990fdc452a93a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20R=C3=B6thlisberger?= Date: Thu, 11 Jul 2019 09:57:44 +0100 Subject: [PATCH 4/5] Fix test_doctest.test_number_non_matches These doctests were expected to fail, but they were failing because of a silly bug (I forgot to replace "{expression}" with the actual expression to be tested), not because of the thing they were meant to be testing. Then I had to fix one of the testcases because it was actually matching: >>> 3.0 #doctest: +NUMBER 2.99 The doctest is saying that the actual output should match to 2 decimal places, i.e. within 0.01 -- which it is, so it passes. I changed the expected output to 2.98 and now it doesn't match (as we expect). --- testing/test_doctest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 678c61af1..40b6d7ebb 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -956,7 +956,7 @@ class TestLiterals: ("3.1", "4.0"), ("8.22e5", "810000.0"), # Only the actual output is rounded up, not the expected output: - ("3.0", "2.99"), + ("3.0", "2.98"), ("1e3", "999"), ], ) @@ -965,7 +965,9 @@ class TestLiterals: test_doc=""" >>> {expression} #doctest: +NUMBER {output} - """ + """.format( + expression=expression, output=output + ) ) reprec = testdir.inline_run() reprec.assertoutcome(passed=0, failed=1) From a740ef20367bad2d401ab6ac0092d26bd2b62379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20R=C3=B6thlisberger?= Date: Thu, 11 Jul 2019 10:04:43 +0100 Subject: [PATCH 5/5] docs: Document doctest +NUMBER limitation with strings Also added an "xfail" testcase for the same. --- doc/en/doctest.rst | 4 +++- testing/test_doctest.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 9839c7ad4..b17327f62 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -152,7 +152,9 @@ pytest also introduces new options: 0.23300000000000001 ``NUMBER`` also supports lists of floating-point numbers -- in fact, it - supports floating-point numbers appearing anywhere in the output. + matches floating-point numbers appearing anywhere in the output, even inside + a string! This means that it may not be appropriate to enable globally in + ``doctest_optionflags`` in your configuration file. Continue on failure diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 40b6d7ebb..4aac5432d 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -958,6 +958,9 @@ class TestLiterals: # Only the actual output is rounded up, not the expected output: ("3.0", "2.98"), ("1e3", "999"), + # The current implementation doesn't understand that numbers inside + # strings shouldn't be treated as numbers: + pytest.param("'3.1416'", "'3.14'", marks=pytest.mark.xfail), ], ) def test_number_non_matches(self, testdir, expression, output):