From 36251e0db473d2d42d602ee759a80eb6f1d80dfc Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 11 Jun 2017 12:15:30 +0200 Subject: [PATCH 01/73] move approx to own file --- _pytest/python.py | 250 ---------------------------------------------- pytest.py | 4 +- 2 files changed, 3 insertions(+), 251 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 06f74ce4b..72269f0f0 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1265,256 +1265,6 @@ class RaisesContext(object): return suppress_exception -# builtin pytest.approx helper - -class approx(object): - """ - Assert that two numbers (or two sets of numbers) are equal to each other - within some tolerance. - - Due to the `intricacies of floating-point arithmetic`__, numbers that we - would intuitively expect to be equal are not always so:: - - >>> 0.1 + 0.2 == 0.3 - False - - __ https://docs.python.org/3/tutorial/floatingpoint.html - - This problem is commonly encountered when writing tests, e.g. when making - sure that floating-point values are what you expect them to be. One way to - deal with this problem is to assert that two floating-point numbers are - equal to within some appropriate tolerance:: - - >>> abs((0.1 + 0.2) - 0.3) < 1e-6 - True - - However, comparisons like this are tedious to write and difficult to - understand. Furthermore, absolute comparisons like the one above are - usually discouraged because there's no tolerance that works well for all - situations. ``1e-6`` is good for numbers around ``1``, but too small for - very big numbers and too big for very small ones. It's better to express - the tolerance as a fraction of the expected value, but relative comparisons - like that are even more difficult to write correctly and concisely. - - The ``approx`` class performs floating-point comparisons using a syntax - that's as intuitive as possible:: - - >>> from pytest import approx - >>> 0.1 + 0.2 == approx(0.3) - True - - The same syntax also works on sequences of numbers:: - - >>> (0.1 + 0.2, 0.2 + 0.4) == approx((0.3, 0.6)) - True - - By default, ``approx`` considers numbers within a relative tolerance of - ``1e-6`` (i.e. one part in a million) of its expected value to be equal. - This treatment would lead to surprising results if the expected value was - ``0.0``, because nothing but ``0.0`` itself is relatively close to ``0.0``. - To handle this case less surprisingly, ``approx`` also considers numbers - within an absolute tolerance of ``1e-12`` of its expected value to be - equal. Infinite numbers are another special case. They are only - considered equal to themselves, regardless of the relative tolerance. Both - the relative and absolute tolerances can be changed by passing arguments to - the ``approx`` constructor:: - - >>> 1.0001 == approx(1) - False - >>> 1.0001 == approx(1, rel=1e-3) - True - >>> 1.0001 == approx(1, abs=1e-3) - True - - If you specify ``abs`` but not ``rel``, the comparison will not consider - the relative tolerance at all. In other words, two numbers that are within - the default relative tolerance of ``1e-6`` will still be considered unequal - if they exceed the specified absolute tolerance. If you specify both - ``abs`` and ``rel``, the numbers will be considered equal if either - tolerance is met:: - - >>> 1 + 1e-8 == approx(1) - True - >>> 1 + 1e-8 == approx(1, abs=1e-12) - False - >>> 1 + 1e-8 == approx(1, rel=1e-6, abs=1e-12) - True - - If you're thinking about using ``approx``, then you might want to know how - it compares to other good ways of comparing floating-point numbers. All of - these algorithms are based on relative and absolute tolerances and should - agree for the most part, but they do have meaningful differences: - - - ``math.isclose(a, b, rel_tol=1e-9, abs_tol=0.0)``: True if the relative - tolerance is met w.r.t. either ``a`` or ``b`` or if the absolute - tolerance is met. Because the relative tolerance is calculated w.r.t. - both ``a`` and ``b``, this test is symmetric (i.e. neither ``a`` nor - ``b`` is a "reference value"). You have to specify an absolute tolerance - if you want to compare to ``0.0`` because there is no tolerance by - default. Only available in python>=3.5. `More information...`__ - - __ https://docs.python.org/3/library/math.html#math.isclose - - - ``numpy.isclose(a, b, rtol=1e-5, atol=1e-8)``: True if the difference - between ``a`` and ``b`` is less that the sum of the relative tolerance - w.r.t. ``b`` and the absolute tolerance. Because the relative tolerance - is only calculated w.r.t. ``b``, this test is asymmetric and you can - think of ``b`` as the reference value. Support for comparing sequences - is provided by ``numpy.allclose``. `More information...`__ - - __ http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.isclose.html - - - ``unittest.TestCase.assertAlmostEqual(a, b)``: True if ``a`` and ``b`` - are within an absolute tolerance of ``1e-7``. No relative tolerance is - considered and the absolute tolerance cannot be changed, so this function - is not appropriate for very large or very small numbers. Also, it's only - available in subclasses of ``unittest.TestCase`` and it's ugly because it - doesn't follow PEP8. `More information...`__ - - __ https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertAlmostEqual - - - ``a == pytest.approx(b, rel=1e-6, abs=1e-12)``: True if the relative - tolerance is met w.r.t. ``b`` or if the absolute tolerance is met. - Because the relative tolerance is only calculated w.r.t. ``b``, this test - is asymmetric and you can think of ``b`` as the reference value. In the - special case that you explicitly specify an absolute tolerance but not a - relative tolerance, only the absolute tolerance is considered. - """ - - def __init__(self, expected, rel=None, abs=None): - self.expected = expected - self.abs = abs - self.rel = rel - - def __repr__(self): - return ', '.join(repr(x) for x in self.expected) - - def __eq__(self, actual): - from collections import Iterable - if not isinstance(actual, Iterable): - actual = [actual] - if len(actual) != len(self.expected): - return False - return all(a == x for a, x in zip(actual, self.expected)) - - __hash__ = None - - def __ne__(self, actual): - return not (actual == self) - - @property - def expected(self): - # Regardless of whether the user-specified expected value is a number - # or a sequence of numbers, return a list of ApproxNotIterable objects - # that can be compared against. - from collections import Iterable - approx_non_iter = lambda x: ApproxNonIterable(x, self.rel, self.abs) - if isinstance(self._expected, Iterable): - return [approx_non_iter(x) for x in self._expected] - else: - return [approx_non_iter(self._expected)] - - @expected.setter - def expected(self, expected): - self._expected = expected - - -class ApproxNonIterable(object): - """ - Perform approximate comparisons for single numbers only. - - In other words, the ``expected`` attribute for objects of this class must - be some sort of number. This is in contrast to the ``approx`` class, where - the ``expected`` attribute can either be a number of a sequence of numbers. - This class is responsible for making comparisons, while ``approx`` is - responsible for abstracting the difference between numbers and sequences of - numbers. Although this class can stand on its own, it's only meant to be - used within ``approx``. - """ - - def __init__(self, expected, rel=None, abs=None): - self.expected = expected - self.abs = abs - self.rel = rel - - def __repr__(self): - if isinstance(self.expected, complex): - return str(self.expected) - - # Infinities aren't compared using tolerances, so don't show a - # tolerance. - if math.isinf(self.expected): - return str(self.expected) - - # If a sensible tolerance can't be calculated, self.tolerance will - # raise a ValueError. In this case, display '???'. - try: - vetted_tolerance = '{:.1e}'.format(self.tolerance) - except ValueError: - vetted_tolerance = '???' - - if sys.version_info[0] == 2: - return '{0} +- {1}'.format(self.expected, vetted_tolerance) - else: - return u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) - - def __eq__(self, actual): - # Short-circuit exact equality. - if actual == self.expected: - return True - - # Infinity shouldn't be approximately equal to anything but itself, but - # if there's a relative tolerance, it will be infinite and infinity - # will seem approximately equal to everything. The equal-to-itself - # case would have been short circuited above, so here we can just - # return false if the expected value is infinite. The abs() call is - # for compatibility with complex numbers. - if math.isinf(abs(self.expected)): - return False - - # Return true if the two numbers are within the tolerance. - return abs(self.expected - actual) <= self.tolerance - - __hash__ = None - - def __ne__(self, actual): - return not (actual == self) - - @property - def tolerance(self): - set_default = lambda x, default: x if x is not None else default - - # Figure out what the absolute tolerance should be. ``self.abs`` is - # either None or a value specified by the user. - absolute_tolerance = set_default(self.abs, 1e-12) - - if absolute_tolerance < 0: - raise ValueError("absolute tolerance can't be negative: {}".format(absolute_tolerance)) - if math.isnan(absolute_tolerance): - raise ValueError("absolute tolerance can't be NaN.") - - # If the user specified an absolute tolerance but not a relative one, - # just return the absolute tolerance. - if self.rel is None: - if self.abs is not None: - return absolute_tolerance - - # Figure out what the relative tolerance should be. ``self.rel`` is - # either None or a value specified by the user. This is done after - # we've made sure the user didn't ask for an absolute tolerance only, - # because we don't want to raise errors about the relative tolerance if - # we aren't even going to use it. - relative_tolerance = set_default(self.rel, 1e-6) * abs(self.expected) - - if relative_tolerance < 0: - raise ValueError("relative tolerance can't be negative: {}".format(absolute_tolerance)) - if math.isnan(relative_tolerance): - raise ValueError("relative tolerance can't be NaN.") - - # Return the larger of the relative and absolute tolerances. - return max(relative_tolerance, absolute_tolerance) - - # # the basic pytest Function item # diff --git a/pytest.py b/pytest.py index 4e4ccb32d..e3c72023c 100644 --- a/pytest.py +++ b/pytest.py @@ -22,10 +22,12 @@ from _pytest.skipping import xfail from _pytest.main import Item, Collector, File, Session from _pytest.fixtures import fillfixtures as _fillfuncargs from _pytest.python import ( - raises, approx, + raises, Module, Class, Instance, Function, Generator, ) +from _pytest.python_api import approx + set_trace = __pytestPDB.set_trace __all__ = [ From 6be57a3711465c70b3cb4e5ddf5ee373608f6fbf Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 11 Jun 2017 12:27:16 +0200 Subject: [PATCH 02/73] move python api helpers out of the python module this separates exposed normal api from collection elements --- _pytest/python.py | 174 ----------------- _pytest/python_api.py | 430 ++++++++++++++++++++++++++++++++++++++++++ pytest.py | 3 +- 3 files changed, 431 insertions(+), 176 deletions(-) create mode 100644 _pytest/python_api.py diff --git a/_pytest/python.py b/_pytest/python.py index 72269f0f0..1a313a59e 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -6,7 +6,6 @@ import inspect import sys import os import collections -import math from itertools import count import py @@ -1091,179 +1090,6 @@ def _showfixtures_main(config, session): red=True) -# builtin pytest.raises helper - -def raises(expected_exception, *args, **kwargs): - """ - Assert that a code block/function call raises ``expected_exception`` - and raise a failure exception otherwise. - - This helper produces a ``ExceptionInfo()`` object (see below). - - If using Python 2.5 or above, you may use this function as a - context manager:: - - >>> with raises(ZeroDivisionError): - ... 1/0 - - .. versionchanged:: 2.10 - - In the context manager form you may use the keyword argument - ``message`` to specify a custom failure message:: - - >>> with raises(ZeroDivisionError, message="Expecting ZeroDivisionError"): - ... pass - Traceback (most recent call last): - ... - Failed: Expecting ZeroDivisionError - - - .. note:: - - When using ``pytest.raises`` as a context manager, it's worthwhile to - note that normal context manager rules apply and that the exception - raised *must* be the final line in the scope of the context manager. - Lines of code after that, within the scope of the context manager will - not be executed. For example:: - - >>> value = 15 - >>> with raises(ValueError) as exc_info: - ... if value > 10: - ... raise ValueError("value must be <= 10") - ... assert exc_info.type == ValueError # this will not execute - - Instead, the following approach must be taken (note the difference in - scope):: - - >>> with raises(ValueError) as exc_info: - ... if value > 10: - ... raise ValueError("value must be <= 10") - ... - >>> assert exc_info.type == ValueError - - Or you can use the keyword argument ``match`` to assert that the - exception matches a text or regex:: - - >>> with raises(ValueError, match='must be 0 or None'): - ... raise ValueError("value must be 0 or None") - - >>> with raises(ValueError, match=r'must be \d+$'): - ... raise ValueError("value must be 42") - - - Or you can specify a callable by passing a to-be-called lambda:: - - >>> raises(ZeroDivisionError, lambda: 1/0) - - - or you can specify an arbitrary callable with arguments:: - - >>> def f(x): return 1/x - ... - >>> raises(ZeroDivisionError, f, 0) - - >>> raises(ZeroDivisionError, f, x=0) - - - A third possibility is to use a string to be executed:: - - >>> raises(ZeroDivisionError, "f(0)") - - - .. autoclass:: _pytest._code.ExceptionInfo - :members: - - .. note:: - Similar to caught exception objects in Python, explicitly clearing - local references to returned ``ExceptionInfo`` objects can - help the Python interpreter speed up its garbage collection. - - Clearing those references breaks a reference cycle - (``ExceptionInfo`` --> caught exception --> frame stack raising - the exception --> current frame stack --> local variables --> - ``ExceptionInfo``) which makes Python keep all objects referenced - from that cycle (including all local variables in the current - frame) alive until the next cyclic garbage collection run. See the - official Python ``try`` statement documentation for more detailed - information. - - """ - __tracebackhide__ = True - msg = ("exceptions must be old-style classes or" - " derived from BaseException, not %s") - if isinstance(expected_exception, tuple): - for exc in expected_exception: - if not isclass(exc): - raise TypeError(msg % type(exc)) - elif not isclass(expected_exception): - raise TypeError(msg % type(expected_exception)) - - message = "DID NOT RAISE {0}".format(expected_exception) - match_expr = None - - if not args: - if "message" in kwargs: - message = kwargs.pop("message") - if "match" in kwargs: - match_expr = kwargs.pop("match") - message += " matching '{0}'".format(match_expr) - return RaisesContext(expected_exception, message, match_expr) - elif isinstance(args[0], str): - code, = args - assert isinstance(code, str) - frame = sys._getframe(1) - loc = frame.f_locals.copy() - loc.update(kwargs) - #print "raises frame scope: %r" % frame.f_locals - try: - code = _pytest._code.Source(code).compile() - py.builtin.exec_(code, frame.f_globals, loc) - # XXX didn'T mean f_globals == f_locals something special? - # this is destroyed here ... - except expected_exception: - return _pytest._code.ExceptionInfo() - else: - func = args[0] - try: - func(*args[1:], **kwargs) - except expected_exception: - return _pytest._code.ExceptionInfo() - fail(message) - - -raises.Exception = fail.Exception - - -class RaisesContext(object): - def __init__(self, expected_exception, message, match_expr): - self.expected_exception = expected_exception - self.message = message - self.match_expr = match_expr - self.excinfo = None - - def __enter__(self): - self.excinfo = object.__new__(_pytest._code.ExceptionInfo) - return self.excinfo - - def __exit__(self, *tp): - __tracebackhide__ = True - if tp[0] is None: - fail(self.message) - if sys.version_info < (2, 7): - # py26: on __exit__() exc_value often does not contain the - # exception value. - # http://bugs.python.org/issue7853 - if not isinstance(tp[1], BaseException): - exc_type, value, traceback = tp - tp = exc_type, exc_type(value), traceback - self.excinfo.__init__(tp) - suppress_exception = issubclass(self.excinfo.type, self.expected_exception) - if sys.version_info[0] == 2 and suppress_exception: - sys.exc_clear() - if self.match_expr: - self.excinfo.match(self.match_expr) - return suppress_exception - # # the basic pytest Function item diff --git a/_pytest/python_api.py b/_pytest/python_api.py new file mode 100644 index 000000000..1b27ba327 --- /dev/null +++ b/_pytest/python_api.py @@ -0,0 +1,430 @@ +import math +import sys + +import py + +from _pytest.compat import isclass +from _pytest.runner import fail +import _pytest._code +# builtin pytest.approx helper + + +class approx(object): + """ + Assert that two numbers (or two sets of numbers) are equal to each other + within some tolerance. + + Due to the `intricacies of floating-point arithmetic`__, numbers that we + would intuitively expect to be equal are not always so:: + + >>> 0.1 + 0.2 == 0.3 + False + + __ https://docs.python.org/3/tutorial/floatingpoint.html + + This problem is commonly encountered when writing tests, e.g. when making + sure that floating-point values are what you expect them to be. One way to + deal with this problem is to assert that two floating-point numbers are + equal to within some appropriate tolerance:: + + >>> abs((0.1 + 0.2) - 0.3) < 1e-6 + True + + However, comparisons like this are tedious to write and difficult to + understand. Furthermore, absolute comparisons like the one above are + usually discouraged because there's no tolerance that works well for all + situations. ``1e-6`` is good for numbers around ``1``, but too small for + very big numbers and too big for very small ones. It's better to express + the tolerance as a fraction of the expected value, but relative comparisons + like that are even more difficult to write correctly and concisely. + + The ``approx`` class performs floating-point comparisons using a syntax + that's as intuitive as possible:: + + >>> from pytest import approx + >>> 0.1 + 0.2 == approx(0.3) + True + + The same syntax also works on sequences of numbers:: + + >>> (0.1 + 0.2, 0.2 + 0.4) == approx((0.3, 0.6)) + True + + By default, ``approx`` considers numbers within a relative tolerance of + ``1e-6`` (i.e. one part in a million) of its expected value to be equal. + This treatment would lead to surprising results if the expected value was + ``0.0``, because nothing but ``0.0`` itself is relatively close to ``0.0``. + To handle this case less surprisingly, ``approx`` also considers numbers + within an absolute tolerance of ``1e-12`` of its expected value to be + equal. Infinite numbers are another special case. They are only + considered equal to themselves, regardless of the relative tolerance. Both + the relative and absolute tolerances can be changed by passing arguments to + the ``approx`` constructor:: + + >>> 1.0001 == approx(1) + False + >>> 1.0001 == approx(1, rel=1e-3) + True + >>> 1.0001 == approx(1, abs=1e-3) + True + + If you specify ``abs`` but not ``rel``, the comparison will not consider + the relative tolerance at all. In other words, two numbers that are within + the default relative tolerance of ``1e-6`` will still be considered unequal + if they exceed the specified absolute tolerance. If you specify both + ``abs`` and ``rel``, the numbers will be considered equal if either + tolerance is met:: + + >>> 1 + 1e-8 == approx(1) + True + >>> 1 + 1e-8 == approx(1, abs=1e-12) + False + >>> 1 + 1e-8 == approx(1, rel=1e-6, abs=1e-12) + True + + If you're thinking about using ``approx``, then you might want to know how + it compares to other good ways of comparing floating-point numbers. All of + these algorithms are based on relative and absolute tolerances and should + agree for the most part, but they do have meaningful differences: + + - ``math.isclose(a, b, rel_tol=1e-9, abs_tol=0.0)``: True if the relative + tolerance is met w.r.t. either ``a`` or ``b`` or if the absolute + tolerance is met. Because the relative tolerance is calculated w.r.t. + both ``a`` and ``b``, this test is symmetric (i.e. neither ``a`` nor + ``b`` is a "reference value"). You have to specify an absolute tolerance + if you want to compare to ``0.0`` because there is no tolerance by + default. Only available in python>=3.5. `More information...`__ + + __ https://docs.python.org/3/library/math.html#math.isclose + + - ``numpy.isclose(a, b, rtol=1e-5, atol=1e-8)``: True if the difference + between ``a`` and ``b`` is less that the sum of the relative tolerance + w.r.t. ``b`` and the absolute tolerance. Because the relative tolerance + is only calculated w.r.t. ``b``, this test is asymmetric and you can + think of ``b`` as the reference value. Support for comparing sequences + is provided by ``numpy.allclose``. `More information...`__ + + __ http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.isclose.html + + - ``unittest.TestCase.assertAlmostEqual(a, b)``: True if ``a`` and ``b`` + are within an absolute tolerance of ``1e-7``. No relative tolerance is + considered and the absolute tolerance cannot be changed, so this function + is not appropriate for very large or very small numbers. Also, it's only + available in subclasses of ``unittest.TestCase`` and it's ugly because it + doesn't follow PEP8. `More information...`__ + + __ https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertAlmostEqual + + - ``a == pytest.approx(b, rel=1e-6, abs=1e-12)``: True if the relative + tolerance is met w.r.t. ``b`` or if the absolute tolerance is met. + Because the relative tolerance is only calculated w.r.t. ``b``, this test + is asymmetric and you can think of ``b`` as the reference value. In the + special case that you explicitly specify an absolute tolerance but not a + relative tolerance, only the absolute tolerance is considered. + """ + + def __init__(self, expected, rel=None, abs=None): + self.expected = expected + self.abs = abs + self.rel = rel + + def __repr__(self): + return ', '.join(repr(x) for x in self.expected) + + def __eq__(self, actual): + from collections import Iterable + if not isinstance(actual, Iterable): + actual = [actual] + if len(actual) != len(self.expected): + return False + return all(a == x for a, x in zip(actual, self.expected)) + + __hash__ = None + + def __ne__(self, actual): + return not (actual == self) + + @property + def expected(self): + # Regardless of whether the user-specified expected value is a number + # or a sequence of numbers, return a list of ApproxNotIterable objects + # that can be compared against. + from collections import Iterable + approx_non_iter = lambda x: ApproxNonIterable(x, self.rel, self.abs) + if isinstance(self._expected, Iterable): + return [approx_non_iter(x) for x in self._expected] + else: + return [approx_non_iter(self._expected)] + + @expected.setter + def expected(self, expected): + self._expected = expected + + +class ApproxNonIterable(object): + """ + Perform approximate comparisons for single numbers only. + + In other words, the ``expected`` attribute for objects of this class must + be some sort of number. This is in contrast to the ``approx`` class, where + the ``expected`` attribute can either be a number of a sequence of numbers. + This class is responsible for making comparisons, while ``approx`` is + responsible for abstracting the difference between numbers and sequences of + numbers. Although this class can stand on its own, it's only meant to be + used within ``approx``. + """ + + def __init__(self, expected, rel=None, abs=None): + self.expected = expected + self.abs = abs + self.rel = rel + + def __repr__(self): + if isinstance(self.expected, complex): + return str(self.expected) + + # Infinities aren't compared using tolerances, so don't show a + # tolerance. + if math.isinf(self.expected): + return str(self.expected) + + # If a sensible tolerance can't be calculated, self.tolerance will + # raise a ValueError. In this case, display '???'. + try: + vetted_tolerance = '{:.1e}'.format(self.tolerance) + except ValueError: + vetted_tolerance = '???' + + if sys.version_info[0] == 2: + return '{0} +- {1}'.format(self.expected, vetted_tolerance) + else: + return u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) + + def __eq__(self, actual): + # Short-circuit exact equality. + if actual == self.expected: + return True + + # Infinity shouldn't be approximately equal to anything but itself, but + # if there's a relative tolerance, it will be infinite and infinity + # will seem approximately equal to everything. The equal-to-itself + # case would have been short circuited above, so here we can just + # return false if the expected value is infinite. The abs() call is + # for compatibility with complex numbers. + if math.isinf(abs(self.expected)): + return False + + # Return true if the two numbers are within the tolerance. + return abs(self.expected - actual) <= self.tolerance + + __hash__ = None + + def __ne__(self, actual): + return not (actual == self) + + @property + def tolerance(self): + set_default = lambda x, default: x if x is not None else default + + # Figure out what the absolute tolerance should be. ``self.abs`` is + # either None or a value specified by the user. + absolute_tolerance = set_default(self.abs, 1e-12) + + if absolute_tolerance < 0: + raise ValueError("absolute tolerance can't be negative: {}".format(absolute_tolerance)) + if math.isnan(absolute_tolerance): + raise ValueError("absolute tolerance can't be NaN.") + + # If the user specified an absolute tolerance but not a relative one, + # just return the absolute tolerance. + if self.rel is None: + if self.abs is not None: + return absolute_tolerance + + # Figure out what the relative tolerance should be. ``self.rel`` is + # either None or a value specified by the user. This is done after + # we've made sure the user didn't ask for an absolute tolerance only, + # because we don't want to raise errors about the relative tolerance if + # we aren't even going to use it. + relative_tolerance = set_default(self.rel, 1e-6) * abs(self.expected) + + if relative_tolerance < 0: + raise ValueError("relative tolerance can't be negative: {}".format(absolute_tolerance)) + if math.isnan(relative_tolerance): + raise ValueError("relative tolerance can't be NaN.") + + # Return the larger of the relative and absolute tolerances. + return max(relative_tolerance, absolute_tolerance) + +# builtin pytest.raises helper + +def raises(expected_exception, *args, **kwargs): + """ + Assert that a code block/function call raises ``expected_exception`` + and raise a failure exception otherwise. + + This helper produces a ``ExceptionInfo()`` object (see below). + + If using Python 2.5 or above, you may use this function as a + context manager:: + + >>> with raises(ZeroDivisionError): + ... 1/0 + + .. versionchanged:: 2.10 + + In the context manager form you may use the keyword argument + ``message`` to specify a custom failure message:: + + >>> with raises(ZeroDivisionError, message="Expecting ZeroDivisionError"): + ... pass + Traceback (most recent call last): + ... + Failed: Expecting ZeroDivisionError + + + .. note:: + + When using ``pytest.raises`` as a context manager, it's worthwhile to + note that normal context manager rules apply and that the exception + raised *must* be the final line in the scope of the context manager. + Lines of code after that, within the scope of the context manager will + not be executed. For example:: + + >>> value = 15 + >>> with raises(ValueError) as exc_info: + ... if value > 10: + ... raise ValueError("value must be <= 10") + ... assert exc_info.type == ValueError # this will not execute + + Instead, the following approach must be taken (note the difference in + scope):: + + >>> with raises(ValueError) as exc_info: + ... if value > 10: + ... raise ValueError("value must be <= 10") + ... + >>> assert exc_info.type == ValueError + + Or you can use the keyword argument ``match`` to assert that the + exception matches a text or regex:: + + >>> with raises(ValueError, match='must be 0 or None'): + ... raise ValueError("value must be 0 or None") + + >>> with raises(ValueError, match=r'must be \d+$'): + ... raise ValueError("value must be 42") + + + Or you can specify a callable by passing a to-be-called lambda:: + + >>> raises(ZeroDivisionError, lambda: 1/0) + + + or you can specify an arbitrary callable with arguments:: + + >>> def f(x): return 1/x + ... + >>> raises(ZeroDivisionError, f, 0) + + >>> raises(ZeroDivisionError, f, x=0) + + + A third possibility is to use a string to be executed:: + + >>> raises(ZeroDivisionError, "f(0)") + + + .. autoclass:: _pytest._code.ExceptionInfo + :members: + + .. note:: + Similar to caught exception objects in Python, explicitly clearing + local references to returned ``ExceptionInfo`` objects can + help the Python interpreter speed up its garbage collection. + + Clearing those references breaks a reference cycle + (``ExceptionInfo`` --> caught exception --> frame stack raising + the exception --> current frame stack --> local variables --> + ``ExceptionInfo``) which makes Python keep all objects referenced + from that cycle (including all local variables in the current + frame) alive until the next cyclic garbage collection run. See the + official Python ``try`` statement documentation for more detailed + information. + + """ + __tracebackhide__ = True + msg = ("exceptions must be old-style classes or" + " derived from BaseException, not %s") + if isinstance(expected_exception, tuple): + for exc in expected_exception: + if not isclass(exc): + raise TypeError(msg % type(exc)) + elif not isclass(expected_exception): + raise TypeError(msg % type(expected_exception)) + + message = "DID NOT RAISE {0}".format(expected_exception) + match_expr = None + + if not args: + if "message" in kwargs: + message = kwargs.pop("message") + if "match" in kwargs: + match_expr = kwargs.pop("match") + message += " matching '{0}'".format(match_expr) + return RaisesContext(expected_exception, message, match_expr) + elif isinstance(args[0], str): + code, = args + assert isinstance(code, str) + frame = sys._getframe(1) + loc = frame.f_locals.copy() + loc.update(kwargs) + #print "raises frame scope: %r" % frame.f_locals + try: + code = _pytest._code.Source(code).compile() + py.builtin.exec_(code, frame.f_globals, loc) + # XXX didn'T mean f_globals == f_locals something special? + # this is destroyed here ... + except expected_exception: + return _pytest._code.ExceptionInfo() + else: + func = args[0] + try: + func(*args[1:], **kwargs) + except expected_exception: + return _pytest._code.ExceptionInfo() + fail(message) + + +raises.Exception = fail.Exception + + +class RaisesContext(object): + def __init__(self, expected_exception, message, match_expr): + self.expected_exception = expected_exception + self.message = message + self.match_expr = match_expr + self.excinfo = None + + def __enter__(self): + self.excinfo = object.__new__(_pytest._code.ExceptionInfo) + return self.excinfo + + def __exit__(self, *tp): + __tracebackhide__ = True + if tp[0] is None: + fail(self.message) + if sys.version_info < (2, 7): + # py26: on __exit__() exc_value often does not contain the + # exception value. + # http://bugs.python.org/issue7853 + if not isinstance(tp[1], BaseException): + exc_type, value, traceback = tp + tp = exc_type, exc_type(value), traceback + self.excinfo.__init__(tp) + suppress_exception = issubclass(self.excinfo.type, self.expected_exception) + if sys.version_info[0] == 2 and suppress_exception: + sys.exc_clear() + if self.match_expr: + self.excinfo.match(self.match_expr) + return suppress_exception diff --git a/pytest.py b/pytest.py index e3c72023c..da6b64910 100644 --- a/pytest.py +++ b/pytest.py @@ -22,11 +22,10 @@ from _pytest.skipping import xfail from _pytest.main import Item, Collector, File, Session from _pytest.fixtures import fillfixtures as _fillfuncargs from _pytest.python import ( - raises, Module, Class, Instance, Function, Generator, ) -from _pytest.python_api import approx +from _pytest.python_api import approx, raises set_trace = __pytestPDB.set_trace From f8b2277413ad3b61d5c86039a3c8a0fa92480bc4 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 11 Jun 2017 14:16:08 +0200 Subject: [PATCH 03/73] changelog fragment --- changelog/2489.trivial | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/2489.trivial diff --git a/changelog/2489.trivial b/changelog/2489.trivial new file mode 100644 index 000000000..c997d7e1e --- /dev/null +++ b/changelog/2489.trivial @@ -0,0 +1 @@ +Internal code move: move code for pytest.approx/pytest.raises to own files in order to cut down the size of python.py \ No newline at end of file From 9f3122fec68488f2fdd41da40eef227372168ad4 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sun, 11 Jun 2017 19:27:41 -0700 Subject: [PATCH 04/73] Add support for numpy arrays (and dicts) to approx. This fixes #1994. It turned out to require a lot of refactoring because subclassing numpy.ndarray was necessary to coerce python into calling the right `__eq__` operator. --- _pytest/python.py | 428 +++++++++++++++++++++++++++------------ testing/python/approx.py | 151 ++++++++++---- 2 files changed, 408 insertions(+), 171 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 06f74ce4b..ea2cfd3c6 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1117,7 +1117,6 @@ def raises(expected_exception, *args, **kwargs): ... Failed: Expecting ZeroDivisionError - .. note:: When using ``pytest.raises`` as a context manager, it's worthwhile to @@ -1150,7 +1149,6 @@ def raises(expected_exception, *args, **kwargs): >>> with raises(ValueError, match=r'must be \d+$'): ... raise ValueError("value must be 42") - Or you can specify a callable by passing a to-be-called lambda:: >>> raises(ZeroDivisionError, lambda: 1/0) @@ -1230,10 +1228,8 @@ def raises(expected_exception, *args, **kwargs): return _pytest._code.ExceptionInfo() fail(message) - raises.Exception = fail.Exception - class RaisesContext(object): def __init__(self, expected_exception, message, match_expr): self.expected_exception = expected_exception @@ -1265,9 +1261,271 @@ class RaisesContext(object): return suppress_exception + # builtin pytest.approx helper -class approx(object): +class ApproxBase(object): + """ + Provide shared utilities for making approximate comparisons between numbers + or sequences of numbers. + """ + + def __init__(self, expected, rel=None, abs=None): + self.expected = expected + self.abs = abs + self.rel = rel + + def __repr__(self): + return ', '.join( + repr(self._approx_scalar(x)) + for x in self._yield_expected()) + + def __eq__(self, actual): + return all( + a == self._approx_scalar(x) + for a, x in self._yield_comparisons(actual)) + + __hash__ = None + + def __ne__(self, actual): + return not (actual == self) + + def _approx_scalar(self, x): + return ApproxScalar(x, rel=self.rel, abs=self.abs) + + def _yield_expected(self, actual): + """ + Yield all the expected values associated with this object. This is + used to implement the `__repr__` method. + """ + raise NotImplementedError + + def _yield_comparisons(self, actual): + """ + Yield all the pairs of numbers to be compared. This is used to + implement the `__eq__` method. + """ + raise NotImplementedError + + + +try: + import numpy as np + + class ApproxNumpy(ApproxBase, np.ndarray): + """ + Perform approximate comparisons for numpy arrays. + + This class must inherit from numpy.ndarray in order to allow the approx + to be on either side of the `==` operator. The reason for this has to + do with how python decides whether to call `a.__eq__()` or `b.__eq__()` + when it encounters `a == b`. + + If `a` and `b` are not related by inheritance, `a` gets priority. So + as long as `a.__eq__` is defined, it will be called. Because most + implementations of `a.__eq__` end up calling `b.__eq__`, this detail + usually doesn't matter. However, `numpy.ndarray.__eq__` raises an + error complaining that "the truth value of an array with more than + one element is ambiguous. Use a.any() or a.all()" when compared with a + custom class, so `b.__eq__` never gets called. + + The trick is that the priority rules change if `a` and `b` are related + by inheritance. Specifically, `b.__eq__` gets priority if `b` is a + subclass of `a`. So we can guarantee that `ApproxNumpy.__eq__` gets + called by inheriting from `numpy.ndarray`. + """ + + def __new__(cls, expected, rel=None, abs=None): + """ + Numpy uses __new__ (rather than __init__) to initialize objects. + + The `expected` argument must be a numpy array. This should be + ensured by the approx() delegator function. + """ + assert isinstance(expected, np.ndarray) + obj = super(ApproxNumpy, cls).__new__(cls, expected.shape) + obj.__init__(expected, rel, abs) + return obj + + def __repr__(self): + # It might be nice to rewrite this function to account for the + # shape of the array... + return '[' + ApproxBase.__repr__(self) + ']' + + def __eq__(self, actual): + try: + actual = np.array(actual) + except: + raise ValueError("cannot cast '{0}' to numpy.ndarray".format(actual)) + + if actual.shape != self.expected.shape: + return False + + return ApproxBase.__eq__(self, actual) + + def _yield_expected(self): + for x in self.expected: + yield x + + def _yield_comparisons(self, actual): + # We can be sure that `actual` is a numpy array, because it's + # casted in `__eq__` before being passed to `ApproxBase.__eq__`, + # which is the only method that calls this one. + for i in np.ndindex(self.expected.shape): + yield actual[i], self.expected[i] + + +except ImportError: + np = None + +class ApproxMapping(ApproxBase): + """ + Perform approximate comparisons for mappings where the values are numbers + (the keys can be anything). + """ + + def __repr__(self): + item = lambda k, v: "'{0}': {1}".format(k, self._approx_scalar(v)) + return '{' + ', '.join(item(k,v) for k,v in self.expected.items()) + '}' + + def __eq__(self, actual): + if actual.keys() != self.expected.keys(): + return False + + return ApproxBase.__eq__(self, actual) + + def _yield_comparisons(self, actual): + for k in self.expected.keys(): + yield actual[k], self.expected[k] + + +class ApproxSequence(ApproxBase): + """ + Perform approximate comparisons for sequences of numbers. + """ + + def __repr__(self): + open, close = '()' if isinstance(self.expected, tuple) else '[]' + return open + ApproxBase.__repr__(self) + close + + def __eq__(self, actual): + if len(actual) != len(self.expected): + return False + return ApproxBase.__eq__(self, actual) + + def _yield_expected(self): + return iter(self.expected) + + def _yield_comparisons(self, actual): + return zip(actual, self.expected) + + +class ApproxScalar(ApproxBase): + """ + Perform approximate comparisons for single numbers only. + """ + + def __repr__(self): + """ + Return a string communicating both the expected value and the tolerance + for the comparison being made, e.g. '1.0 +- 1e-6'. Use the unicode + plus/minus symbol if this is python3 (it's too hard to get right for + python2). + """ + if isinstance(self.expected, complex): + return str(self.expected) + + # Infinities aren't compared using tolerances, so don't show a + # tolerance. + if math.isinf(self.expected): + return str(self.expected) + + # If a sensible tolerance can't be calculated, self.tolerance will + # raise a ValueError. In this case, display '???'. + try: + vetted_tolerance = '{:.1e}'.format(self.tolerance) + except ValueError: + vetted_tolerance = '???' + + if sys.version_info[0] == 2: + return '{0} +- {1}'.format(self.expected, vetted_tolerance) + else: + return u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) + + def __eq__(self, actual): + """ + Return true if the given value is equal to the expected value within + the pre-specified tolerance. + """ + from numbers import Number + + # Give a good error message we get values to compare that aren't + # numbers, rather than choking on them later on. + if not isinstance(actual, Number): + raise ValueError("approx can only compare numbers, not '{0}'".format(actual)) + if not isinstance(self.expected, Number): + raise ValueError("approx can only compare numbers, not '{0}'".format(self.expected)) + + # Short-circuit exact equality. + if actual == self.expected: + return True + + # Infinity shouldn't be approximately equal to anything but itself, but + # if there's a relative tolerance, it will be infinite and infinity + # will seem approximately equal to everything. The equal-to-itself + # case would have been short circuited above, so here we can just + # return false if the expected value is infinite. The abs() call is + # for compatibility with complex numbers. + if math.isinf(abs(self.expected)): + return False + + # Return true if the two numbers are within the tolerance. + return abs(self.expected - actual) <= self.tolerance + + __hash__ = None + + @property + def tolerance(self): + """ + Return the tolerance for the comparison. This could be either an + absolute tolerance or a relative tolerance, depending on what the user + specified or which would be larger. + """ + set_default = lambda x, default: x if x is not None else default + + # Figure out what the absolute tolerance should be. ``self.abs`` is + # either None or a value specified by the user. + absolute_tolerance = set_default(self.abs, 1e-12) + + if absolute_tolerance < 0: + raise ValueError("absolute tolerance can't be negative: {}".format(absolute_tolerance)) + if math.isnan(absolute_tolerance): + raise ValueError("absolute tolerance can't be NaN.") + + # If the user specified an absolute tolerance but not a relative one, + # just return the absolute tolerance. + if self.rel is None: + if self.abs is not None: + return absolute_tolerance + + # Figure out what the relative tolerance should be. ``self.rel`` is + # either None or a value specified by the user. This is done after + # we've made sure the user didn't ask for an absolute tolerance only, + # because we don't want to raise errors about the relative tolerance if + # we aren't even going to use it. + relative_tolerance = set_default(self.rel, 1e-6) * abs(self.expected) + + if relative_tolerance < 0: + raise ValueError("relative tolerance can't be negative: {}".format(absolute_tolerance)) + if math.isnan(relative_tolerance): + raise ValueError("relative tolerance can't be NaN.") + + # Return the larger of the relative and absolute tolerances. + return max(relative_tolerance, absolute_tolerance) + + + +def approx(expected, rel=None, abs=None): """ Assert that two numbers (or two sets of numbers) are equal to each other within some tolerance. @@ -1307,6 +1565,8 @@ class approx(object): >>> (0.1 + 0.2, 0.2 + 0.4) == approx((0.3, 0.6)) True + >>> {'a': 0.1 + 0.2, 'b': 0.2 + 0.4} == approx({'a': 0.3, 'b': 0.6}) + True By default, ``approx`` considers numbers within a relative tolerance of ``1e-6`` (i.e. one part in a million) of its expected value to be equal. @@ -1380,139 +1640,37 @@ class approx(object): special case that you explicitly specify an absolute tolerance but not a relative tolerance, only the absolute tolerance is considered. """ + + from collections import Mapping, Sequence + try: + String = basestring # python2 + except NameError: + String = str, bytes # python3 - def __init__(self, expected, rel=None, abs=None): - self.expected = expected - self.abs = abs - self.rel = rel + # Delegate the comparison to a class that knows how to deal with the type + # of the expected value (e.g. int, float, list, dict, numpy.array, etc). + # + # This architecture is really driven by the need to support numpy arrays. + # The only way to override `==` for arrays without requiring that approx be + # the left operand is to inherit the approx object from `numpy.ndarray`. + # But that can't be a general solution, because it requires (1) numpy to be + # installed and (2) the expected value to be a numpy array. So the general + # solution is to delegate each type of expected value to a different class. + # + # This has the advantage that it made it easy to support mapping types + # (i.e. dict). The old code accepted mapping types, but would only compare + # their keys, which is probably not what most people would expect. - def __repr__(self): - return ', '.join(repr(x) for x in self.expected) + if np and isinstance(expected, np.ndarray): + cls = ApproxNumpy + elif isinstance(expected, Mapping): + cls = ApproxMapping + elif isinstance(expected, Sequence) and not isinstance(expected, String): + cls = ApproxSequence + else: + cls = ApproxScalar - def __eq__(self, actual): - from collections import Iterable - if not isinstance(actual, Iterable): - actual = [actual] - if len(actual) != len(self.expected): - return False - return all(a == x for a, x in zip(actual, self.expected)) - - __hash__ = None - - def __ne__(self, actual): - return not (actual == self) - - @property - def expected(self): - # Regardless of whether the user-specified expected value is a number - # or a sequence of numbers, return a list of ApproxNotIterable objects - # that can be compared against. - from collections import Iterable - approx_non_iter = lambda x: ApproxNonIterable(x, self.rel, self.abs) - if isinstance(self._expected, Iterable): - return [approx_non_iter(x) for x in self._expected] - else: - return [approx_non_iter(self._expected)] - - @expected.setter - def expected(self, expected): - self._expected = expected - - -class ApproxNonIterable(object): - """ - Perform approximate comparisons for single numbers only. - - In other words, the ``expected`` attribute for objects of this class must - be some sort of number. This is in contrast to the ``approx`` class, where - the ``expected`` attribute can either be a number of a sequence of numbers. - This class is responsible for making comparisons, while ``approx`` is - responsible for abstracting the difference between numbers and sequences of - numbers. Although this class can stand on its own, it's only meant to be - used within ``approx``. - """ - - def __init__(self, expected, rel=None, abs=None): - self.expected = expected - self.abs = abs - self.rel = rel - - def __repr__(self): - if isinstance(self.expected, complex): - return str(self.expected) - - # Infinities aren't compared using tolerances, so don't show a - # tolerance. - if math.isinf(self.expected): - return str(self.expected) - - # If a sensible tolerance can't be calculated, self.tolerance will - # raise a ValueError. In this case, display '???'. - try: - vetted_tolerance = '{:.1e}'.format(self.tolerance) - except ValueError: - vetted_tolerance = '???' - - if sys.version_info[0] == 2: - return '{0} +- {1}'.format(self.expected, vetted_tolerance) - else: - return u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) - - def __eq__(self, actual): - # Short-circuit exact equality. - if actual == self.expected: - return True - - # Infinity shouldn't be approximately equal to anything but itself, but - # if there's a relative tolerance, it will be infinite and infinity - # will seem approximately equal to everything. The equal-to-itself - # case would have been short circuited above, so here we can just - # return false if the expected value is infinite. The abs() call is - # for compatibility with complex numbers. - if math.isinf(abs(self.expected)): - return False - - # Return true if the two numbers are within the tolerance. - return abs(self.expected - actual) <= self.tolerance - - __hash__ = None - - def __ne__(self, actual): - return not (actual == self) - - @property - def tolerance(self): - set_default = lambda x, default: x if x is not None else default - - # Figure out what the absolute tolerance should be. ``self.abs`` is - # either None or a value specified by the user. - absolute_tolerance = set_default(self.abs, 1e-12) - - if absolute_tolerance < 0: - raise ValueError("absolute tolerance can't be negative: {}".format(absolute_tolerance)) - if math.isnan(absolute_tolerance): - raise ValueError("absolute tolerance can't be NaN.") - - # If the user specified an absolute tolerance but not a relative one, - # just return the absolute tolerance. - if self.rel is None: - if self.abs is not None: - return absolute_tolerance - - # Figure out what the relative tolerance should be. ``self.rel`` is - # either None or a value specified by the user. This is done after - # we've made sure the user didn't ask for an absolute tolerance only, - # because we don't want to raise errors about the relative tolerance if - # we aren't even going to use it. - relative_tolerance = set_default(self.rel, 1e-6) * abs(self.expected) - - if relative_tolerance < 0: - raise ValueError("relative tolerance can't be negative: {}".format(absolute_tolerance)) - if math.isnan(relative_tolerance): - raise ValueError("relative tolerance can't be NaN.") - - # Return the larger of the relative and absolute tolerances. - return max(relative_tolerance, absolute_tolerance) + return cls(expected, rel, abs) # diff --git a/testing/python/approx.py b/testing/python/approx.py index d7063e215..8b9605a00 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -9,7 +9,6 @@ from decimal import Decimal from fractions import Fraction inf, nan = float('inf'), float('nan') - class MyDocTestRunner(doctest.DocTestRunner): def __init__(self): @@ -29,12 +28,19 @@ class TestApprox(object): if sys.version_info[:2] == (2, 6): tol1, tol2, infr = '???', '???', '???' assert repr(approx(1.0)) == '1.0 {pm} {tol1}'.format(pm=plus_minus, tol1=tol1) - assert repr(approx([1.0, 2.0])) == '1.0 {pm} {tol1}, 2.0 {pm} {tol2}'.format(pm=plus_minus, tol1=tol1, tol2=tol2) + assert repr(approx([1.0, 2.0])) == '[1.0 {pm} {tol1}, 2.0 {pm} {tol2}]'.format(pm=plus_minus, tol1=tol1, tol2=tol2) + assert repr(approx((1.0, 2.0))) == '(1.0 {pm} {tol1}, 2.0 {pm} {tol2})'.format(pm=plus_minus, tol1=tol1, tol2=tol2) assert repr(approx(inf)) == 'inf' assert repr(approx(1.0, rel=nan)) == '1.0 {pm} ???'.format(pm=plus_minus) assert repr(approx(1.0, rel=inf)) == '1.0 {pm} {infr}'.format(pm=plus_minus, infr=infr) assert repr(approx(1.0j, rel=inf)) == '1j' + # Dictionaries aren't ordered, so we need to check both orders. + assert repr(approx({'a': 1.0, 'b': 2.0})) in ( + "{{'a': 1.0 {pm} {tol1}, 'b': 2.0 {pm} {tol2}}}".format(pm=plus_minus, tol1=tol1, tol2=tol2), + "{{'b': 2.0 {pm} {tol2}, 'a': 1.0 {pm} {tol1}}}".format(pm=plus_minus, tol1=tol1, tol2=tol2), + ) + def test_operator_overloading(self): assert 1 == approx(1, rel=1e-6, abs=1e-12) assert not (1 != approx(1, rel=1e-6, abs=1e-12)) @@ -228,18 +234,38 @@ class TestApprox(object): # tolerance, so only an absolute tolerance is calculated. assert a != approx(x, abs=inf) - def test_expecting_sequence(self): - within_1e8 = [ - (1e8 + 1e0, 1e8), - (1e0 + 1e-8, 1e0), - (1e-8 + 1e-16, 1e-8), + def test_int(self): + within_1e6 = [ + (1000001, 1000000), + (-1000001, -1000000), ] - actual, expected = zip(*within_1e8) - assert actual == approx(expected, rel=5e-8, abs=0.0) + for a, x in within_1e6: + assert a == approx(x, rel=5e-6, abs=0) + assert a != approx(x, rel=5e-7, abs=0) + assert approx(x, rel=5e-6, abs=0) == a + assert approx(x, rel=5e-7, abs=0) != a - def test_expecting_sequence_wrong_len(self): - assert [1, 2] != approx([1]) - assert [1, 2] != approx([1,2,3]) + def test_decimal(self): + within_1e6 = [ + (Decimal('1.000001'), Decimal('1.0')), + (Decimal('-1.000001'), Decimal('-1.0')), + ] + for a, x in within_1e6: + assert a == approx(x, rel=Decimal('5e-6'), abs=0) + assert a != approx(x, rel=Decimal('5e-7'), abs=0) + assert approx(x, rel=Decimal('5e-6'), abs=0) == a + assert approx(x, rel=Decimal('5e-7'), abs=0) != a + + def test_fraction(self): + within_1e6 = [ + (1 + Fraction(1, 1000000), Fraction(1)), + (-1 - Fraction(-1, 1000000), Fraction(-1)), + ] + for a, x in within_1e6: + assert a == approx(x, rel=5e-6, abs=0) + assert a != approx(x, rel=5e-7, abs=0) + assert approx(x, rel=5e-6, abs=0) == a + assert approx(x, rel=5e-7, abs=0) != a def test_complex(self): within_1e6 = [ @@ -251,33 +277,86 @@ class TestApprox(object): for a, x in within_1e6: assert a == approx(x, rel=5e-6, abs=0) assert a != approx(x, rel=5e-7, abs=0) + assert approx(x, rel=5e-6, abs=0) == a + assert approx(x, rel=5e-7, abs=0) != a - def test_int(self): - within_1e6 = [ - (1000001, 1000000), - (-1000001, -1000000), - ] - for a, x in within_1e6: - assert a == approx(x, rel=5e-6, abs=0) - assert a != approx(x, rel=5e-7, abs=0) + def test_list(self): + actual = [1 + 1e-7, 2 + 1e-8] + expected = [1, 2] - def test_decimal(self): - within_1e6 = [ - (Decimal('1.000001'), Decimal('1.0')), - (Decimal('-1.000001'), Decimal('-1.0')), - ] - for a, x in within_1e6: - assert a == approx(x, rel=Decimal('5e-6'), abs=0) - assert a != approx(x, rel=Decimal('5e-7'), abs=0) + # Return false if any element is outside the tolerance. + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == actual + assert approx(expected, rel=5e-8, abs=0) != actual - def test_fraction(self): - within_1e6 = [ - (1 + Fraction(1, 1000000), Fraction(1)), - (-1 - Fraction(-1, 1000000), Fraction(-1)), - ] - for a, x in within_1e6: - assert a == approx(x, rel=5e-6, abs=0) - assert a != approx(x, rel=5e-7, abs=0) + def test_list_wrong_len(self): + assert [1, 2] != approx([1]) + assert [1, 2] != approx([1,2,3]) + + def test_tuple(self): + actual = (1 + 1e-7, 2 + 1e-8) + expected = (1, 2) + + # Return false if any element is outside the tolerance. + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == actual + assert approx(expected, rel=5e-8, abs=0) != actual + + def test_tuple_wrong_len(self): + assert (1, 2) != approx((1,)) + assert (1, 2) != approx((1,2,3)) + + def test_dict(self): + actual = {'a': 1 + 1e-7, 'b': 2 + 1e-8} + expected = {'b': 2, 'a': 1} # Dictionaries became ordered in python3.6, + # so make sure the order doesn't matter + + # Return false if any element is outside the tolerance. + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == actual + assert approx(expected, rel=5e-8, abs=0) != actual + + def test_dict_wrong_len(self): + assert {'a': 1, 'b': 2} != approx({'a': 1}) + assert {'a': 1, 'b': 2} != approx({'a': 1, 'c': 2}) + assert {'a': 1, 'b': 2} != approx({'a': 1, 'b': 2, 'c': 3}) + + def test_numpy_array(self): + try: + import numpy as np + except ImportError: + pytest.skip("numpy not installed") + + actual = np.array([1 + 1e-7, 2 + 1e-8]) + expected = np.array([1, 2]) + + # Return false if any element is outside the tolerance. + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == expected + assert approx(expected, rel=5e-8, abs=0) != actual + + def test_numpy_array_wrong_shape(self): + try: + import numpy as np + except ImportError: + pytest.skip("numpy not installed") + + import numpy as np + a12 = np.array([[1, 2]]) + a21 = np.array([[1],[2]]) + + assert a12 != approx(a21) + assert a21 != approx(a12) + + def test_non_number(self): + with pytest.raises(ValueError): + 1 == approx("1") + with pytest.raises(ValueError): + "1" == approx(1) def test_doctests(self): parser = doctest.DocTestParser() From 89292f08dc73ef9d20a3488098b07d794a4b3248 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sun, 11 Jun 2017 19:51:21 -0700 Subject: [PATCH 05/73] Add a changelog entry. --- changelog/1994.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/1994.feature diff --git a/changelog/1994.feature b/changelog/1994.feature new file mode 100644 index 000000000..f3c596e63 --- /dev/null +++ b/changelog/1994.feature @@ -0,0 +1 @@ +Add support for numpy arrays (and dicts) to approx. From 8badb47db60d43966a56cfe5f0630e38358c309a Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Thu, 15 Jun 2017 09:03:31 -0700 Subject: [PATCH 06/73] Implement suggestions from code review. - Avoid importing numpy unless necessary. - Mention numpy arrays and dictionaries in the docs. - Add numpy to the list of tox dependencies. - Don't unnecessarily copy arrays or allocate empty space for them. - Use code from compat.py rather than writing py2/3 versions of things myself. - Avoid reimplementing __repr__ for built-in types. - Add an option to consider NaN == NaN, because sometimes people use NaN to mean "missing data". --- _pytest/compat.py | 3 +- _pytest/python_api.py | 266 +++++++++++++++++++++------------------ testing/python/approx.py | 56 ++++----- tox.ini | 1 + 4 files changed, 174 insertions(+), 152 deletions(-) diff --git a/_pytest/compat.py b/_pytest/compat.py index 8c200af5f..0554efeb7 100644 --- a/_pytest/compat.py +++ b/_pytest/compat.py @@ -125,6 +125,7 @@ if sys.version_info[:2] == (2, 6): if _PY3: import codecs imap = map + izip = zip STRING_TYPES = bytes, str UNICODE_TYPES = str, @@ -160,7 +161,7 @@ else: STRING_TYPES = bytes, str, unicode UNICODE_TYPES = unicode, - from itertools import imap # NOQA + from itertools import imap, izip # NOQA def _escape_strings(val): """In py2 bytes and str are the same type, so return if it's a bytes diff --git a/_pytest/python_api.py b/_pytest/python_api.py index 029276c95..a2942b742 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -3,7 +3,7 @@ import sys import py -from _pytest.compat import isclass +from _pytest.compat import isclass, izip from _pytest.runner import fail import _pytest._code @@ -11,19 +11,18 @@ import _pytest._code class ApproxBase(object): """ - Provide shared utilities for making approximate comparisons between numbers + Provide shared utilities for making approximate comparisons between numbers or sequences of numbers. """ - def __init__(self, expected, rel=None, abs=None): + def __init__(self, expected, rel=None, abs=None, nan_ok=False): self.expected = expected self.abs = abs self.rel = rel + self.nan_ok = nan_ok def __repr__(self): - return ', '.join( - repr(self._approx_scalar(x)) - for x in self._yield_expected()) + raise NotImplementedError def __eq__(self, actual): return all( @@ -36,109 +35,109 @@ class ApproxBase(object): return not (actual == self) def _approx_scalar(self, x): - return ApproxScalar(x, rel=self.rel, abs=self.abs) - - def _yield_expected(self, actual): - """ - Yield all the expected values associated with this object. This is - used to implement the `__repr__` method. - """ - raise NotImplementedError + return ApproxScalar(x, rel=self.rel, abs=self.abs, nan_ok=self.nan_ok) def _yield_comparisons(self, actual): """ - Yield all the pairs of numbers to be compared. This is used to + Yield all the pairs of numbers to be compared. This is used to implement the `__eq__` method. """ raise NotImplementedError +class ApproxNumpyBase(ApproxBase): + """ + Perform approximate comparisons for numpy arrays. -try: - import numpy as np + This class should not be used directly. Instead, it should be used to make + a subclass that also inherits from `np.ndarray`, e.g.:: - class ApproxNumpy(ApproxBase, np.ndarray): + import numpy as np + ApproxNumpy = type('ApproxNumpy', (ApproxNumpyBase, np.ndarray), {}) + + This bizarre invocation is necessary because the object doing the + approximate comparison must inherit from `np.ndarray`, or it will only work + on the left side of the `==` operator. But importing numpy is relatively + expensive, so we also want to avoid that unless we actually have a numpy + array to compare. + + The reason why the approx object needs to inherit from `np.ndarray` has to + do with how python decides whether to call `a.__eq__()` or `b.__eq__()` + when it parses `a == b`. If `a` and `b` are not related by inheritance, + `a` gets priority. So as long as `a.__eq__` is defined, it will be called. + Because most implementations of `a.__eq__` end up calling `b.__eq__`, this + detail usually doesn't matter. However, `np.ndarray.__eq__` treats the + approx object as a scalar and builds a new array by comparing it to each + item in the original array. `b.__eq__` is called to compare against each + individual element in the array, but it has no way (that I can see) to + prevent the return value from being an boolean array, and boolean arrays + can't be used with assert because "the truth value of an array with more + than one element is ambiguous." + + The trick is that the priority rules change if `a` and `b` are related + by inheritance. Specifically, `b.__eq__` gets priority if `b` is a + subclass of `a`. So by inheriting from `np.ndarray`, we can guarantee that + `ApproxNumpy.__eq__` gets called no matter which side of the `==` operator + it appears on. + """ + + def __new__(cls, expected, rel=None, abs=None, nan_ok=False): """ - Perform approximate comparisons for numpy arrays. + Numpy uses __new__ (rather than __init__) to initialize objects. - This class must inherit from numpy.ndarray in order to allow the approx - to be on either side of the `==` operator. The reason for this has to - do with how python decides whether to call `a.__eq__()` or `b.__eq__()` - when it encounters `a == b`. - - If `a` and `b` are not related by inheritance, `a` gets priority. So - as long as `a.__eq__` is defined, it will be called. Because most - implementations of `a.__eq__` end up calling `b.__eq__`, this detail - usually doesn't matter. However, `numpy.ndarray.__eq__` raises an - error complaining that "the truth value of an array with more than - one element is ambiguous. Use a.any() or a.all()" when compared with a - custom class, so `b.__eq__` never gets called. - - The trick is that the priority rules change if `a` and `b` are related - by inheritance. Specifically, `b.__eq__` gets priority if `b` is a - subclass of `a`. So we can guarantee that `ApproxNumpy.__eq__` gets - called by inheriting from `numpy.ndarray`. + The `expected` argument must be a numpy array. This should be + ensured by the approx() delegator function. """ + obj = super(ApproxNumpyBase, cls).__new__(cls, ()) + obj.__init__(expected, rel, abs, nan_ok) + return obj - def __new__(cls, expected, rel=None, abs=None): - """ - Numpy uses __new__ (rather than __init__) to initialize objects. - - The `expected` argument must be a numpy array. This should be - ensured by the approx() delegator function. - """ - assert isinstance(expected, np.ndarray) - obj = super(ApproxNumpy, cls).__new__(cls, expected.shape) - obj.__init__(expected, rel, abs) - return obj + def __repr__(self): + # It might be nice to rewrite this function to account for the + # shape of the array... + return repr(list( + self._approx_scalar(x) for x in self.expected)) - def __repr__(self): - # It might be nice to rewrite this function to account for the - # shape of the array... - return '[' + ApproxBase.__repr__(self) + ']' + def __eq__(self, actual): + import numpy as np - def __eq__(self, actual): - try: - actual = np.array(actual) - except: - raise ValueError("cannot cast '{0}' to numpy.ndarray".format(actual)) + try: + actual = np.asarray(actual) + except: + raise ValueError("cannot cast '{0}' to numpy.ndarray".format(actual)) - if actual.shape != self.expected.shape: - return False + if actual.shape != self.expected.shape: + return False - return ApproxBase.__eq__(self, actual) + return ApproxBase.__eq__(self, actual) - def _yield_expected(self): - for x in self.expected: - yield x + def _yield_comparisons(self, actual): + import numpy as np - def _yield_comparisons(self, actual): - # We can be sure that `actual` is a numpy array, because it's - # casted in `__eq__` before being passed to `ApproxBase.__eq__`, - # which is the only method that calls this one. - for i in np.ndindex(self.expected.shape): - yield actual[i], self.expected[i] + # We can be sure that `actual` is a numpy array, because it's + # casted in `__eq__` before being passed to `ApproxBase.__eq__`, + # which is the only method that calls this one. + for i in np.ndindex(self.expected.shape): + yield actual[i], self.expected[i] -except ImportError: - np = None - class ApproxMapping(ApproxBase): """ - Perform approximate comparisons for mappings where the values are numbers + Perform approximate comparisons for mappings where the values are numbers (the keys can be anything). """ def __repr__(self): - item = lambda k, v: "'{0}': {1}".format(k, self._approx_scalar(v)) - return '{' + ', '.join(item(k,v) for k,v in self.expected.items()) + '}' + return repr({ + k: self._approx_scalar(v) + for k,v in self.expected.items()}) def __eq__(self, actual): if actual.keys() != self.expected.keys(): return False return ApproxBase.__eq__(self, actual) - + def _yield_comparisons(self, actual): for k in self.expected.keys(): yield actual[k], self.expected[k] @@ -150,19 +149,19 @@ class ApproxSequence(ApproxBase): """ def __repr__(self): - open, close = '()' if isinstance(self.expected, tuple) else '[]' - return open + ApproxBase.__repr__(self) + close + seq_type = type(self.expected) + if seq_type not in (tuple, list, set): + seq_type = list + return repr(seq_type( + self._approx_scalar(x) for x in self.expected)) def __eq__(self, actual): if len(actual) != len(self.expected): return False return ApproxBase.__eq__(self, actual) - - def _yield_expected(self): - return iter(self.expected) def _yield_comparisons(self, actual): - return zip(actual, self.expected) + return izip(actual, self.expected) class ApproxScalar(ApproxBase): @@ -172,9 +171,9 @@ class ApproxScalar(ApproxBase): def __repr__(self): """ - Return a string communicating both the expected value and the tolerance - for the comparison being made, e.g. '1.0 +- 1e-6'. Use the unicode - plus/minus symbol if this is python3 (it's too hard to get right for + Return a string communicating both the expected value and the tolerance + for the comparison being made, e.g. '1.0 +- 1e-6'. Use the unicode + plus/minus symbol if this is python3 (it's too hard to get right for python2). """ if isinstance(self.expected, complex): @@ -199,22 +198,20 @@ class ApproxScalar(ApproxBase): def __eq__(self, actual): """ - Return true if the given value is equal to the expected value within + Return true if the given value is equal to the expected value within the pre-specified tolerance. """ - from numbers import Number - - # Give a good error message we get values to compare that aren't - # numbers, rather than choking on them later on. - if not isinstance(actual, Number): - raise ValueError("approx can only compare numbers, not '{0}'".format(actual)) - if not isinstance(self.expected, Number): - raise ValueError("approx can only compare numbers, not '{0}'".format(self.expected)) # Short-circuit exact equality. if actual == self.expected: return True + # Allow the user to control whether NaNs are considered equal to each + # other or not. The abs() calls are for compatibility with complex + # numbers. + if math.isnan(abs(self.expected)): + return self.nan_ok and math.isnan(abs(actual)) + # Infinity shouldn't be approximately equal to anything but itself, but # if there's a relative tolerance, it will be infinite and infinity # will seem approximately equal to everything. The equal-to-itself @@ -232,8 +229,8 @@ class ApproxScalar(ApproxBase): @property def tolerance(self): """ - Return the tolerance for the comparison. This could be either an - absolute tolerance or a relative tolerance, depending on what the user + Return the tolerance for the comparison. This could be either an + absolute tolerance or a relative tolerance, depending on what the user specified or which would be larger. """ set_default = lambda x, default: x if x is not None else default @@ -270,7 +267,7 @@ class ApproxScalar(ApproxBase): -def approx(expected, rel=None, abs=None): +def approx(expected, rel=None, abs=None, nan_ok=False): """ Assert that two numbers (or two sets of numbers) are equal to each other within some tolerance. @@ -306,23 +303,35 @@ def approx(expected, rel=None, abs=None): >>> 0.1 + 0.2 == approx(0.3) True - The same syntax also works on sequences of numbers:: + The same syntax also works for sequences of numbers:: >>> (0.1 + 0.2, 0.2 + 0.4) == approx((0.3, 0.6)) True + + Dictionary *values*:: + >>> {'a': 0.1 + 0.2, 'b': 0.2 + 0.4} == approx({'a': 0.3, 'b': 0.6}) True + And ``numpy`` arrays:: + + >>> np.array([0.1, 0.2]) + np.array([0.2, 0.4]) == approx(np.array([0.3, 0.6])) + True + By default, ``approx`` considers numbers within a relative tolerance of ``1e-6`` (i.e. one part in a million) of its expected value to be equal. This treatment would lead to surprising results if the expected value was ``0.0``, because nothing but ``0.0`` itself is relatively close to ``0.0``. To handle this case less surprisingly, ``approx`` also considers numbers within an absolute tolerance of ``1e-12`` of its expected value to be - equal. Infinite numbers are another special case. They are only - considered equal to themselves, regardless of the relative tolerance. Both - the relative and absolute tolerances can be changed by passing arguments to - the ``approx`` constructor:: + equal. Infinity and NaN are special cases. Infinity is only considered + equal to itself, regardless of the relative tolerance. NaN is not + considered equal to anything by default, but you can make it be equal to + itself by setting the ``nan_ok`` argument to True. (This is meant to + facilitate comparing arrays that use NaN to mean "no data".) + + Both the relative and absolute tolerances can be changed by passing + arguments to the ``approx`` constructor:: >>> 1.0001 == approx(1) False @@ -385,29 +394,29 @@ def approx(expected, rel=None, abs=None): special case that you explicitly specify an absolute tolerance but not a relative tolerance, only the absolute tolerance is considered. """ - - from collections import Mapping, Sequence - try: - String = basestring # python2 - except NameError: - String = str, bytes # python3 - # Delegate the comparison to a class that knows how to deal with the type - # of the expected value (e.g. int, float, list, dict, numpy.array, etc). + from collections import Mapping, Sequence + from _pytest.compat import STRING_TYPES as String + + # Delegate the comparison to a class that knows how to deal with the type + # of the expected value (e.g. int, float, list, dict, numpy.array, etc). # - # This architecture is really driven by the need to support numpy arrays. - # The only way to override `==` for arrays without requiring that approx be - # the left operand is to inherit the approx object from `numpy.ndarray`. - # But that can't be a general solution, because it requires (1) numpy to be - # installed and (2) the expected value to be a numpy array. So the general + # This architecture is really driven by the need to support numpy arrays. + # The only way to override `==` for arrays without requiring that approx be + # the left operand is to inherit the approx object from `numpy.ndarray`. + # But that can't be a general solution, because it requires (1) numpy to be + # installed and (2) the expected value to be a numpy array. So the general # solution is to delegate each type of expected value to a different class. # - # This has the advantage that it made it easy to support mapping types - # (i.e. dict). The old code accepted mapping types, but would only compare + # This has the advantage that it made it easy to support mapping types + # (i.e. dict). The old code accepted mapping types, but would only compare # their keys, which is probably not what most people would expect. - if np and isinstance(expected, np.ndarray): - cls = ApproxNumpy + if _is_numpy_array(expected): + # Create the delegate class on the fly. This allow us to inherit from + # ``np.ndarray`` while still not importing numpy unless we need to. + import numpy as np + cls = type('ApproxNumpy', (ApproxNumpyBase, np.ndarray), {}) elif isinstance(expected, Mapping): cls = ApproxMapping elif isinstance(expected, Sequence) and not isinstance(expected, String): @@ -415,7 +424,25 @@ def approx(expected, rel=None, abs=None): else: cls = ApproxScalar - return cls(expected, rel, abs) + return cls(expected, rel, abs, nan_ok) + + +def _is_numpy_array(obj): + """ + Return true if the given object is a numpy array. Make a special effort to + avoid importing numpy unless it's really necessary. + """ + import inspect + + for cls in inspect.getmro(type(obj)): + if cls.__module__ == 'numpy': + try: + import numpy as np + return isinstance(obj, np.ndarray) + except ImportError: + pass + + return False # builtin pytest.raises helper @@ -555,6 +582,7 @@ def raises(expected_exception, *args, **kwargs): return _pytest._code.ExceptionInfo() fail(message) + raises.Exception = fail.Exception class RaisesContext(object): diff --git a/testing/python/approx.py b/testing/python/approx.py index 8b9605a00..d67500b15 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -218,21 +218,18 @@ class TestApprox(object): def test_expecting_nan(self): examples = [ - (nan, nan), - (-nan, -nan), - (nan, -nan), - (0.0, nan), - (inf, nan), + (eq, nan, nan), + (eq, -nan, -nan), + (eq, nan, -nan), + (ne, 0.0, nan), + (ne, inf, nan), ] - for a, x in examples: - # If there is a relative tolerance and the expected value is NaN, - # the actual tolerance is a NaN, which should be an error. - with pytest.raises(ValueError): - a != approx(x, rel=inf) + for op, a, x in examples: + # Nothing is equal to NaN by default. + assert a != approx(x) - # You can make comparisons against NaN by not specifying a relative - # tolerance, so only an absolute tolerance is calculated. - assert a != approx(x, abs=inf) + # If ``nan_ok=True``, then NaN is equal to NaN. + assert op(a, approx(x, nan_ok=True)) def test_int(self): within_1e6 = [ @@ -310,8 +307,9 @@ class TestApprox(object): def test_dict(self): actual = {'a': 1 + 1e-7, 'b': 2 + 1e-8} - expected = {'b': 2, 'a': 1} # Dictionaries became ordered in python3.6, - # so make sure the order doesn't matter + # Dictionaries became ordered in python3.6, so switch up the order here + # to make sure it doesn't matter. + expected = {'b': 2, 'a': 1} # Return false if any element is outside the tolerance. assert actual == approx(expected, rel=5e-7, abs=0) @@ -325,10 +323,7 @@ class TestApprox(object): assert {'a': 1, 'b': 2} != approx({'a': 1, 'b': 2, 'c': 3}) def test_numpy_array(self): - try: - import numpy as np - except ImportError: - pytest.skip("numpy not installed") + np = pytest.importorskip('numpy') actual = np.array([1 + 1e-7, 2 + 1e-8]) expected = np.array([1, 2]) @@ -339,30 +334,27 @@ class TestApprox(object): assert approx(expected, rel=5e-7, abs=0) == expected assert approx(expected, rel=5e-8, abs=0) != actual - def test_numpy_array_wrong_shape(self): - try: - import numpy as np - except ImportError: - pytest.skip("numpy not installed") + # Should be able to compare lists with numpy arrays. + assert list(actual) == approx(expected, rel=5e-7, abs=0) + assert list(actual) != approx(expected, rel=5e-8, abs=0) + assert actual == approx(list(expected), rel=5e-7, abs=0) + assert actual != approx(list(expected), rel=5e-8, abs=0) + + def test_numpy_array_wrong_shape(self): + np = pytest.importorskip('numpy') - import numpy as np a12 = np.array([[1, 2]]) a21 = np.array([[1],[2]]) assert a12 != approx(a21) assert a21 != approx(a12) - def test_non_number(self): - with pytest.raises(ValueError): - 1 == approx("1") - with pytest.raises(ValueError): - "1" == approx(1) - def test_doctests(self): + np = pytest.importorskip('numpy') parser = doctest.DocTestParser() test = parser.get_doctest( approx.__doc__, - {'approx': approx}, + {'approx': approx ,'np': np}, approx.__name__, None, None, ) diff --git a/tox.ini b/tox.ini index b73deca7d..188b073da 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ deps= nose mock requests + numpy [testenv:py26] commands= pytest --lsof -rfsxX {posargs:testing} From b41852c93b2f7e5f3dae60d09077d83d368aa730 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Thu, 15 Jun 2017 14:52:39 -0700 Subject: [PATCH 07/73] Use `autofunction` to document approx. It used to be a class, but it's a function now. --- doc/en/builtin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/en/builtin.rst b/doc/en/builtin.rst index 26dbd44cb..af0dd9a74 100644 --- a/doc/en/builtin.rst +++ b/doc/en/builtin.rst @@ -38,7 +38,7 @@ Examples at :ref:`assertraises`. Comparing floating point numbers -------------------------------- -.. autoclass:: approx +.. autofunction:: approx Raising a specific test outcome -------------------------------------- From 50769557e820e3b58d00d2727c9355d6d8dd6833 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Thu, 15 Jun 2017 14:53:27 -0700 Subject: [PATCH 08/73] Skip the numpy doctests. They seem like more trouble that they're worth. --- _pytest/python_api.py | 3 ++- testing/python/approx.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index a2942b742..0c0a6bb74 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -315,7 +315,8 @@ def approx(expected, rel=None, abs=None, nan_ok=False): And ``numpy`` arrays:: - >>> np.array([0.1, 0.2]) + np.array([0.2, 0.4]) == approx(np.array([0.3, 0.6])) + >>> import numpy as np # doctest: +SKIP + >>> np.array([0.1, 0.2]) + np.array([0.2, 0.4]) == approx(np.array([0.3, 0.6])) # doctest: +SKIP True By default, ``approx`` considers numbers within a relative tolerance of diff --git a/testing/python/approx.py b/testing/python/approx.py index d67500b15..3005a7bbe 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -350,11 +350,10 @@ class TestApprox(object): assert a21 != approx(a12) def test_doctests(self): - np = pytest.importorskip('numpy') parser = doctest.DocTestParser() test = parser.get_doctest( approx.__doc__, - {'approx': approx ,'np': np}, + {'approx': approx}, approx.__name__, None, None, ) From 5d2496862a12b0c05896ae6dac04d09f26eaa74b Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Thu, 15 Jun 2017 18:41:13 -0700 Subject: [PATCH 09/73] Only test numpy with py27 and py35. Travis was not successfully installing numpy with python<=2.6, python<=3.3, or PyPy. I decided that it didn't make sense to use numpy for all the tests, so instead I made new testing environments specifically for numpy. --- .travis.yml | 3 +++ appveyor.yml | 2 ++ tox.ini | 13 +++++++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a71e7dc1..29647f9bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,13 +16,16 @@ env: - TOXENV=py33 - TOXENV=py34 - TOXENV=py35 + - TOXENV=py36 - TOXENV=pypy - TOXENV=py27-pexpect - TOXENV=py27-xdist - TOXENV=py27-trial + - TOXENV=py27-numpy - TOXENV=py35-pexpect - TOXENV=py35-xdist - TOXENV=py35-trial + - TOXENV=py35-numpy - TOXENV=py27-nobyte - TOXENV=doctesting - TOXENV=freeze diff --git a/appveyor.yml b/appveyor.yml index cc72b4b70..abf033b4c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,9 +20,11 @@ environment: - TOXENV: "py27-pexpect" - TOXENV: "py27-xdist" - TOXENV: "py27-trial" + - TOXENV: "py27-numpy" - TOXENV: "py35-pexpect" - TOXENV: "py35-xdist" - TOXENV: "py35-trial" + - TOXENV: "py35-numpy" - TOXENV: "py27-nobyte" - TOXENV: "doctesting" - TOXENV: "freeze" diff --git a/tox.ini b/tox.ini index 188b073da..9b5cdc64a 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ envlist= py36 py37 pypy - {py27,py35}-{pexpect,xdist,trial} + {py27,py35}-{pexpect,xdist,trial,numpy} py27-nobyte doctesting freeze @@ -26,7 +26,6 @@ deps= nose mock requests - numpy [testenv:py26] commands= pytest --lsof -rfsxX {posargs:testing} @@ -111,6 +110,16 @@ deps={[testenv:py27-trial]deps} commands= pytest -ra {posargs:testing/test_unittest.py} +[testenv:py27-numpy] +deps=numpy +commands= + pytest -rfsxX {posargs:testing/python/approx.py} + +[testenv:py35-numpy] +deps=numpy +commands= + pytest -rfsxX {posargs:testing/python/approx.py} + [testenv:docs] skipsdist=True usedevelop=True From 4d02863b161108d022b5840f1a8e4b58a75ed088 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Thu, 15 Jun 2017 18:56:09 -0700 Subject: [PATCH 10/73] Remove a dict-comprehension. Not compatible with python26. --- _pytest/python_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index 0c0a6bb74..91605aa55 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -128,9 +128,9 @@ class ApproxMapping(ApproxBase): """ def __repr__(self): - return repr({ - k: self._approx_scalar(v) - for k,v in self.expected.items()}) + return repr(dict( + (k, self._approx_scalar(v)) + for k,v in self.expected.items())) def __eq__(self, actual): if actual.keys() != self.expected.keys(): From d6000e5ab1bc6fd5d8b4d90f230453704d5a9282 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Thu, 15 Jun 2017 20:34:36 -0700 Subject: [PATCH 11/73] Remove py36 from .travis.yml I thought the file was just out of date, but adding py36 made Travis complain "InterpreterNotFound: python3.6", so I guess it was correct as it was. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29647f9bc..d3dce9141 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,6 @@ env: - TOXENV=py33 - TOXENV=py34 - TOXENV=py35 - - TOXENV=py36 - TOXENV=pypy - TOXENV=py27-pexpect - TOXENV=py27-xdist From 9597e674d924f2bc026b9f2743b6716a2a5434c9 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Fri, 16 Jun 2017 08:25:13 -0700 Subject: [PATCH 12/73] Use sets to compare dictionary keys. --- _pytest/python_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index 91605aa55..acc3ea286 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -133,7 +133,7 @@ class ApproxMapping(ApproxBase): for k,v in self.expected.items())) def __eq__(self, actual): - if actual.keys() != self.expected.keys(): + if set(actual.keys()) != set(self.expected.keys()): return False return ApproxBase.__eq__(self, actual) From bdec2c8f9e1b39914b5e9fbc6d73adc29cf88f3a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 22 Jun 2017 08:45:10 +0200 Subject: [PATCH 13/73] move marker transfer to _pytest.mark --- _pytest/mark.py | 29 +++++++++++++++++++++++++++++ _pytest/python.py | 31 +------------------------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index 8b40a4f6e..928690d07 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -389,3 +389,32 @@ class MarkInfo(object): MARK_GEN = MarkGenerator() + + +def _marked(func, mark): + """ Returns True if :func: is already marked with :mark:, False otherwise. + This can happen if marker is applied to class and the test file is + invoked more than once. + """ + try: + func_mark = getattr(func, mark.name) + except AttributeError: + return False + return mark.args == func_mark.args and mark.kwargs == func_mark.kwargs + + +def transfer_markers(funcobj, cls, mod): + # XXX this should rather be code in the mark plugin or the mark + # plugin should merge with the python plugin. + for holder in (cls, mod): + try: + pytestmark = holder.pytestmark + except AttributeError: + continue + if isinstance(pytestmark, list): + for mark in pytestmark: + if not _marked(funcobj, mark): + mark(funcobj) + else: + if not _marked(funcobj, pytestmark): + pytestmark(funcobj) diff --git a/_pytest/python.py b/_pytest/python.py index 1a313a59e..c5af9106a 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -23,7 +23,7 @@ from _pytest.compat import ( safe_str, getlocation, enum, ) from _pytest.runner import fail - +from _pytest.mark import transfer_markers cutdir1 = py.path.local(pluggy.__file__.rstrip("oc")) cutdir2 = py.path.local(_pytest.__file__).dirpath() cutdir3 = py.path.local(py.__file__).dirpath() @@ -361,35 +361,6 @@ class PyCollector(PyobjMixin, main.Collector): ) -def _marked(func, mark): - """ Returns True if :func: is already marked with :mark:, False otherwise. - This can happen if marker is applied to class and the test file is - invoked more than once. - """ - try: - func_mark = getattr(func, mark.name) - except AttributeError: - return False - return mark.args == func_mark.args and mark.kwargs == func_mark.kwargs - - -def transfer_markers(funcobj, cls, mod): - # XXX this should rather be code in the mark plugin or the mark - # plugin should merge with the python plugin. - for holder in (cls, mod): - try: - pytestmark = holder.pytestmark - except AttributeError: - continue - if isinstance(pytestmark, list): - for mark in pytestmark: - if not _marked(funcobj, mark): - mark(funcobj) - else: - if not _marked(funcobj, pytestmark): - pytestmark(funcobj) - - class Module(main.File, PyCollector): """ Collector for test classes and functions. """ From 64ae6ae25dba3169529ea7ed56d65b749314599b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 22 Jun 2017 10:41:28 +0200 Subject: [PATCH 14/73] extract application of marks and legacy markinfos --- _pytest/mark.py | 70 ++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index 928690d07..4a3694049 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -8,6 +8,7 @@ from .compat import imap def alias(name): + # todo: introduce deprecationwarnings return property(attrgetter(name), doc='alias for ' + name) @@ -329,30 +330,39 @@ class MarkDecorator: is_class = inspect.isclass(func) if len(args) == 1 and (istestfunc(func) or is_class): if is_class: - if hasattr(func, 'pytestmark'): - mark_list = func.pytestmark - if not isinstance(mark_list, list): - mark_list = [mark_list] - # always work on a copy to avoid updating pytestmark - # from a superclass by accident - mark_list = mark_list + [self] - func.pytestmark = mark_list - else: - func.pytestmark = [self] + apply_mark(func, self.mark) else: - holder = getattr(func, self.name, None) - if holder is None: - holder = MarkInfo(self.mark) - setattr(func, self.name, holder) - else: - holder.add_mark(self.mark) + apply_legacy_mark(func, self.mark) return func mark = Mark(self.name, args, kwargs) return self.__class__(self.mark.combined_with(mark)) +def apply_mark(obj, mark): + assert isinstance(mark, Mark), mark + """applies a marker to an object, + makrer transfers only update legacy markinfo objects + """ + mark_list = getattr(obj, 'pytestmark', []) + if not isinstance(mark_list, list): + mark_list = [mark_list] + # always work on a copy to avoid updating pytestmark + # from a superclass by accident + mark_list = mark_list + [mark] + obj.pytestmark = mark_list + + +def apply_legacy_mark(func, mark): + if not isinstance(mark, Mark): + raise TypeError("got {mark!r} instead of a Mark".format(mark=mark)) + holder = getattr(func, mark.name, None) + if holder is None: + holder = MarkInfo(mark) + setattr(func, mark.name, holder) + else: + holder.add_mark(mark) class Mark(namedtuple('Mark', 'name, args, kwargs')): @@ -404,17 +414,17 @@ def _marked(func, mark): def transfer_markers(funcobj, cls, mod): - # XXX this should rather be code in the mark plugin or the mark - # plugin should merge with the python plugin. - for holder in (cls, mod): - try: - pytestmark = holder.pytestmark - except AttributeError: - continue - if isinstance(pytestmark, list): - for mark in pytestmark: - if not _marked(funcobj, mark): - mark(funcobj) - else: - if not _marked(funcobj, pytestmark): - pytestmark(funcobj) + """ + transfer legacy markers to the function level marminfo objects + this one is a major fsckup for mark breakages + """ + for obj in (cls, mod): + mark_list = getattr(obj, 'pytestmark', []) + + if not isinstance(mark_list, list): + mark_list = [mark_list] + + for mark in mark_list: + mark = getattr(mark, 'mark', mark) # unpack MarkDecorator + if not _marked(funcobj, mark): + apply_legacy_mark(funcobj, mark) From 19b12b22e7e5c23bb9a9a0e74ca36d441481973f Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 22 Jun 2017 10:48:45 +0200 Subject: [PATCH 15/73] store pristine marks on function.pytestmark fixes #2516 --- _pytest/mark.py | 1 + testing/test_mark.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index 4a3694049..08a0bd164 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -333,6 +333,7 @@ class MarkDecorator: apply_mark(func, self.mark) else: apply_legacy_mark(func, self.mark) + apply_mark(func, self.mark) return func mark = Mark(self.name, args, kwargs) diff --git a/testing/test_mark.py b/testing/test_mark.py index 0792b04fd..dff04f407 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -3,7 +3,7 @@ import os import sys import pytest -from _pytest.mark import MarkGenerator as Mark, ParameterSet +from _pytest.mark import MarkGenerator as Mark, ParameterSet, transfer_markers class TestMark(object): def test_markinfo_repr(self): @@ -772,3 +772,25 @@ class TestKeywordSelection(object): def test_parameterset_extractfrom(argval, expected): extracted = ParameterSet.extract_from(argval) assert extracted == expected + + +def test_legacy_transfer(): + + class FakeModule(object): + pytestmark = [] + + class FakeClass(object): + pytestmark = pytest.mark.nofun + + @pytest.mark.fun + def fake_method(self): + pass + + + transfer_markers(fake_method, FakeClass, FakeModule) + + # legacy marks transfer smeared + assert fake_method.nofun + assert fake_method.fun + # pristine marks dont transfer + assert fake_method.pytestmark == [pytest.mark.fun.mark] \ No newline at end of file From c791895c93308699458cdf73fd647b92ed1779c0 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 22 Jun 2017 10:50:46 +0200 Subject: [PATCH 16/73] changelog addition --- changelog/2516.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/2516.feature diff --git a/changelog/2516.feature b/changelog/2516.feature new file mode 100644 index 000000000..35b9ebae0 --- /dev/null +++ b/changelog/2516.feature @@ -0,0 +1 @@ +store unmeshed marks on functions pytestmark attribute From 1d926011a4d684c4bd505a3e7ecc444f43cdc446 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 22 Jun 2017 15:12:50 +0200 Subject: [PATCH 17/73] add deprecation warnings for using markinfo attributes --- _pytest/deprecated.py | 8 ++++++++ _pytest/mark.py | 18 ++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/_pytest/deprecated.py b/_pytest/deprecated.py index e75ff099e..44af1b48e 100644 --- a/_pytest/deprecated.py +++ b/_pytest/deprecated.py @@ -7,6 +7,11 @@ be removed when the time comes. """ from __future__ import absolute_import, division, print_function + +class RemovedInPytest4_0Warning(DeprecationWarning): + "warning class for features removed in pytest 4.0" + + MAIN_STR_ARGS = 'passing a string to pytest.main() is deprecated, ' \ 'pass a list of arguments instead.' @@ -22,3 +27,6 @@ SETUP_CFG_PYTEST = '[pytest] section in setup.cfg files is deprecated, use [tool GETFUNCARGVALUE = "use of getfuncargvalue is deprecated, use getfixturevalue" RESULT_LOG = '--result-log is deprecated and scheduled for removal in pytest 4.0' + +MARK_INFO_ATTRIBUTE = RemovedInPytest4_0Warning( + "Markinfo attributes are deprecated, please iterate the mark Collection") \ No newline at end of file diff --git a/_pytest/mark.py b/_pytest/mark.py index 08a0bd164..a3f84867e 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -2,14 +2,20 @@ from __future__ import absolute_import, division, print_function import inspect +import warnings from collections import namedtuple from operator import attrgetter from .compat import imap +from .deprecated import MARK_INFO_ATTRIBUTE +def alias(name, warning=None): + getter = attrgetter(name) -def alias(name): - # todo: introduce deprecationwarnings - return property(attrgetter(name), doc='alias for ' + name) + def warned(self): + warnings.warn(warning, stacklevel=2) + return getter(self) + + return property(getter if warning is None else warned, doc='alias for ' + name) class ParameterSet(namedtuple('ParameterSet', 'values, marks, id')): @@ -382,9 +388,9 @@ class MarkInfo(object): self.combined = mark self._marks = [mark] - name = alias('combined.name') - args = alias('combined.args') - kwargs = alias('combined.kwargs') + name = alias('combined.name', warning=MARK_INFO_ATTRIBUTE) + args = alias('combined.args', warning=MARK_INFO_ATTRIBUTE) + kwargs = alias('combined.kwargs', warning=MARK_INFO_ATTRIBUTE) def __repr__(self): return "".format(self.combined) From 23d016f1149812babbaf42da3f6c21acd70597c2 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 23 Jun 2017 11:05:38 +0200 Subject: [PATCH 18/73] address review comments * enhance api for fetching marks off an object * rename functions for storing marks * enhance deprecation message for MarkInfo --- _pytest/deprecated.py | 9 ++++---- _pytest/mark.py | 54 +++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/_pytest/deprecated.py b/_pytest/deprecated.py index 44af1b48e..c5aedd0c9 100644 --- a/_pytest/deprecated.py +++ b/_pytest/deprecated.py @@ -8,8 +8,8 @@ be removed when the time comes. from __future__ import absolute_import, division, print_function -class RemovedInPytest4_0Warning(DeprecationWarning): - "warning class for features removed in pytest 4.0" +class RemovedInPytest4Warning(DeprecationWarning): + """warning class for features removed in pytest 4.0""" MAIN_STR_ARGS = 'passing a string to pytest.main() is deprecated, ' \ @@ -28,5 +28,6 @@ GETFUNCARGVALUE = "use of getfuncargvalue is deprecated, use getfixturevalue" RESULT_LOG = '--result-log is deprecated and scheduled for removal in pytest 4.0' -MARK_INFO_ATTRIBUTE = RemovedInPytest4_0Warning( - "Markinfo attributes are deprecated, please iterate the mark Collection") \ No newline at end of file +MARK_INFO_ATTRIBUTE = RemovedInPytest4Warning( + "MarkInfo objects are deprecated as they contain the merged marks" +) \ No newline at end of file diff --git a/_pytest/mark.py b/_pytest/mark.py index a3f84867e..11f1e30d6 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -336,32 +336,42 @@ class MarkDecorator: is_class = inspect.isclass(func) if len(args) == 1 and (istestfunc(func) or is_class): if is_class: - apply_mark(func, self.mark) + store_mark(func, self.mark) else: - apply_legacy_mark(func, self.mark) - apply_mark(func, self.mark) + store_legacy_markinfo(func, self.mark) + store_mark(func, self.mark) return func mark = Mark(self.name, args, kwargs) return self.__class__(self.mark.combined_with(mark)) - -def apply_mark(obj, mark): - assert isinstance(mark, Mark), mark - """applies a marker to an object, - makrer transfers only update legacy markinfo objects +def get_unpacked_marks(obj): + """ + obtain the unpacked marks that are stored on a object """ mark_list = getattr(obj, 'pytestmark', []) if not isinstance(mark_list, list): mark_list = [mark_list] - # always work on a copy to avoid updating pytestmark - # from a superclass by accident - mark_list = mark_list + [mark] - obj.pytestmark = mark_list + return [ + getattr(mark, 'mark', mark) # unpack MarkDecorator + for mark in mark_list + ] -def apply_legacy_mark(func, mark): +def store_mark(obj, mark): + """store a Mark on a object + this is used to implement the Mark declarations/decorators correctly + """ + assert isinstance(mark, Mark), mark + # always reassign name to avoid updating pytestmark + # in a referene that was only borrowed + obj.pytestmark = get_unpacked_marks(obj) + [mark] + + +def store_legacy_markinfo(func, mark): + """create the legacy MarkInfo objects and put them onto the function + """ if not isinstance(mark, Mark): raise TypeError("got {mark!r} instead of a Mark".format(mark=mark)) holder = getattr(func, mark.name, None) @@ -422,16 +432,14 @@ def _marked(func, mark): def transfer_markers(funcobj, cls, mod): """ - transfer legacy markers to the function level marminfo objects - this one is a major fsckup for mark breakages + this function transfers class level markers and module level markers + into function level markinfo objects + + this is the main reason why marks are so broken + the resolution will involve phasing out function level MarkInfo objects + """ for obj in (cls, mod): - mark_list = getattr(obj, 'pytestmark', []) - - if not isinstance(mark_list, list): - mark_list = [mark_list] - - for mark in mark_list: - mark = getattr(mark, 'mark', mark) # unpack MarkDecorator + for mark in get_unpacked_marks(obj): if not _marked(funcobj, mark): - apply_legacy_mark(funcobj, mark) + store_legacy_markinfo(funcobj, mark) From b0b6c355f7ab17fbaccfea6f70efe03866ed5940 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 23 Jun 2017 11:09:16 +0200 Subject: [PATCH 19/73] fixup changelog, thanks Bruno --- changelog/2516.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/2516.feature b/changelog/2516.feature index 35b9ebae0..6436de16a 100644 --- a/changelog/2516.feature +++ b/changelog/2516.feature @@ -1 +1 @@ -store unmeshed marks on functions pytestmark attribute +Now test function objects have a ``pytestmark`` attribute containing a list of marks applied directly to the test function, as opposed to marks inherited from parent classes or modules. \ No newline at end of file From 8d5f2872d3aa7549299a7bfe7dfca1bd8f022bf5 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 23 Jun 2017 11:59:03 +0200 Subject: [PATCH 20/73] minor code style fix --- _pytest/python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/_pytest/python.py b/_pytest/python.py index c5af9106a..e10282a8c 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -24,6 +24,7 @@ from _pytest.compat import ( ) from _pytest.runner import fail from _pytest.mark import transfer_markers + cutdir1 = py.path.local(pluggy.__file__.rstrip("oc")) cutdir2 = py.path.local(_pytest.__file__).dirpath() cutdir3 = py.path.local(py.__file__).dirpath() From 0d0b01bded8ab758118d9be1ea64db493045dcf4 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 23 Jun 2017 21:06:18 +0200 Subject: [PATCH 21/73] introduce deprecation warnings for legacy parametersets, fixes #2427 --- _pytest/deprecated.py | 6 ++++++ _pytest/mark.py | 5 ++++- changelog/2427.removal | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changelog/2427.removal diff --git a/_pytest/deprecated.py b/_pytest/deprecated.py index c5aedd0c9..1eeb74918 100644 --- a/_pytest/deprecated.py +++ b/_pytest/deprecated.py @@ -30,4 +30,10 @@ RESULT_LOG = '--result-log is deprecated and scheduled for removal in pytest 4.0 MARK_INFO_ATTRIBUTE = RemovedInPytest4Warning( "MarkInfo objects are deprecated as they contain the merged marks" +) + +MARK_PARAMETERSET_UNPACKING = RemovedInPytest4Warning( + "Applying marks directly to parameters is deprecated," + " please use pytest.param(..., marks=...) instead.\n" + "For more details, see: https://docs.pytest.org/en/latest/parametrize.html" ) \ No newline at end of file diff --git a/_pytest/mark.py b/_pytest/mark.py index 11f1e30d6..961c3c409 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -6,7 +6,7 @@ import warnings from collections import namedtuple from operator import attrgetter from .compat import imap -from .deprecated import MARK_INFO_ATTRIBUTE +from .deprecated import MARK_INFO_ATTRIBUTE, MARK_PARAMETERSET_UNPACKING def alias(name, warning=None): getter = attrgetter(name) @@ -61,6 +61,9 @@ class ParameterSet(namedtuple('ParameterSet', 'values, marks, id')): if legacy_force_tuple: argval = argval, + if newmarks: + warnings.warn(MARK_PARAMETERSET_UNPACKING) + return cls(argval, marks=newmarks, id=None) @property diff --git a/changelog/2427.removal b/changelog/2427.removal new file mode 100644 index 000000000..c7ed8e17a --- /dev/null +++ b/changelog/2427.removal @@ -0,0 +1 @@ +introduce deprecation warnings for legacy marks based parametersets From 655d44b41347b97d53f86f91468bbd4e305b0048 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 24 Jun 2017 00:37:40 -0300 Subject: [PATCH 22/73] Add changelog entry explicitly deprecating old-style classes from pytest API Related to #2147 --- changelog/2147.removal | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/2147.removal diff --git a/changelog/2147.removal b/changelog/2147.removal new file mode 100644 index 000000000..d5f80a108 --- /dev/null +++ b/changelog/2147.removal @@ -0,0 +1 @@ +All old-style specific behavior in current classes in the pytest's API is considered deprecated at this point and will be removed in a future release. This affects Python 2 users only and in rare situations. From 9b9fede5beef291887853ebf0d4c2cc60f6fbfc2 Mon Sep 17 00:00:00 2001 From: Nathaniel Waisbrot Date: Sun, 25 Jun 2017 13:56:50 -0400 Subject: [PATCH 23/73] allow staticmethods to be detected as test functions Allow a class method decorated `@staticmethod` to be collected as a test function (if it meets the usual criteria). This feature will not work in Python 2.6 -- static methods will still be ignored there. --- AUTHORS | 1 + _pytest/python.py | 17 +++++++++++++---- changelog/2528.feature | 1 + testing/python/collect.py | 20 ++++++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 changelog/2528.feature diff --git a/AUTHORS b/AUTHORS index ca282870f..2fd164470 100644 --- a/AUTHORS +++ b/AUTHORS @@ -118,6 +118,7 @@ Michael Droettboom Michael Seifert Michal Wajszczuk Mike Lundy +Nathaniel Waisbrot Ned Batchelder Neven Mundar Nicolas Delaby diff --git a/_pytest/python.py b/_pytest/python.py index e10282a8c..3750f61da 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -271,10 +271,19 @@ class PyCollector(PyobjMixin, main.Collector): return self._matches_prefix_or_glob_option('python_classes', name) def istestfunction(self, obj, name): - return ( - (self.funcnamefilter(name) or self.isnosetest(obj)) and - safe_getattr(obj, "__call__", False) and fixtures.getfixturemarker(obj) is None - ) + if self.funcnamefilter(name) or self.isnosetest(obj): + if isinstance(obj, staticmethod): + # static methods need to be unwrapped + obj = safe_getattr(obj, '__func__', False) + if obj is False: + # Python 2.6 wraps in a different way that we won't try to handle + self.warn(code="C2", message="cannot collect static method %r because it is not a function (always the case in Python 2.6)" % name) + return False + return ( + safe_getattr(obj, "__call__", False) and fixtures.getfixturemarker(obj) is None + ) + else: + return False def istestclass(self, obj, name): return self.classnamefilter(name) or self.isnosetest(obj) diff --git a/changelog/2528.feature b/changelog/2528.feature new file mode 100644 index 000000000..c91cdb5d7 --- /dev/null +++ b/changelog/2528.feature @@ -0,0 +1 @@ +Allow class methods decorated as ``@staticmethod`` to be candidates for collection as a test function. (Only for Python 2.7 and above. Python 2.6 will still ignore static methods.) diff --git a/testing/python/collect.py b/testing/python/collect.py index 236421f1c..1b62fb770 100644 --- a/testing/python/collect.py +++ b/testing/python/collect.py @@ -143,6 +143,26 @@ class TestClass(object): "*collected 0*", ]) + def test_static_method(self, testdir): + testdir.getmodulecol(""" + class Test(object): + @staticmethod + def test_something(): + pass + """) + result = testdir.runpytest() + if sys.version_info < (2,7): + # in 2.6, the code to handle static methods doesn't work + result.stdout.fnmatch_lines([ + "*collected 0 items*", + "*cannot collect static method*", + ]) + else: + result.stdout.fnmatch_lines([ + "*collected 1 item*", + "*1 passed in*", + ]) + def test_setup_teardown_class_as_classmethod(self, testdir): testdir.makepyfile(test_mod1=""" class TestClassMethod(object): From 8524a5707549225f60587ef7d965c5e3eb288681 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 3 Jul 2017 22:44:37 -0700 Subject: [PATCH 24/73] Add "approx" to all the repr-strings. --- _pytest/python_api.py | 6 +++--- testing/python/approx.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index acc3ea286..ab7a0bc5d 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -95,7 +95,7 @@ class ApproxNumpyBase(ApproxBase): def __repr__(self): # It might be nice to rewrite this function to account for the # shape of the array... - return repr(list( + return "approx({0!r})".format(list( self._approx_scalar(x) for x in self.expected)) def __eq__(self, actual): @@ -128,7 +128,7 @@ class ApproxMapping(ApproxBase): """ def __repr__(self): - return repr(dict( + return "approx({0!r})".format(dict( (k, self._approx_scalar(v)) for k,v in self.expected.items())) @@ -152,7 +152,7 @@ class ApproxSequence(ApproxBase): seq_type = type(self.expected) if seq_type not in (tuple, list, set): seq_type = list - return repr(seq_type( + return "approx({0!r})".format(seq_type( self._approx_scalar(x) for x in self.expected)) def __eq__(self, actual): diff --git a/testing/python/approx.py b/testing/python/approx.py index 3005a7bbe..a21f644f5 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -28,8 +28,8 @@ class TestApprox(object): if sys.version_info[:2] == (2, 6): tol1, tol2, infr = '???', '???', '???' assert repr(approx(1.0)) == '1.0 {pm} {tol1}'.format(pm=plus_minus, tol1=tol1) - assert repr(approx([1.0, 2.0])) == '[1.0 {pm} {tol1}, 2.0 {pm} {tol2}]'.format(pm=plus_minus, tol1=tol1, tol2=tol2) - assert repr(approx((1.0, 2.0))) == '(1.0 {pm} {tol1}, 2.0 {pm} {tol2})'.format(pm=plus_minus, tol1=tol1, tol2=tol2) + assert repr(approx([1.0, 2.0])) == 'approx([1.0 {pm} {tol1}, 2.0 {pm} {tol2}])'.format(pm=plus_minus, tol1=tol1, tol2=tol2) + assert repr(approx((1.0, 2.0))) == 'approx((1.0 {pm} {tol1}, 2.0 {pm} {tol2}))'.format(pm=plus_minus, tol1=tol1, tol2=tol2) assert repr(approx(inf)) == 'inf' assert repr(approx(1.0, rel=nan)) == '1.0 {pm} ???'.format(pm=plus_minus) assert repr(approx(1.0, rel=inf)) == '1.0 {pm} {infr}'.format(pm=plus_minus, infr=infr) @@ -37,8 +37,8 @@ class TestApprox(object): # Dictionaries aren't ordered, so we need to check both orders. assert repr(approx({'a': 1.0, 'b': 2.0})) in ( - "{{'a': 1.0 {pm} {tol1}, 'b': 2.0 {pm} {tol2}}}".format(pm=plus_minus, tol1=tol1, tol2=tol2), - "{{'b': 2.0 {pm} {tol2}, 'a': 1.0 {pm} {tol1}}}".format(pm=plus_minus, tol1=tol1, tol2=tol2), + "approx({{'a': 1.0 {pm} {tol1}, 'b': 2.0 {pm} {tol2}}})".format(pm=plus_minus, tol1=tol1, tol2=tol2), + "approx({{'b': 2.0 {pm} {tol2}, 'a': 1.0 {pm} {tol1}}})".format(pm=plus_minus, tol1=tol1, tol2=tol2), ) def test_operator_overloading(self): From c111e9dac31664fd8398b9231bd3dbf87ad6646f Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 3 Jul 2017 22:45:24 -0700 Subject: [PATCH 25/73] Avoid making multiple ApproxNumpy types. --- _pytest/python_api.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index ab7a0bc5d..264d925c3 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -49,13 +49,9 @@ class ApproxNumpyBase(ApproxBase): """ Perform approximate comparisons for numpy arrays. - This class should not be used directly. Instead, it should be used to make - a subclass that also inherits from `np.ndarray`, e.g.:: - - import numpy as np - ApproxNumpy = type('ApproxNumpy', (ApproxNumpyBase, np.ndarray), {}) - - This bizarre invocation is necessary because the object doing the + This class should not be used directly. Instead, the `inherit_ndarray()` + class method should be used to make a subclass that also inherits from + `np.ndarray`. This indirection is necessary because the object doing the approximate comparison must inherit from `np.ndarray`, or it will only work on the left side of the `==` operator. But importing numpy is relatively expensive, so we also want to avoid that unless we actually have a numpy @@ -81,6 +77,18 @@ class ApproxNumpyBase(ApproxBase): it appears on. """ + subclass = None + + @classmethod + def inherit_ndarray(cls): + import numpy as np + assert not isinstance(cls, np.ndarray) + + if cls.subclass is None: + cls.subclass = type('ApproxNumpy', (ApproxNumpyBase, np.ndarray), {}) + + return cls.subclass + def __new__(cls, expected, rel=None, abs=None, nan_ok=False): """ Numpy uses __new__ (rather than __init__) to initialize objects. @@ -416,8 +424,7 @@ def approx(expected, rel=None, abs=None, nan_ok=False): if _is_numpy_array(expected): # Create the delegate class on the fly. This allow us to inherit from # ``np.ndarray`` while still not importing numpy unless we need to. - import numpy as np - cls = type('ApproxNumpy', (ApproxNumpyBase, np.ndarray), {}) + cls = ApproxNumpyBase.inherit_ndarray() elif isinstance(expected, Mapping): cls = ApproxMapping elif isinstance(expected, Sequence) and not isinstance(expected, String): From 7a1a439049c01630665aed04bbe18486f3eeda83 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Tue, 4 Jul 2017 09:20:52 -0700 Subject: [PATCH 26/73] Use `cls` instead of `ApproxNumpyBase`. Slightly more general, probably doesn't make a difference. --- _pytest/python_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index 264d925c3..cb7d5e459 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -85,7 +85,7 @@ class ApproxNumpyBase(ApproxBase): assert not isinstance(cls, np.ndarray) if cls.subclass is None: - cls.subclass = type('ApproxNumpy', (ApproxNumpyBase, np.ndarray), {}) + cls.subclass = type('ApproxNumpy', (cls, np.ndarray), {}) return cls.subclass From f471eef6613d7408faa3e57964c043342a4330a9 Mon Sep 17 00:00:00 2001 From: "V.Kuznetsov" Date: Fri, 7 Jul 2017 13:07:06 +0300 Subject: [PATCH 27/73] ini option cache_dir --- _pytest/cacheprovider.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py index 7fc08fff3..8e64f5b8d 100755 --- a/_pytest/cacheprovider.py +++ b/_pytest/cacheprovider.py @@ -8,13 +8,14 @@ from __future__ import absolute_import, division, print_function import py import pytest import json +import os from os.path import sep as _sep, altsep as _altsep class Cache(object): def __init__(self, config): self.config = config - self._cachedir = config.rootdir.join(".cache") + self._cachedir = Cache.cache_dir_from_config(config) self.trace = config.trace.root.get("cache") if config.getvalue("cacheclear"): self.trace("clearing cachedir") @@ -22,6 +23,16 @@ class Cache(object): self._cachedir.remove() self._cachedir.mkdir() + @staticmethod + def cache_dir_from_config(config): + cache_dir = config.getini("cache_dir") + cache_dir = os.path.expanduser(cache_dir) + cache_dir = os.path.expandvars(cache_dir) + if os.path.isabs(cache_dir): + return py.path.local(cache_dir) + else: + return config.rootdir.join(cache_dir) + def makedir(self, name): """ return a directory path object with the given name. If the directory does not yet exist, it will be created. You can use it @@ -171,6 +182,9 @@ def pytest_addoption(parser): group.addoption( '--cache-clear', action='store_true', dest="cacheclear", help="remove all cache contents at start of test run.") + parser.addini( + "cache_dir", default='.cache', + help="cache directory path.") def pytest_cmdline_main(config): From 7a9fc694358f3d6958de0c7785bcbf67c6784076 Mon Sep 17 00:00:00 2001 From: "V.Kuznetsov" Date: Fri, 7 Jul 2017 13:07:33 +0300 Subject: [PATCH 28/73] tests for ini option cache_dir --- testing/test_cache.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/testing/test_cache.py b/testing/test_cache.py index 600b5e6d9..c7e92063b 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, division, print_function import sys - +import py import _pytest import pytest import os @@ -87,7 +87,36 @@ class TestNewAPI(object): assert result.ret == 0 result.stdout.fnmatch_lines(["*1 passed*"]) + def test_custom_rel_cache_dir(self, testdir): + rel_cache_dir = os.path.join('custom_cache_dir', 'subdir') + testdir.makeini(""" + [pytest] + cache_dir = {cache_dir} + """.format(cache_dir=rel_cache_dir)) + testdir.makepyfile(test_errored='def test_error():\n assert False') + testdir.runpytest() + assert testdir.tmpdir.join(rel_cache_dir).isdir() + def test_custom_abs_cache_dir(self, testdir, tmpdir_factory): + tmp = str(tmpdir_factory.mktemp('tmp')) + abs_cache_dir = os.path.join(tmp, 'custom_cache_dir') + testdir.makeini(""" + [pytest] + cache_dir = {cache_dir} + """.format(cache_dir=abs_cache_dir)) + testdir.makepyfile(test_errored='def test_error():\n assert False') + testdir.runpytest() + assert py.path.local(abs_cache_dir).isdir() + + def test_custom_cache_dir_with_env_var(self, testdir, monkeypatch): + monkeypatch.setenv('env_var', 'custom_cache_dir') + testdir.makeini(""" + [pytest] + cache_dir = {cache_dir} + """.format(cache_dir='$env_var')) + testdir.makepyfile(test_errored='def test_error():\n assert False') + testdir.runpytest() + assert testdir.tmpdir.join('custom_cache_dir').isdir() def test_cache_reportheader(testdir): testdir.makepyfile(""" From 91418eda3b47504c930d8f46a5f96309a7e6909f Mon Sep 17 00:00:00 2001 From: "V.Kuznetsov" Date: Fri, 7 Jul 2017 13:08:12 +0300 Subject: [PATCH 29/73] docs for ini option cache_dir --- doc/en/cache.rst | 2 ++ doc/en/customize.rst | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/doc/en/cache.rst b/doc/en/cache.rst index 688b6dd04..e4071a8f8 100644 --- a/doc/en/cache.rst +++ b/doc/en/cache.rst @@ -1,3 +1,5 @@ +.. _`cache_provider`: + Cache: working with cross-testrun state ======================================= diff --git a/doc/en/customize.rst b/doc/en/customize.rst index ce0a36c11..b0c48f0e3 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -262,3 +262,14 @@ Builtin configuration file options This tells pytest to ignore deprecation warnings and turn all other warnings into errors. For more information please refer to :ref:`warnings`. + +.. confval:: cache_dir + + .. versionadded:: 3.2 + + Sets a directory where stores content of cache plugin. Default directory is + ``.cache`` which is created in :ref:`rootdir `. Directory may be + relative or absolute path. If setting relative path, then directory is created + relative to :ref:`rootdir `. Additionally path may contain environment + variables, that will be expanded. For more information about cache plugin + please refer to :ref:`cache_provider`. From bd52eebab4fb840992fc3f2ca99a40920421b221 Mon Sep 17 00:00:00 2001 From: "V.Kuznetsov" Date: Fri, 7 Jul 2017 13:20:39 +0300 Subject: [PATCH 30/73] changelog for ini option cache_dir --- changelog/2543.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/2543.feature diff --git a/changelog/2543.feature b/changelog/2543.feature new file mode 100644 index 000000000..6d65a376f --- /dev/null +++ b/changelog/2543.feature @@ -0,0 +1 @@ +New ``cache_dir`` ini option: sets a directory where stores content of cache plugin. Default directory is ``.cache`` which is created in ``rootdir``. Directory may be relative or absolute path. If setting relative path, then directory is created relative to ``rootdir``. Additionally path may contain environment variables, that will be expanded. From 89c73582caf9dda84f237bd6d4986d4db7d11a2e Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 11:52:16 -0500 Subject: [PATCH 31/73] ignore the active python installation unless told otherwise --- _pytest/main.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/_pytest/main.py b/_pytest/main.py index 1a6ba2781..5b6409664 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -70,6 +70,8 @@ def pytest_addoption(parser): group.addoption('--keepduplicates', '--keep-duplicates', action="store_true", dest="keepduplicates", default=False, help="Keep duplicate tests.") + group.addoption('--collect-in-virtualenv', action='store_true', + help="Collect tests in the current Python installation (default False)") group = parser.getgroup("debugconfig", "test session debugging and configuration") @@ -177,6 +179,16 @@ def pytest_ignore_collect(path, config): if py.path.local(path) in ignore_paths: return True + invenv = py.path.local(sys.prefix) == path + allow_invenv = config.getoption("collect_in_virtualenv") + if invenv and not allow_invenv: + config.warn(RuntimeWarning, + 'Path "%s" appears to be a Python installation; skipping\n' + 'Pass --collect-in-virtualenv to force collection of tests in "%s"\n' + 'Use --ignore="%s" to silence this warning' % (path, path, path) + ) + return True + # Skip duplicate paths. keepduplicates = config.getoption("keepduplicates") duplicate_paths = config.pluginmanager._duplicatepaths From c2d49e39a2603f3ee8ec3a0e13be4afc3303aca1 Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 13:01:56 -0500 Subject: [PATCH 32/73] add news item --- changelog/2518.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/2518.feature diff --git a/changelog/2518.feature b/changelog/2518.feature new file mode 100644 index 000000000..3b97cc18f --- /dev/null +++ b/changelog/2518.feature @@ -0,0 +1 @@ +Collection ignores the currently active Python installation by default; `--collect-in-virtualenv` overrides this behavior. From 676c4f970d2d2f3dcece052adc65491e1f9e1588 Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 13:31:11 -0500 Subject: [PATCH 33/73] trim trailing ws --- _pytest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/main.py b/_pytest/main.py index 5b6409664..bd33ab95f 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -179,7 +179,7 @@ def pytest_ignore_collect(path, config): if py.path.local(path) in ignore_paths: return True - invenv = py.path.local(sys.prefix) == path + invenv = py.path.local(sys.prefix) == path allow_invenv = config.getoption("collect_in_virtualenv") if invenv and not allow_invenv: config.warn(RuntimeWarning, From b32cfc88daad55f6518fc828db7aa770d4e4c80a Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 14:32:09 -0500 Subject: [PATCH 34/73] use presence of activate script rather than sys.prefix to determine if a dir is a virtualenv --- _pytest/main.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/_pytest/main.py b/_pytest/main.py index bd33ab95f..caf2ca813 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -169,6 +169,17 @@ def pytest_runtestloop(session): return True +def _in_venv(path): + """Attempts to detect if ``path`` is the root of a Virtual Environment by + checking for the existence of the appropriate activate script""" + bindir = path.join('Scripts' if sys.platform.startswith('win') else 'bin') + if not bindir.exists(): + return False + activates = ('activate', 'activate.csh', 'activate.fish', + 'Activate', 'Activate.bat', 'Activate.ps1') + return any([fname.basename in activates for fname in bindir.listdir()]) + + def pytest_ignore_collect(path, config): ignore_paths = config._getconftest_pathlist("collect_ignore", path=path.dirpath()) ignore_paths = ignore_paths or [] @@ -179,11 +190,10 @@ def pytest_ignore_collect(path, config): if py.path.local(path) in ignore_paths: return True - invenv = py.path.local(sys.prefix) == path - allow_invenv = config.getoption("collect_in_virtualenv") - if invenv and not allow_invenv: + allow_in_venv = config.getoption("collect_in_virtualenv") + if _in_venv(path) and not allow_in_venv: config.warn(RuntimeWarning, - 'Path "%s" appears to be a Python installation; skipping\n' + 'Path "%s" appears to be a Python virtual installation; skipping\n' 'Pass --collect-in-virtualenv to force collection of tests in "%s"\n' 'Use --ignore="%s" to silence this warning' % (path, path, path) ) From 67fca040503b30ff7d23f6962c1387c8f621c80c Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 23:14:38 -0500 Subject: [PATCH 35/73] update docs and note; add virtualenv collection tests --- _pytest/main.py | 8 ++----- changelog/2518.feature | 2 +- doc/en/customize.rst | 11 ++++++++- testing/test_collection.py | 49 +++++++++++++++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/_pytest/main.py b/_pytest/main.py index caf2ca813..a7ecc5149 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -71,7 +71,8 @@ def pytest_addoption(parser): dest="keepduplicates", default=False, help="Keep duplicate tests.") group.addoption('--collect-in-virtualenv', action='store_true', - help="Collect tests in the current Python installation (default False)") + dest='collect_in_virtualenv', default=False, + help="Don't ignore tests in a local virtualenv directory") group = parser.getgroup("debugconfig", "test session debugging and configuration") @@ -192,11 +193,6 @@ def pytest_ignore_collect(path, config): allow_in_venv = config.getoption("collect_in_virtualenv") if _in_venv(path) and not allow_in_venv: - config.warn(RuntimeWarning, - 'Path "%s" appears to be a Python virtual installation; skipping\n' - 'Pass --collect-in-virtualenv to force collection of tests in "%s"\n' - 'Use --ignore="%s" to silence this warning' % (path, path, path) - ) return True # Skip duplicate paths. diff --git a/changelog/2518.feature b/changelog/2518.feature index 3b97cc18f..2f6597a97 100644 --- a/changelog/2518.feature +++ b/changelog/2518.feature @@ -1 +1 @@ -Collection ignores the currently active Python installation by default; `--collect-in-virtualenv` overrides this behavior. +Collection ignores local virtualenvs by default; `--collect-in-virtualenv` overrides this behavior. diff --git a/doc/en/customize.rst b/doc/en/customize.rst index ce0a36c11..1920028a1 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -171,7 +171,16 @@ Builtin configuration file options norecursedirs = .svn _build tmp* This would tell ``pytest`` to not look into typical subversion or - sphinx-build directories or into any ``tmp`` prefixed directory. + sphinx-build directories or into any ``tmp`` prefixed directory. + + Additionally, ``pytest`` will attempt to intelligently identify and ignore a + virtualenv by the presence of an activation script. Any directory deemed to + be the root of a virtual environment will not be considered during test + collection unless ``‑‑collect‑in‑virtualenv`` is given. Note also that + ``norecursedirs`` takes precedence over ``‑‑collect‑in‑virtualenv``; e.g. if + you intend to run tests in a virtualenv with a base directory that matches + ``'.*'`` you *must* override ``norecursedirs`` in addition to using the + ``‑‑collect‑in‑virtualenv`` flag. .. confval:: testpaths diff --git a/testing/test_collection.py b/testing/test_collection.py index a90269789..a3c323e61 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function import pytest, py -from _pytest.main import Session, EXIT_NOTESTSCOLLECTED +from _pytest.main import Session, EXIT_NOTESTSCOLLECTED, _in_venv class TestCollector(object): def test_collect_versus_item(self): @@ -121,6 +121,53 @@ class TestCollectFS(object): assert "test_notfound" not in s assert "test_found" in s + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test_ignored_virtualenvs(self, testdir, fname): + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + testdir.tmpdir.ensure("virtual", bindir, fname) + testfile = testdir.tmpdir.ensure("virtual", "test_invenv.py") + testfile.write("def test_hello(): pass") + + # by default, ignore tests inside a virtualenv + result = testdir.runpytest() + assert "test_invenv" not in result.stdout.str() + # allow test collection if user insists + result = testdir.runpytest("--collect-in-virtualenv") + assert "test_invenv" in result.stdout.str() + # allow test collection if user directly passes in the directory + result = testdir.runpytest("virtual") + assert "test_invenv" in result.stdout.str() + + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test_ignored_virtualenvs_norecursedirs_precedence(self, testdir, fname): + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + # norecursedirs takes priority + testdir.tmpdir.ensure(".virtual", bindir, fname) + testfile = testdir.tmpdir.ensure(".virtual", "test_invenv.py") + testfile.write("def test_hello(): pass") + result = testdir.runpytest("--collect-in-virtualenv") + assert "test_invenv" not in result.stdout.str() + # ...unless the virtualenv is explicitly given on the CLI + result = testdir.runpytest("--collect-in-virtualenv", ".virtual") + assert "test_invenv" in result.stdout.str() + + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test__in_venv(self, testdir, fname): + """Directly test the virtual env detection function""" + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + # no bin/activate, not a virtualenv + base_path = testdir.tmpdir.mkdir('venv') + assert _in_venv(base_path) is False + # with bin/activate, totally a virtualenv + base_path.ensure(bindir, fname) + assert _in_venv(base_path) is True + def test_custom_norecursedirs(self, testdir): testdir.makeini(""" [pytest] From 7b1870a94ed27b6e4517026d248c756df1ce9b8e Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 17 Jul 2017 21:16:14 -0300 Subject: [PATCH 36/73] Fix flake8 in features branch --- _pytest/deprecated.py | 2 +- _pytest/mark.py | 2 ++ _pytest/python.py | 6 +++++- _pytest/python_api.py | 13 +++++++------ testing/code/test_excinfo.py | 16 ++++++++-------- testing/code/test_source.py | 7 +++---- testing/python/approx.py | 29 +++++++++++++++-------------- testing/python/collect.py | 2 +- testing/test_cache.py | 1 + testing/test_capture.py | 26 +++++++++++++------------- testing/test_conftest.py | 4 ++-- testing/test_mark.py | 3 +-- testing/test_runner.py | 4 ++-- 13 files changed, 61 insertions(+), 54 deletions(-) diff --git a/_pytest/deprecated.py b/_pytest/deprecated.py index 385742111..4f7b9e936 100644 --- a/_pytest/deprecated.py +++ b/_pytest/deprecated.py @@ -36,4 +36,4 @@ MARK_PARAMETERSET_UNPACKING = RemovedInPytest4Warning( "Applying marks directly to parameters is deprecated," " please use pytest.param(..., marks=...) instead.\n" "For more details, see: https://docs.pytest.org/en/latest/parametrize.html" -) \ No newline at end of file +) diff --git a/_pytest/mark.py b/_pytest/mark.py index 7714c7f5e..c2959606c 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -8,6 +8,7 @@ from operator import attrgetter from .compat import imap from .deprecated import MARK_INFO_ATTRIBUTE, MARK_PARAMETERSET_UNPACKING + def alias(name, warning=None): getter = attrgetter(name) @@ -351,6 +352,7 @@ class MarkDecorator: mark = Mark(self.name, args, kwargs) return self.__class__(self.mark.combined_with(mark)) + def get_unpacked_marks(obj): """ obtain the unpacked marks that are stored on a object diff --git a/_pytest/python.py b/_pytest/python.py index bf3b1965d..dca900a6a 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -281,7 +281,10 @@ class PyCollector(PyobjMixin, main.Collector): obj = safe_getattr(obj, '__func__', False) if obj is False: # Python 2.6 wraps in a different way that we won't try to handle - self.warn(code="C2", message="cannot collect static method %r because it is not a function (always the case in Python 2.6)" % name) + msg = "cannot collect static method %r because " \ + "it is not a function (always the case in Python 2.6)" + self.warn( + code="C2", message=msg % name) return False return ( safe_getattr(obj, "__call__", False) and fixtures.getfixturemarker(obj) is None @@ -1510,6 +1513,7 @@ class ApproxNonIterable(object): # the basic pytest Function item # + class Function(FunctionMixin, main.Item, fixtures.FuncargnamesCompatAttr): """ a Function Item is responsible for setting up and executing a Python test function. diff --git a/_pytest/python_api.py b/_pytest/python_api.py index cb7d5e459..c5493f495 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -9,6 +9,7 @@ import _pytest._code # builtin pytest.approx helper + class ApproxBase(object): """ Provide shared utilities for making approximate comparisons between numbers @@ -26,8 +27,8 @@ class ApproxBase(object): def __eq__(self, actual): return all( - a == self._approx_scalar(x) - for a, x in self._yield_comparisons(actual)) + a == self._approx_scalar(x) + for a, x in self._yield_comparisons(actual)) __hash__ = None @@ -138,7 +139,7 @@ class ApproxMapping(ApproxBase): def __repr__(self): return "approx({0!r})".format(dict( (k, self._approx_scalar(v)) - for k,v in self.expected.items())) + for k, v in self.expected.items())) def __eq__(self, actual): if set(actual.keys()) != set(self.expected.keys()): @@ -241,7 +242,7 @@ class ApproxScalar(ApproxBase): absolute tolerance or a relative tolerance, depending on what the user specified or which would be larger. """ - set_default = lambda x, default: x if x is not None else default + def set_default(x, default): return x if x is not None else default # Figure out what the absolute tolerance should be. ``self.abs`` is # either None or a value specified by the user. @@ -274,7 +275,6 @@ class ApproxScalar(ApproxBase): return max(relative_tolerance, absolute_tolerance) - def approx(expected, rel=None, abs=None, nan_ok=False): """ Assert that two numbers (or two sets of numbers) are equal to each other @@ -574,7 +574,7 @@ def raises(expected_exception, *args, **kwargs): frame = sys._getframe(1) loc = frame.f_locals.copy() loc.update(kwargs) - #print "raises frame scope: %r" % frame.f_locals + # print "raises frame scope: %r" % frame.f_locals try: code = _pytest._code.Source(code).compile() py.builtin.exec_(code, frame.f_globals, loc) @@ -593,6 +593,7 @@ def raises(expected_exception, *args, **kwargs): raises.Exception = fail.Exception + class RaisesContext(object): def __init__(self, expected_exception, message, match_expr): self.expected_exception = expected_exception diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index a7dfe80a6..37ceeb423 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -144,10 +144,10 @@ class TestTraceback_f_g_h(object): xyz() """) try: - exec (source.compile()) + exec(source.compile()) except NameError: tb = _pytest._code.ExceptionInfo().traceback - print (tb[-1].getsource()) + print(tb[-1].getsource()) s = str(tb[-1].getsource()) assert s.startswith("def xyz():\n try:") assert s.strip().endswith("except somenoname:") @@ -341,7 +341,7 @@ def test_excinfo_errisinstance(): def test_excinfo_no_sourcecode(): try: - exec ("raise ValueError()") + exec("raise ValueError()") except ValueError: excinfo = _pytest._code.ExceptionInfo() s = str(excinfo.traceback[-1]) @@ -431,7 +431,7 @@ class TestFormattedExcinfo(object): def excinfo_from_exec(self, source): source = _pytest._code.Source(source).strip() try: - exec (source.compile()) + exec(source.compile()) except KeyboardInterrupt: raise except: @@ -471,7 +471,7 @@ class TestFormattedExcinfo(object): pr = FormattedExcinfo() co = compile("raise ValueError()", "", "exec") try: - exec (co) + exec(co) except ValueError: excinfo = _pytest._code.ExceptionInfo() repr = pr.repr_excinfo(excinfo) @@ -486,7 +486,7 @@ a = 1 raise ValueError() """, "", "exec") try: - exec (co) + exec(co) except ValueError: excinfo = _pytest._code.ExceptionInfo() repr = pr.repr_excinfo(excinfo) @@ -992,7 +992,7 @@ raise ValueError() tw = TWMock() r.toterminal(tw) for line in tw.lines: - print (line) + print(line) assert tw.lines[0] == "" assert tw.lines[1] == " def f():" assert tw.lines[2] == "> g()" @@ -1040,7 +1040,7 @@ raise ValueError() tw = TWMock() r.toterminal(tw) for line in tw.lines: - print (line) + print(line) assert tw.lines[0] == "" assert tw.lines[1] == " def f():" assert tw.lines[2] == " try:" diff --git a/testing/code/test_source.py b/testing/code/test_source.py index f7f272c7b..6432bf95c 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -170,12 +170,12 @@ class TestSourceParsingAndCompiling(object): def test_compile(self): co = _pytest._code.compile("x=3") d = {} - exec (co, d) + exec(co, d) assert d['x'] == 3 def test_compile_and_getsource_simple(self): co = _pytest._code.compile("x=3") - exec (co) + exec(co) source = _pytest._code.Source(co) assert str(source) == "x=3" @@ -342,8 +342,7 @@ def test_getstartingblock_multiline(): self.source = _pytest._code.Frame(frame).statement x = A('x', - 'y' - , + 'y', 'z') l = [i for i in x.source.lines if i.strip()] diff --git a/testing/python/approx.py b/testing/python/approx.py index 9aed0fa0a..ebd0234de 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -9,6 +9,7 @@ from decimal import Decimal from fractions import Fraction inf, nan = float('inf'), float('nan') + class MyDocTestRunner(doctest.DocTestRunner): def __init__(self): @@ -37,8 +38,8 @@ class TestApprox(object): # Dictionaries aren't ordered, so we need to check both orders. assert repr(approx({'a': 1.0, 'b': 2.0})) in ( - "approx({{'a': 1.0 {pm} {tol1}, 'b': 2.0 {pm} {tol2}}})".format(pm=plus_minus, tol1=tol1, tol2=tol2), - "approx({{'b': 2.0 {pm} {tol2}, 'a': 1.0 {pm} {tol1}}})".format(pm=plus_minus, tol1=tol1, tol2=tol2), + "approx({{'a': 1.0 {pm} {tol1}, 'b': 2.0 {pm} {tol2}}})".format(pm=plus_minus, tol1=tol1, tol2=tol2), + "approx({{'b': 2.0 {pm} {tol2}, 'a': 1.0 {pm} {tol1}}})".format(pm=plus_minus, tol1=tol1, tol2=tol2), ) def test_operator_overloading(self): @@ -218,11 +219,11 @@ class TestApprox(object): def test_expecting_nan(self): examples = [ - (eq, nan, nan), - (eq, -nan, -nan), - (eq, nan, -nan), - (ne, 0.0, nan), - (ne, inf, nan), + (eq, nan, nan), + (eq, -nan, -nan), + (eq, nan, -nan), + (ne, 0.0, nan), + (ne, inf, nan), ] for op, a, x in examples: # Nothing is equal to NaN by default. @@ -266,10 +267,10 @@ class TestApprox(object): def test_complex(self): within_1e6 = [ - ( 1.000001 + 1.0j, 1.0 + 1.0j), - (1.0 + 1.000001j, 1.0 + 1.0j), - (-1.000001 + 1.0j, -1.0 + 1.0j), - (1.0 - 1.000001j, 1.0 - 1.0j), + (1.000001 + 1.0j, 1.0 + 1.0j), + (1.0 + 1.000001j, 1.0 + 1.0j), + (-1.000001 + 1.0j, -1.0 + 1.0j), + (1.0 - 1.000001j, 1.0 - 1.0j), ] for a, x in within_1e6: assert a == approx(x, rel=5e-6, abs=0) @@ -289,7 +290,7 @@ class TestApprox(object): def test_list_wrong_len(self): assert [1, 2] != approx([1]) - assert [1, 2] != approx([1,2,3]) + assert [1, 2] != approx([1, 2, 3]) def test_tuple(self): actual = (1 + 1e-7, 2 + 1e-8) @@ -303,7 +304,7 @@ class TestApprox(object): def test_tuple_wrong_len(self): assert (1, 2) != approx((1,)) - assert (1, 2) != approx((1,2,3)) + assert (1, 2) != approx((1, 2, 3)) def test_dict(self): actual = {'a': 1 + 1e-7, 'b': 2 + 1e-8} @@ -344,7 +345,7 @@ class TestApprox(object): np = pytest.importorskip('numpy') a12 = np.array([[1, 2]]) - a21 = np.array([[1],[2]]) + a21 = np.array([[1], [2]]) assert a12 != approx(a21) assert a21 != approx(a12) diff --git a/testing/python/collect.py b/testing/python/collect.py index 64a4ff7aa..977ef1c82 100644 --- a/testing/python/collect.py +++ b/testing/python/collect.py @@ -151,7 +151,7 @@ class TestClass(object): pass """) result = testdir.runpytest() - if sys.version_info < (2,7): + if sys.version_info < (2, 7): # in 2.6, the code to handle static methods doesn't work result.stdout.fnmatch_lines([ "*collected 0 items*", diff --git a/testing/test_cache.py b/testing/test_cache.py index 7fea5cdfd..36059ec29 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -119,6 +119,7 @@ class TestNewAPI(object): testdir.runpytest() assert testdir.tmpdir.join('custom_cache_dir').isdir() + def test_cache_reportheader(testdir): testdir.makepyfile(""" def test_hello(): diff --git a/testing/test_capture.py b/testing/test_capture.py index 302a02d10..a5d8c9c13 100644 --- a/testing/test_capture.py +++ b/testing/test_capture.py @@ -83,14 +83,14 @@ class TestCaptureManager(object): assert outerr == ("", "") outerr = capman.suspendcapture() assert outerr == ("", "") - print ("hello") + print("hello") out, err = capman.suspendcapture() if method == "no": assert old == (sys.stdout, sys.stderr, sys.stdin) else: assert not out capman.resumecapture() - print ("hello") + print("hello") out, err = capman.suspendcapture() if method != "no": assert out == "hello\n" @@ -305,7 +305,7 @@ class TestLoggingInteraction(object): assert 0 """) for optargs in (('--capture=sys',), ('--capture=fd',)): - print (optargs) + print(optargs) result = testdir.runpytest_subprocess(p, *optargs) s = result.stdout.str() result.stdout.fnmatch_lines([ @@ -331,7 +331,7 @@ class TestLoggingInteraction(object): assert 0 """) for optargs in (('--capture=sys',), ('--capture=fd',)): - print (optargs) + print(optargs) result = testdir.runpytest_subprocess(p, *optargs) s = result.stdout.str() result.stdout.fnmatch_lines([ @@ -879,7 +879,7 @@ class TestStdCapture(object): def test_capturing_readouterr(self): with self.getcapture() as cap: - print ("hello world") + print("hello world") sys.stderr.write("hello error\n") out, err = cap.readouterr() assert out == "hello world\n" @@ -890,7 +890,7 @@ class TestStdCapture(object): def test_capturing_readouterr_unicode(self): with self.getcapture() as cap: - print ("hx\xc4\x85\xc4\x87") + print("hx\xc4\x85\xc4\x87") out, err = cap.readouterr() assert out == py.builtin._totext("hx\xc4\x85\xc4\x87\n", "utf8") @@ -905,7 +905,7 @@ class TestStdCapture(object): def test_reset_twice_error(self): with self.getcapture() as cap: - print ("hello") + print("hello") out, err = cap.readouterr() pytest.raises(ValueError, cap.stop_capturing) assert out == "hello\n" @@ -919,7 +919,7 @@ class TestStdCapture(object): sys.stderr.write("world") sys.stdout = capture.CaptureIO() sys.stderr = capture.CaptureIO() - print ("not seen") + print("not seen") sys.stderr.write("not seen\n") out, err = cap.readouterr() assert out == "hello" @@ -929,9 +929,9 @@ class TestStdCapture(object): def test_capturing_error_recursive(self): with self.getcapture() as cap1: - print ("cap1") + print("cap1") with self.getcapture() as cap2: - print ("cap2") + print("cap2") out2, err2 = cap2.readouterr() out1, err1 = cap1.readouterr() assert out1 == "cap1\n" @@ -961,9 +961,9 @@ class TestStdCapture(object): assert sys.stdin is old def test_stdin_nulled_by_default(self): - print ("XXX this test may well hang instead of crashing") - print ("XXX which indicates an error in the underlying capturing") - print ("XXX mechanisms") + print("XXX this test may well hang instead of crashing") + print("XXX which indicates an error in the underlying capturing") + print("XXX mechanisms") with self.getcapture(): pytest.raises(IOError, "sys.stdin.read()") diff --git a/testing/test_conftest.py b/testing/test_conftest.py index 05453f766..39590f5f2 100644 --- a/testing/test_conftest.py +++ b/testing/test_conftest.py @@ -321,9 +321,9 @@ class TestConftestVisibility(object): # use value from parent dir's """)) - print ("created directory structure:") + print("created directory structure:") for x in testdir.tmpdir.visit(): - print (" " + x.relto(testdir.tmpdir)) + print(" " + x.relto(testdir.tmpdir)) return { "runner": runner, diff --git a/testing/test_mark.py b/testing/test_mark.py index 744e6ce52..f3966d733 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -791,11 +791,10 @@ def test_legacy_transfer(): def fake_method(self): pass - transfer_markers(fake_method, FakeClass, FakeModule) # legacy marks transfer smeared assert fake_method.nofun assert fake_method.fun # pristine marks dont transfer - assert fake_method.pytestmark == [pytest.mark.fun.mark] \ No newline at end of file + assert fake_method.pytestmark == [pytest.mark.fun.mark] diff --git a/testing/test_runner.py b/testing/test_runner.py index 842810f1b..567b98eeb 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -226,7 +226,7 @@ class BaseFunctionalTests(object): raise ValueError(42) """) reps = rec.getreports("pytest_runtest_logreport") - print (reps) + print(reps) for i in range(2): assert reps[i].nodeid.endswith("test_method") assert reps[i].passed @@ -253,7 +253,7 @@ class BaseFunctionalTests(object): assert True """) reps = rec.getreports("pytest_runtest_logreport") - print (reps) + print(reps) assert len(reps) == 3 # assert reps[0].nodeid.endswith("test_method") From 3a1c9c0e45de5d05b34b200c2491d24ce4b236b0 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 18 Jul 2017 15:37:01 -0300 Subject: [PATCH 37/73] Clarify in the docs how PYTEST_ADDOPTS and addopts ini option work together --- doc/en/customize.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/doc/en/customize.rst b/doc/en/customize.rst index f50d8b46e..7be7ca2e5 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -112,15 +112,27 @@ progress output, you can write it into a configuration file: # content of pytest.ini # (or tox.ini or setup.cfg) [pytest] - addopts = -rsxX -q + addopts = -ra -q -Alternatively, you can set a PYTEST_ADDOPTS environment variable to add command +Alternatively, you can set a ``PYTEST_ADDOPTS`` environment variable to add command line options while the environment is in use:: - export PYTEST_ADDOPTS="-rsxX -q" + export PYTEST_ADDOPTS="-v" -From now on, running ``pytest`` will add the specified options. +Here's how the command-line is built in the presence of ``addopts`` or the environment variable:: + $PYTEST_ADDOTPS + +So if the user executes in the command-line:: + + pytest -m slow + +The actual command line executed is:: + + pytest -ra -q -v -m slow + +Note that as usual for other command-line applications, in case of conflicting options the last one wins, so the example +above will show verbose output because ``-v`` overwrites ``-q``. Builtin configuration file options From 637e566d05c677d9ec71177412c787ef1af3548d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 18 Jul 2017 17:04:37 -0300 Subject: [PATCH 38/73] Separate all options for running/selecting tests into sections --- doc/en/usage.rst | 69 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/doc/en/usage.rst b/doc/en/usage.rst index 763328f5a..64c072886 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -52,23 +52,64 @@ To stop the testing process after the first (N) failures:: Specifying tests / selecting tests --------------------------------------------------- -Several test run options:: +Pytest supports several ways to run and select tests from the command-line. - pytest test_mod.py # run tests in module - pytest somepath # run all tests below somepath - pytest -k stringexpr # only run tests with names that match the - # "string expression", e.g. "MyClass and not method" - # will select TestMyClass.test_something - # but not TestMyClass.test_method_simple - pytest test_mod.py::test_func # only run tests that match the "node ID", - # e.g. "test_mod.py::test_func" will select - # only test_func in test_mod.py - pytest test_mod.py::TestClass::test_method # run a single method in - # a single class +**Run tests in a module** -Import 'pkg' and use its filesystem location to find and run tests:: +:: - pytest --pyargs pkg # run all tests found below directory of pkg + pytest test_mod.py + +**Run tests in a directory** + +:: + + pytest testing/ + +**Run tests by keyword expressions** + +:: + + pytest -k "MyClass and not method" + +This will run tests which contain names that match the given *string expression*, which can +include Python operators that use filenames, class names and function names as variables. +The example above will run ``TestMyClass.test_something`` but not ``TestMyClass.test_method_simple``. + +.. _nodeids: + +**Run tests by node ids** + +Each collected test is assigned a unique ``nodeid`` which consist of the module filename followed +by specifiers like class names, function names and parameters from parametrization, separated by ``::`` characters. + +To run a specific test within a module:: + + pytest test_mod.py::test_func + + +Another example specifying a test method in the command line:: + + pytest test_mod.py::TestClass::test_method + +**Run tests by marker expressions** + +:: + + pytest -m slow + +Will run all tests which are decorated with the ``@pytest.mark.slow`` decorator. + +For more information see :ref:`marks `. + +**Run tests from packages** + +:: + + pytest --pyargs pkg.testing + +This will import ``pkg.testing`` and use its filesystem location to find and run tests from. + Modifying Python traceback printing ---------------------------------------------- From 62556bada660ca209e7e83bc775559d9039f31b1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 18 Jul 2017 08:28:44 +0200 Subject: [PATCH 39/73] remove the MARK_INFO_ATTRIBUTE warning until we can fix internal usage fixes #2573 --- _pytest/mark.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index 961c3c409..61562330f 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -6,7 +6,7 @@ import warnings from collections import namedtuple from operator import attrgetter from .compat import imap -from .deprecated import MARK_INFO_ATTRIBUTE, MARK_PARAMETERSET_UNPACKING +from .deprecated import MARK_PARAMETERSET_UNPACKING def alias(name, warning=None): getter = attrgetter(name) @@ -401,9 +401,9 @@ class MarkInfo(object): self.combined = mark self._marks = [mark] - name = alias('combined.name', warning=MARK_INFO_ATTRIBUTE) - args = alias('combined.args', warning=MARK_INFO_ATTRIBUTE) - kwargs = alias('combined.kwargs', warning=MARK_INFO_ATTRIBUTE) + name = alias('combined.name') + args = alias('combined.args') + kwargs = alias('combined.kwargs') def __repr__(self): return "".format(self.combined) From 2d4f1f022eb83d3029a35c3ef854534263af6c16 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 18 Jul 2017 17:18:34 -0300 Subject: [PATCH 40/73] Introduce PYTEST_CURRENT_TEST environment variable Fix #2583 --- _pytest/runner.py | 19 ++++++++++++++++++ changelog/2583.feature | 2 ++ doc/en/example/simple.rst | 41 +++++++++++++++++++++++++++++++++++++++ testing/test_runner.py | 27 ++++++++++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 changelog/2583.feature diff --git a/_pytest/runner.py b/_pytest/runner.py index fd0b549a9..27be8f4d1 100644 --- a/_pytest/runner.py +++ b/_pytest/runner.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function import bdb +import os import sys from time import time @@ -91,9 +92,11 @@ def show_test_item(item): tw.write(' (fixtures used: {0})'.format(', '.join(used_fixtures))) def pytest_runtest_setup(item): + _update_current_test_var(item, 'setup') item.session._setupstate.prepare(item) def pytest_runtest_call(item): + _update_current_test_var(item, 'call') try: item.runtest() except Exception: @@ -107,7 +110,23 @@ def pytest_runtest_call(item): raise def pytest_runtest_teardown(item, nextitem): + _update_current_test_var(item, 'teardown') item.session._setupstate.teardown_exact(item, nextitem) + _update_current_test_var(item, None) + + +def _update_current_test_var(item, when): + """ + Update PYTEST_CURRENT_TEST to reflect the current item and stage. + + If ``when`` is None, delete PYTEST_CURRENT_TEST from the environment. + """ + var_name = 'PYTEST_CURRENT_TEST' + if when: + os.environ[var_name] = '{0} ({1})'.format(item.nodeid, when) + else: + os.environ.pop(var_name) + def pytest_report_teststatus(report): if report.when in ("setup", "teardown"): diff --git a/changelog/2583.feature b/changelog/2583.feature new file mode 100644 index 000000000..315f2378e --- /dev/null +++ b/changelog/2583.feature @@ -0,0 +1,2 @@ +Introduce the ``PYTEST_CURRENT_TEST`` environment variable that is set with the ``nodeid`` and stage (``setup``, ``call`` and +``teardown``) of the test being currently executed. See the `documentation `_ for more info. diff --git a/doc/en/example/simple.rst b/doc/en/example/simple.rst index da831244b..6b5d5a868 100644 --- a/doc/en/example/simple.rst +++ b/doc/en/example/simple.rst @@ -761,6 +761,47 @@ and run it:: You'll see that the fixture finalizers could use the precise reporting information. +``PYTEST_CURRENT_TEST`` environment variable +-------------------------------------------- + +.. versionadded:: 3.2 + +Sometimes a test session might get stuck and there might be no easy way to figure out +which test got stuck, for example if pytest was run in quiet mode (``-q``) or you don't have access to the console +output. This is particularly a problem if the problem helps only sporadically, the famous "flaky" kind of tests. + +``pytest`` sets a ``PYTEST_CURRENT_TEST`` environment variable when running tests, which can be inspected +by process monitoring utilities or libraries like `psutil `_ to discover which +test got stuck if necessary: + +.. code-block:: python + + import psutil + + for pid in psutil.pids(): + environ = psutil.Process(pid).environ() + if 'PYTEST_CURRENT_TEST' in environ: + print(f'pytest process {pid} running: {environ["PYTEST_CURRENT_TEST"]}') + +During the test session pytest will set ``PYTEST_CURRENT_TEST`` to the current test +:ref:`nodeid ` and the current stage, which can be ``setup``, ``call`` +and ``teardown``. + +For example, when running a single test function named ``test_foo`` from ``foo_module.py``, +``PYTEST_CURRENT_TEST`` will be set to: + +#. ``foo_module.py::test_foo (setup)`` +#. ``foo_module.py::test_foo (call)`` +#. ``foo_module.py::test_foo (teardown)`` + +In that order. + +.. note:: + + The contents of ``PYTEST_CURRENT_TEST`` is meant to be human readable and the actual format + can be changed between releases (even bug fixes) so it shouldn't be relied on for scripting + or automation. + Freezing pytest --------------- diff --git a/testing/test_runner.py b/testing/test_runner.py index def80ea5f..e70d955ac 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -681,6 +681,8 @@ def test_store_except_info_on_eror(): """ # Simulate item that raises a specific exception class ItemThatRaises(object): + nodeid = 'item_that_raises' + def runtest(self): raise IndexError('TEST') try: @@ -693,6 +695,31 @@ def test_store_except_info_on_eror(): assert sys.last_traceback +def test_current_test_env_var(testdir, monkeypatch): + pytest_current_test_vars = [] + monkeypatch.setattr(sys, 'pytest_current_test_vars', pytest_current_test_vars, raising=False) + testdir.makepyfile(''' + import pytest + import sys + import os + + @pytest.fixture + def fix(): + sys.pytest_current_test_vars.append(('setup', os.environ['PYTEST_CURRENT_TEST'])) + yield + sys.pytest_current_test_vars.append(('teardown', os.environ['PYTEST_CURRENT_TEST'])) + + def test(fix): + sys.pytest_current_test_vars.append(('call', os.environ['PYTEST_CURRENT_TEST'])) + ''') + result = testdir.runpytest_inprocess() + assert result.ret == 0 + test_id = 'test_current_test_env_var.py::test' + assert pytest_current_test_vars == [ + ('setup', test_id + ' (setup)'), ('call', test_id + ' (call)'), ('teardown', test_id + ' (teardown)')] + assert 'PYTEST_CURRENT_TEST' not in os.environ + + class TestReportContents(object): """ Test user-level API of ``TestReport`` objects. From d7f182ac4fcc1a416b456a02b0ed0508c1659d1c Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 19 Jul 2017 10:02:13 -0300 Subject: [PATCH 41/73] Remove SETUPTOOLS_SCM_PRETEND_VERSION during linting It was needed because of check-manifest, but we no longer have a MANIFEST file so it is not necessary --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index cc9e64b1c..3dd094866 100644 --- a/tox.ini +++ b/tox.ini @@ -49,9 +49,6 @@ commands= skipsdist=True usedevelop=True basepython = python2.7 -# needed to keep check-manifest working -setenv = - SETUPTOOLS_SCM_PRETEND_VERSION=2.0.1 deps = flake8 # pygments required by rst-lint From 24da93832138b048c9126bbd7f1ab7bf6c091edf Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 19 Jul 2017 17:42:21 -0300 Subject: [PATCH 42/73] Fix additional flake8 errors --- testing/code/test_source.py | 3 ++- testing/python/approx.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 6432bf95c..f8b6af3ee 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -342,7 +342,8 @@ def test_getstartingblock_multiline(): self.source = _pytest._code.Frame(frame).statement x = A('x', - 'y', + 'y' + , 'z') l = [i for i in x.source.lines if i.strip()] diff --git a/testing/python/approx.py b/testing/python/approx.py index ebd0234de..876226e06 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -29,7 +29,9 @@ class TestApprox(object): if sys.version_info[:2] == (2, 6): tol1, tol2, infr = '???', '???', '???' assert repr(approx(1.0)) == '1.0 {pm} {tol1}'.format(pm=plus_minus, tol1=tol1) - assert repr(approx([1.0, 2.0])) == '1.0 {pm} {tol1}, 2.0 {pm} {tol2}'.format( + assert repr(approx([1.0, 2.0])) == 'approx([1.0 {pm} {tol1}, 2.0 {pm} {tol2}])'.format( + pm=plus_minus, tol1=tol1, tol2=tol2) + assert repr(approx((1.0, 2.0))) == 'approx((1.0 {pm} {tol1}, 2.0 {pm} {tol2}))'.format( pm=plus_minus, tol1=tol1, tol2=tol2) assert repr(approx(inf)) == 'inf' assert repr(approx(1.0, rel=nan)) == '1.0 {pm} ???'.format(pm=plus_minus) From 7341da1bc127f69b51957784252e1cd00623dfc0 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 20 Jul 2017 22:02:21 -0300 Subject: [PATCH 43/73] Introduce pytest.mark.filterwarnings --- _pytest/warnings.py | 5 +++++ changelog/2598.feature | 2 ++ doc/en/warnings.rst | 34 ++++++++++++++++++++++++++++++++++ testing/test_warnings.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+) create mode 100644 changelog/2598.feature diff --git a/_pytest/warnings.py b/_pytest/warnings.py index 915862a9d..1b3c8387e 100644 --- a/_pytest/warnings.py +++ b/_pytest/warnings.py @@ -59,6 +59,11 @@ def catch_warnings_for_item(item): for arg in inifilters: _setoption(warnings, arg) + mark = item.get_marker('filterwarnings') + if mark: + for arg in mark.args: + warnings._setoption(arg) + yield for warning in log: diff --git a/changelog/2598.feature b/changelog/2598.feature new file mode 100644 index 000000000..b811b9120 --- /dev/null +++ b/changelog/2598.feature @@ -0,0 +1,2 @@ +Introduced ``@pytest.mark.filterwarnings`` mark which allows overwriting the warnings filter on a per test, class or module level. +See the `docs `_ for more information. diff --git a/doc/en/warnings.rst b/doc/en/warnings.rst index 34dc1ece0..c84277173 100644 --- a/doc/en/warnings.rst +++ b/doc/en/warnings.rst @@ -78,6 +78,40 @@ Both ``-W`` command-line option and ``filterwarnings`` ini option are based on P `-W option`_ and `warnings.simplefilter`_, so please refer to those sections in the Python documentation for other examples and advanced usage. +``@pytest.mark.filterwarnings`` +------------------------------- + +.. versionadded:: 3.2 + +You can use the ``@pytest.mark.filterwarnings`` to add warning filters to specific test items, +allowing you to have finer control of which warnings should be captured at test, class or +even module level: + +.. code-block:: python + + import warnings + + def api_v1(): + warnings.warn(UserWarning("api v1, should use functions from v2")) + return 1 + + @pytest.mark.filterwarnings('ignore:api v1') + def test_one(): + assert api_v1() == 1 + + +Filters applied using a mark take precedence over filters passed on the command line or configured +by the ``filterwarnings`` ini option. + +You may apply a filter to all tests of a class by using the ``filterwarnings`` mark as a class +decorator or to all tests in a module by setting the ``pytestmark`` variable: + +.. code-block:: python + + # turns all warnings into errors for this module + pytestmark = @pytest.mark.filterwarnings('error') + + .. note:: ``DeprecationWarning`` and ``PendingDeprecationWarning`` are hidden by the standard library diff --git a/testing/test_warnings.py b/testing/test_warnings.py index 858932846..58e6bcf74 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -188,3 +188,32 @@ def test_works_with_filterwarnings(testdir): result.stdout.fnmatch_lines([ '*== 1 passed in *', ]) + + +@pytest.mark.parametrize('default_config', ['ini', 'cmdline']) +def test_filterwarnings_mark(testdir, default_config): + """ + Test ``filterwarnings`` mark works and takes precedence over command line and ini options. + """ + if default_config == 'ini': + testdir.makeini(""" + [pytest] + filterwarnings = always + """) + testdir.makepyfile(""" + import warnings + import pytest + + @pytest.mark.filterwarnings('ignore::RuntimeWarning') + def test_ignore_runtime_warning(): + warnings.warn(RuntimeWarning()) + + @pytest.mark.filterwarnings('error') + def test_warning_error(): + warnings.warn(RuntimeWarning()) + + def test_show_warning(): + warnings.warn(RuntimeWarning()) + """) + result = testdir.runpytest('-W always' if default_config == 'cmdline' else '') + result.stdout.fnmatch_lines(['*= 1 failed, 2 passed, 1 warnings in *']) From 65b2de13a370ef743bf9be3259fc34ed43da29f1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 21 Jul 2017 07:26:56 +0200 Subject: [PATCH 44/73] fix #2540, introduce mark.with_args --- _pytest/mark.py | 15 ++++++++++++--- changelog/2540.feature | 1 + testing/test_mark.py | 13 +++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 changelog/2540.feature diff --git a/_pytest/mark.py b/_pytest/mark.py index 5ded0afe8..74473a9d7 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -335,6 +335,17 @@ class MarkDecorator: def __repr__(self): return "" % (self.mark,) + def with_args(self, *args, **kwargs): + """ return a MarkDecorator with extra arguments added + + unlike call this can be used even if the sole argument is a callable/class + + :return: MarkDecorator + """ + + mark = Mark(self.name, args, kwargs) + return self.__class__(self.mark.combined_with(mark)) + def __call__(self, *args, **kwargs): """ if passed a single callable argument: decorate it with mark info. otherwise add *args/**kwargs in-place to mark information. """ @@ -348,9 +359,7 @@ class MarkDecorator: store_legacy_markinfo(func, self.mark) store_mark(func, self.mark) return func - - mark = Mark(self.name, args, kwargs) - return self.__class__(self.mark.combined_with(mark)) + return self.with_args(*args, **kwargs) def get_unpacked_marks(obj): diff --git a/changelog/2540.feature b/changelog/2540.feature new file mode 100644 index 000000000..d65b1ea56 --- /dev/null +++ b/changelog/2540.feature @@ -0,0 +1 @@ +Introduce ``mark.with_args`` in order to allow passing functions/classes as sole argument to marks. \ No newline at end of file diff --git a/testing/test_mark.py b/testing/test_mark.py index f3966d733..cc46702a5 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -22,6 +22,19 @@ class TestMark(object): mark = Mark() pytest.raises((AttributeError, TypeError), mark) + def test_mark_with_param(self): + def some_function(abc): + pass + + class SomeClass(object): + pass + + assert pytest.mark.fun(some_function) is some_function + assert pytest.mark.fun.with_args(some_function) is not some_function + + assert pytest.mark.fun(SomeClass) is SomeClass + assert pytest.mark.fun.with_args(SomeClass) is not SomeClass + def test_pytest_mark_name_starts_with_underscore(self): mark = Mark() pytest.raises(AttributeError, getattr, mark, '_some_name') From 495f731760341130deeeb05bef504921d20b7473 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sat, 22 Jul 2017 07:52:03 -0700 Subject: [PATCH 45/73] Simplify how comparisons with numpy arrays work. Previously I was subverting the natural order of operations by subclassing from `ndarray`, but it turns out that you can tell just numpy to call your operator instead of its by setting the `__array_priority__` attribute on your class. This is much simpler, and it turns out the be a little more robust, too. --- _pytest/python_api.py | 62 ++++++------------------------------------- 1 file changed, 8 insertions(+), 54 deletions(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index c5493f495..fe2222059 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -46,60 +46,13 @@ class ApproxBase(object): raise NotImplementedError -class ApproxNumpyBase(ApproxBase): +class ApproxNumpy(ApproxBase): """ Perform approximate comparisons for numpy arrays. - - This class should not be used directly. Instead, the `inherit_ndarray()` - class method should be used to make a subclass that also inherits from - `np.ndarray`. This indirection is necessary because the object doing the - approximate comparison must inherit from `np.ndarray`, or it will only work - on the left side of the `==` operator. But importing numpy is relatively - expensive, so we also want to avoid that unless we actually have a numpy - array to compare. - - The reason why the approx object needs to inherit from `np.ndarray` has to - do with how python decides whether to call `a.__eq__()` or `b.__eq__()` - when it parses `a == b`. If `a` and `b` are not related by inheritance, - `a` gets priority. So as long as `a.__eq__` is defined, it will be called. - Because most implementations of `a.__eq__` end up calling `b.__eq__`, this - detail usually doesn't matter. However, `np.ndarray.__eq__` treats the - approx object as a scalar and builds a new array by comparing it to each - item in the original array. `b.__eq__` is called to compare against each - individual element in the array, but it has no way (that I can see) to - prevent the return value from being an boolean array, and boolean arrays - can't be used with assert because "the truth value of an array with more - than one element is ambiguous." - - The trick is that the priority rules change if `a` and `b` are related - by inheritance. Specifically, `b.__eq__` gets priority if `b` is a - subclass of `a`. So by inheriting from `np.ndarray`, we can guarantee that - `ApproxNumpy.__eq__` gets called no matter which side of the `==` operator - it appears on. """ - subclass = None - - @classmethod - def inherit_ndarray(cls): - import numpy as np - assert not isinstance(cls, np.ndarray) - - if cls.subclass is None: - cls.subclass = type('ApproxNumpy', (cls, np.ndarray), {}) - - return cls.subclass - - def __new__(cls, expected, rel=None, abs=None, nan_ok=False): - """ - Numpy uses __new__ (rather than __init__) to initialize objects. - - The `expected` argument must be a numpy array. This should be - ensured by the approx() delegator function. - """ - obj = super(ApproxNumpyBase, cls).__new__(cls, ()) - obj.__init__(expected, rel, abs, nan_ok) - return obj + # Tell numpy to use our `__eq__` operator instead of its. + __array_priority__ = 100 def __repr__(self): # It might be nice to rewrite this function to account for the @@ -113,7 +66,7 @@ class ApproxNumpyBase(ApproxBase): try: actual = np.asarray(actual) except: - raise ValueError("cannot cast '{0}' to numpy.ndarray".format(actual)) + raise ValueError("cannot compare '{0}' to numpy.ndarray".format(actual)) if actual.shape != self.expected.shape: return False @@ -157,6 +110,9 @@ class ApproxSequence(ApproxBase): Perform approximate comparisons for sequences of numbers. """ + # Tell numpy to use our `__eq__` operator instead of its. + __array_priority__ = 100 + def __repr__(self): seq_type = type(self.expected) if seq_type not in (tuple, list, set): @@ -422,9 +378,7 @@ def approx(expected, rel=None, abs=None, nan_ok=False): # their keys, which is probably not what most people would expect. if _is_numpy_array(expected): - # Create the delegate class on the fly. This allow us to inherit from - # ``np.ndarray`` while still not importing numpy unless we need to. - cls = ApproxNumpyBase.inherit_ndarray() + cls = ApproxNumpy elif isinstance(expected, Mapping): cls = ApproxMapping elif isinstance(expected, Sequence) and not isinstance(expected, String): From 4c45bc997149582f05a1c4853838190194a074be Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sat, 22 Jul 2017 08:24:45 -0700 Subject: [PATCH 46/73] Add the numpy tests back into tox.ini I'm not sure why they were removed... --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1217f4032..da5ae9fec 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ envlist = py36 py37 pypy - {py27,py35}-{pexpect,xdist,trial} + {py27,py35}-{pexpect,xdist,trial,numpy} py27-nobyte doctesting freeze From a3b35e1c4b73ee93dc3399d1a0ce8035efaf9e8b Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sat, 22 Jul 2017 08:36:15 -0700 Subject: [PATCH 47/73] Remove `raises` and `approx` from `python.py`. These two classes were recently moved to `python_api.py`, but it seems that they found their way back into the original file somehow. This commit removes them again to avoid out-of-date code duplication. --- _pytest/python.py | 432 ---------------------------------------------- 1 file changed, 432 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 74998c93e..557471b36 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1096,438 +1096,6 @@ def write_docstring(tw, doc): tw.write(INDENT + line + "\n") -# builtin pytest.raises helper - -def raises(expected_exception, *args, **kwargs): - """ - Assert that a code block/function call raises ``expected_exception`` - and raise a failure exception otherwise. - - This helper produces a ``ExceptionInfo()`` object (see below). - - If using Python 2.5 or above, you may use this function as a - context manager:: - - >>> with raises(ZeroDivisionError): - ... 1/0 - - .. versionchanged:: 2.10 - - In the context manager form you may use the keyword argument - ``message`` to specify a custom failure message:: - - >>> with raises(ZeroDivisionError, message="Expecting ZeroDivisionError"): - ... pass - Traceback (most recent call last): - ... - Failed: Expecting ZeroDivisionError - - - .. note:: - - When using ``pytest.raises`` as a context manager, it's worthwhile to - note that normal context manager rules apply and that the exception - raised *must* be the final line in the scope of the context manager. - Lines of code after that, within the scope of the context manager will - not be executed. For example:: - - >>> value = 15 - >>> with raises(ValueError) as exc_info: - ... if value > 10: - ... raise ValueError("value must be <= 10") - ... assert exc_info.type == ValueError # this will not execute - - Instead, the following approach must be taken (note the difference in - scope):: - - >>> with raises(ValueError) as exc_info: - ... if value > 10: - ... raise ValueError("value must be <= 10") - ... - >>> assert exc_info.type == ValueError - - Or you can use the keyword argument ``match`` to assert that the - exception matches a text or regex:: - - >>> with raises(ValueError, match='must be 0 or None'): - ... raise ValueError("value must be 0 or None") - - >>> with raises(ValueError, match=r'must be \d+$'): - ... raise ValueError("value must be 42") - - - Or you can specify a callable by passing a to-be-called lambda:: - - >>> raises(ZeroDivisionError, lambda: 1/0) - - - or you can specify an arbitrary callable with arguments:: - - >>> def f(x): return 1/x - ... - >>> raises(ZeroDivisionError, f, 0) - - >>> raises(ZeroDivisionError, f, x=0) - - - A third possibility is to use a string to be executed:: - - >>> raises(ZeroDivisionError, "f(0)") - - - .. autoclass:: _pytest._code.ExceptionInfo - :members: - - .. note:: - Similar to caught exception objects in Python, explicitly clearing - local references to returned ``ExceptionInfo`` objects can - help the Python interpreter speed up its garbage collection. - - Clearing those references breaks a reference cycle - (``ExceptionInfo`` --> caught exception --> frame stack raising - the exception --> current frame stack --> local variables --> - ``ExceptionInfo``) which makes Python keep all objects referenced - from that cycle (including all local variables in the current - frame) alive until the next cyclic garbage collection run. See the - official Python ``try`` statement documentation for more detailed - information. - - """ - __tracebackhide__ = True - msg = ("exceptions must be old-style classes or" - " derived from BaseException, not %s") - if isinstance(expected_exception, tuple): - for exc in expected_exception: - if not isclass(exc): - raise TypeError(msg % type(exc)) - elif not isclass(expected_exception): - raise TypeError(msg % type(expected_exception)) - - message = "DID NOT RAISE {0}".format(expected_exception) - match_expr = None - - if not args: - if "message" in kwargs: - message = kwargs.pop("message") - if "match" in kwargs: - match_expr = kwargs.pop("match") - message += " matching '{0}'".format(match_expr) - return RaisesContext(expected_exception, message, match_expr) - elif isinstance(args[0], str): - code, = args - assert isinstance(code, str) - frame = sys._getframe(1) - loc = frame.f_locals.copy() - loc.update(kwargs) - # print "raises frame scope: %r" % frame.f_locals - try: - code = _pytest._code.Source(code).compile() - py.builtin.exec_(code, frame.f_globals, loc) - # XXX didn'T mean f_globals == f_locals something special? - # this is destroyed here ... - except expected_exception: - return _pytest._code.ExceptionInfo() - else: - func = args[0] - try: - func(*args[1:], **kwargs) - except expected_exception: - return _pytest._code.ExceptionInfo() - fail(message) - - -raises.Exception = fail.Exception - - -class RaisesContext(object): - def __init__(self, expected_exception, message, match_expr): - self.expected_exception = expected_exception - self.message = message - self.match_expr = match_expr - self.excinfo = None - - def __enter__(self): - self.excinfo = object.__new__(_pytest._code.ExceptionInfo) - return self.excinfo - - def __exit__(self, *tp): - __tracebackhide__ = True - if tp[0] is None: - fail(self.message) - if sys.version_info < (2, 7): - # py26: on __exit__() exc_value often does not contain the - # exception value. - # http://bugs.python.org/issue7853 - if not isinstance(tp[1], BaseException): - exc_type, value, traceback = tp - tp = exc_type, exc_type(value), traceback - self.excinfo.__init__(tp) - suppress_exception = issubclass(self.excinfo.type, self.expected_exception) - if sys.version_info[0] == 2 and suppress_exception: - sys.exc_clear() - if self.match_expr: - self.excinfo.match(self.match_expr) - return suppress_exception - - -# builtin pytest.approx helper - -class approx(object): - """ - Assert that two numbers (or two sets of numbers) are equal to each other - within some tolerance. - - Due to the `intricacies of floating-point arithmetic`__, numbers that we - would intuitively expect to be equal are not always so:: - - >>> 0.1 + 0.2 == 0.3 - False - - __ https://docs.python.org/3/tutorial/floatingpoint.html - - This problem is commonly encountered when writing tests, e.g. when making - sure that floating-point values are what you expect them to be. One way to - deal with this problem is to assert that two floating-point numbers are - equal to within some appropriate tolerance:: - - >>> abs((0.1 + 0.2) - 0.3) < 1e-6 - True - - However, comparisons like this are tedious to write and difficult to - understand. Furthermore, absolute comparisons like the one above are - usually discouraged because there's no tolerance that works well for all - situations. ``1e-6`` is good for numbers around ``1``, but too small for - very big numbers and too big for very small ones. It's better to express - the tolerance as a fraction of the expected value, but relative comparisons - like that are even more difficult to write correctly and concisely. - - The ``approx`` class performs floating-point comparisons using a syntax - that's as intuitive as possible:: - - >>> from pytest import approx - >>> 0.1 + 0.2 == approx(0.3) - True - - The same syntax also works on sequences of numbers:: - - >>> (0.1 + 0.2, 0.2 + 0.4) == approx((0.3, 0.6)) - True - - By default, ``approx`` considers numbers within a relative tolerance of - ``1e-6`` (i.e. one part in a million) of its expected value to be equal. - This treatment would lead to surprising results if the expected value was - ``0.0``, because nothing but ``0.0`` itself is relatively close to ``0.0``. - To handle this case less surprisingly, ``approx`` also considers numbers - within an absolute tolerance of ``1e-12`` of its expected value to be - equal. Infinite numbers are another special case. They are only - considered equal to themselves, regardless of the relative tolerance. Both - the relative and absolute tolerances can be changed by passing arguments to - the ``approx`` constructor:: - - >>> 1.0001 == approx(1) - False - >>> 1.0001 == approx(1, rel=1e-3) - True - >>> 1.0001 == approx(1, abs=1e-3) - True - - If you specify ``abs`` but not ``rel``, the comparison will not consider - the relative tolerance at all. In other words, two numbers that are within - the default relative tolerance of ``1e-6`` will still be considered unequal - if they exceed the specified absolute tolerance. If you specify both - ``abs`` and ``rel``, the numbers will be considered equal if either - tolerance is met:: - - >>> 1 + 1e-8 == approx(1) - True - >>> 1 + 1e-8 == approx(1, abs=1e-12) - False - >>> 1 + 1e-8 == approx(1, rel=1e-6, abs=1e-12) - True - - If you're thinking about using ``approx``, then you might want to know how - it compares to other good ways of comparing floating-point numbers. All of - these algorithms are based on relative and absolute tolerances and should - agree for the most part, but they do have meaningful differences: - - - ``math.isclose(a, b, rel_tol=1e-9, abs_tol=0.0)``: True if the relative - tolerance is met w.r.t. either ``a`` or ``b`` or if the absolute - tolerance is met. Because the relative tolerance is calculated w.r.t. - both ``a`` and ``b``, this test is symmetric (i.e. neither ``a`` nor - ``b`` is a "reference value"). You have to specify an absolute tolerance - if you want to compare to ``0.0`` because there is no tolerance by - default. Only available in python>=3.5. `More information...`__ - - __ https://docs.python.org/3/library/math.html#math.isclose - - - ``numpy.isclose(a, b, rtol=1e-5, atol=1e-8)``: True if the difference - between ``a`` and ``b`` is less that the sum of the relative tolerance - w.r.t. ``b`` and the absolute tolerance. Because the relative tolerance - is only calculated w.r.t. ``b``, this test is asymmetric and you can - think of ``b`` as the reference value. Support for comparing sequences - is provided by ``numpy.allclose``. `More information...`__ - - __ http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.isclose.html - - - ``unittest.TestCase.assertAlmostEqual(a, b)``: True if ``a`` and ``b`` - are within an absolute tolerance of ``1e-7``. No relative tolerance is - considered and the absolute tolerance cannot be changed, so this function - is not appropriate for very large or very small numbers. Also, it's only - available in subclasses of ``unittest.TestCase`` and it's ugly because it - doesn't follow PEP8. `More information...`__ - - __ https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertAlmostEqual - - - ``a == pytest.approx(b, rel=1e-6, abs=1e-12)``: True if the relative - tolerance is met w.r.t. ``b`` or if the absolute tolerance is met. - Because the relative tolerance is only calculated w.r.t. ``b``, this test - is asymmetric and you can think of ``b`` as the reference value. In the - special case that you explicitly specify an absolute tolerance but not a - relative tolerance, only the absolute tolerance is considered. - """ - - def __init__(self, expected, rel=None, abs=None): - self.expected = expected - self.abs = abs - self.rel = rel - - def __repr__(self): - return ', '.join(repr(x) for x in self.expected) - - def __eq__(self, actual): - from collections import Iterable - if not isinstance(actual, Iterable): - actual = [actual] - if len(actual) != len(self.expected): - return False - return all(a == x for a, x in zip(actual, self.expected)) - - __hash__ = None - - def __ne__(self, actual): - return not (actual == self) - - @property - def expected(self): - # Regardless of whether the user-specified expected value is a number - # or a sequence of numbers, return a list of ApproxNotIterable objects - # that can be compared against. - from collections import Iterable - - def approx_non_iter(x): - return ApproxNonIterable(x, self.rel, self.abs) - - if isinstance(self._expected, Iterable): - return [approx_non_iter(x) for x in self._expected] - else: - return [approx_non_iter(self._expected)] - - @expected.setter - def expected(self, expected): - self._expected = expected - - -class ApproxNonIterable(object): - """ - Perform approximate comparisons for single numbers only. - - In other words, the ``expected`` attribute for objects of this class must - be some sort of number. This is in contrast to the ``approx`` class, where - the ``expected`` attribute can either be a number of a sequence of numbers. - This class is responsible for making comparisons, while ``approx`` is - responsible for abstracting the difference between numbers and sequences of - numbers. Although this class can stand on its own, it's only meant to be - used within ``approx``. - """ - - def __init__(self, expected, rel=None, abs=None): - self.expected = expected - self.abs = abs - self.rel = rel - - def __repr__(self): - if isinstance(self.expected, complex): - return str(self.expected) - - # Infinities aren't compared using tolerances, so don't show a - # tolerance. - if math.isinf(self.expected): - return str(self.expected) - - # If a sensible tolerance can't be calculated, self.tolerance will - # raise a ValueError. In this case, display '???'. - try: - vetted_tolerance = '{:.1e}'.format(self.tolerance) - except ValueError: - vetted_tolerance = '???' - - if sys.version_info[0] == 2: - return '{0} +- {1}'.format(self.expected, vetted_tolerance) - else: - return u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) - - def __eq__(self, actual): - # Short-circuit exact equality. - if actual == self.expected: - return True - - # Infinity shouldn't be approximately equal to anything but itself, but - # if there's a relative tolerance, it will be infinite and infinity - # will seem approximately equal to everything. The equal-to-itself - # case would have been short circuited above, so here we can just - # return false if the expected value is infinite. The abs() call is - # for compatibility with complex numbers. - if math.isinf(abs(self.expected)): - return False - - # Return true if the two numbers are within the tolerance. - return abs(self.expected - actual) <= self.tolerance - - __hash__ = None - - def __ne__(self, actual): - return not (actual == self) - - @property - def tolerance(self): - def set_default(x, default): - return x if x is not None else default - - # Figure out what the absolute tolerance should be. ``self.abs`` is - # either None or a value specified by the user. - absolute_tolerance = set_default(self.abs, 1e-12) - - if absolute_tolerance < 0: - raise ValueError("absolute tolerance can't be negative: {}".format(absolute_tolerance)) - if math.isnan(absolute_tolerance): - raise ValueError("absolute tolerance can't be NaN.") - - # If the user specified an absolute tolerance but not a relative one, - # just return the absolute tolerance. - if self.rel is None: - if self.abs is not None: - return absolute_tolerance - - # Figure out what the relative tolerance should be. ``self.rel`` is - # either None or a value specified by the user. This is done after - # we've made sure the user didn't ask for an absolute tolerance only, - # because we don't want to raise errors about the relative tolerance if - # we aren't even going to use it. - relative_tolerance = set_default(self.rel, 1e-6) * abs(self.expected) - - if relative_tolerance < 0: - raise ValueError("relative tolerance can't be negative: {}".format(absolute_tolerance)) - if math.isnan(relative_tolerance): - raise ValueError("relative tolerance can't be NaN.") - - # Return the larger of the relative and absolute tolerances. - return max(relative_tolerance, absolute_tolerance) - -# -# the basic pytest Function item -# - - class Function(FunctionMixin, main.Item, fixtures.FuncargnamesCompatAttr): """ a Function Item is responsible for setting up and executing a Python test function. From ebc7346be4da2d56e3c6f1686f545105f70b9c89 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sat, 22 Jul 2017 09:05:12 -0700 Subject: [PATCH 48/73] Raise TypeError for types that can't be compared to arrays. --- _pytest/python_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index fe2222059..176aff590 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -66,7 +66,7 @@ class ApproxNumpy(ApproxBase): try: actual = np.asarray(actual) except: - raise ValueError("cannot compare '{0}' to numpy.ndarray".format(actual)) + raise TypeError("cannot compare '{0}' to numpy.ndarray".format(actual)) if actual.shape != self.expected.shape: return False From 7e0553267dbe80fc389d68583a16c05d11382607 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sat, 22 Jul 2017 09:19:13 -0700 Subject: [PATCH 49/73] Remove unused import. --- _pytest/python.py | 1 - 1 file changed, 1 deletion(-) diff --git a/_pytest/python.py b/_pytest/python.py index 557471b36..c505484e4 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -9,7 +9,6 @@ import collections from textwrap import dedent from itertools import count -import math import py from _pytest.mark import MarkerError from _pytest.config import hookimpl From 0726d9a09f26dece8f0638bb644add34c3e9359f Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 20 Jul 2017 23:11:14 -0300 Subject: [PATCH 50/73] Turn warnings into errors in pytest's own test suite Fix #2588 --- changelog/2588.trivial | 1 + testing/python/collect.py | 12 +++++++++++- testing/python/metafunc.py | 3 ++- testing/python/setup_only.py | 2 +- testing/test_mark.py | 1 + testing/test_recwarn.py | 28 +--------------------------- testing/test_unittest.py | 14 +++++++------- testing/test_warnings.py | 4 ++++ tox.ini | 1 + 9 files changed, 29 insertions(+), 37 deletions(-) create mode 100644 changelog/2588.trivial diff --git a/changelog/2588.trivial b/changelog/2588.trivial new file mode 100644 index 000000000..44ff69f74 --- /dev/null +++ b/changelog/2588.trivial @@ -0,0 +1 @@ +Turn warnings into errors in pytest's own test suite in order to catch regressions due to deprecations more promptly. diff --git a/testing/python/collect.py b/testing/python/collect.py index 977ef1c82..bd7013b44 100644 --- a/testing/python/collect.py +++ b/testing/python/collect.py @@ -12,6 +12,9 @@ from _pytest.main import ( ) +ignore_parametrized_marks = pytest.mark.filterwarnings('ignore:Applying marks directly to parameters') + + class TestModule(object): def test_failing_import(self, testdir): modcol = testdir.getmodulecol("import alksdjalskdjalkjals") @@ -567,7 +570,8 @@ class TestFunction(object): rec = testdir.inline_run() rec.assertoutcome(passed=1) - def test_parametrize_with_mark(selfself, testdir): + @ignore_parametrized_marks + def test_parametrize_with_mark(self, testdir): items = testdir.getitems(""" import pytest @pytest.mark.foo @@ -640,6 +644,7 @@ class TestFunction(object): assert colitems[2].name == 'test2[a-c]' assert colitems[3].name == 'test2[b-c]' + @ignore_parametrized_marks def test_parametrize_skipif(self, testdir): testdir.makepyfile(""" import pytest @@ -653,6 +658,7 @@ class TestFunction(object): result = testdir.runpytest() result.stdout.fnmatch_lines('* 2 passed, 1 skipped in *') + @ignore_parametrized_marks def test_parametrize_skip(self, testdir): testdir.makepyfile(""" import pytest @@ -666,6 +672,7 @@ class TestFunction(object): result = testdir.runpytest() result.stdout.fnmatch_lines('* 2 passed, 1 skipped in *') + @ignore_parametrized_marks def test_parametrize_skipif_no_skip(self, testdir): testdir.makepyfile(""" import pytest @@ -679,6 +686,7 @@ class TestFunction(object): result = testdir.runpytest() result.stdout.fnmatch_lines('* 1 failed, 2 passed in *') + @ignore_parametrized_marks def test_parametrize_xfail(self, testdir): testdir.makepyfile(""" import pytest @@ -692,6 +700,7 @@ class TestFunction(object): result = testdir.runpytest() result.stdout.fnmatch_lines('* 2 passed, 1 xfailed in *') + @ignore_parametrized_marks def test_parametrize_passed(self, testdir): testdir.makepyfile(""" import pytest @@ -705,6 +714,7 @@ class TestFunction(object): result = testdir.runpytest() result.stdout.fnmatch_lines('* 2 passed, 1 xpassed in *') + @ignore_parametrized_marks def test_parametrize_xfail_passed(self, testdir): testdir.makepyfile(""" import pytest diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 5eee2ffba..a0025a15a 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -1289,8 +1289,9 @@ class TestMetafuncFunctionalAuto(object): assert output.count('preparing foo-3') == 1 +@pytest.mark.filterwarnings('ignore:Applying marks directly to parameters') +@pytest.mark.issue308 class TestMarkersWithParametrization(object): - pytestmark = pytest.mark.issue308 def test_simple_mark(self, testdir): s = """ diff --git a/testing/python/setup_only.py b/testing/python/setup_only.py index c780b197e..18af56477 100644 --- a/testing/python/setup_only.py +++ b/testing/python/setup_only.py @@ -187,7 +187,7 @@ def test_dynamic_fixture_request(testdir): pass @pytest.fixture() def dependent_fixture(request): - request.getfuncargvalue('dynamically_requested_fixture') + request.getfixturevalue('dynamically_requested_fixture') def test_dyn(dependent_fixture): pass ''') diff --git a/testing/test_mark.py b/testing/test_mark.py index cc46702a5..4c495fde0 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -787,6 +787,7 @@ class TestKeywordSelection(object): marks=[pytest.mark.xfail, pytest.mark.skip], id=None)), ]) +@pytest.mark.filterwarnings('ignore') def test_parameterset_extractfrom(argval, expected): extracted = ParameterSet.extract_from(argval) assert extracted == expected diff --git a/testing/test_recwarn.py b/testing/test_recwarn.py index 5611d1a44..6895b1140 100644 --- a/testing/test_recwarn.py +++ b/testing/test_recwarn.py @@ -2,7 +2,6 @@ from __future__ import absolute_import, division, print_function import warnings import re import py -import sys import pytest from _pytest.recwarn import WarningsRecorder @@ -125,6 +124,7 @@ class TestDeprecatedCall(object): @pytest.mark.parametrize('warning_type', [PendingDeprecationWarning, DeprecationWarning]) @pytest.mark.parametrize('mode', ['context_manager', 'call']) @pytest.mark.parametrize('call_f_first', [True, False]) + @pytest.mark.filterwarnings('ignore') def test_deprecated_call_modes(self, warning_type, mode, call_f_first): """Ensure deprecated_call() captures a deprecation warning as expected inside its block/function. @@ -170,32 +170,6 @@ class TestDeprecatedCall(object): with pytest.deprecated_call(): f() - def test_deprecated_function_already_called(self, testdir): - """deprecated_call should be able to catch a call to a deprecated - function even if that function has already been called in the same - module. See #1190. - """ - testdir.makepyfile(""" - import warnings - import pytest - - def deprecated_function(): - warnings.warn("deprecated", DeprecationWarning) - - def test_one(): - deprecated_function() - - def test_two(): - pytest.deprecated_call(deprecated_function) - """) - result = testdir.runpytest() - # for some reason in py26 catch_warnings manages to catch the deprecation warning - # from deprecated_function(), even with default filters active (which ignore deprecation - # warnings) - py26 = sys.version_info[:2] == (2, 6) - expected = '*=== 2 passed in *===' if not py26 else '*=== 2 passed, 1 warnings in *===' - result.stdout.fnmatch_lines(expected) - class TestWarns(object): def test_strings(self): diff --git a/testing/test_unittest.py b/testing/test_unittest.py index b869ec4ad..c59e472e0 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -9,9 +9,9 @@ def test_simple_unittest(testdir): import unittest class MyTestCase(unittest.TestCase): def testpassing(self): - self.assertEquals('foo', 'foo') + self.assertEqual('foo', 'foo') def test_failing(self): - self.assertEquals('foo', 'bar') + self.assertEqual('foo', 'bar') """) reprec = testdir.inline_run(testpath) assert reprec.matchreport("testpassing").passed @@ -23,10 +23,10 @@ def test_runTest_method(testdir): import unittest class MyTestCaseWithRunTest(unittest.TestCase): def runTest(self): - self.assertEquals('foo', 'foo') + self.assertEqual('foo', 'foo') class MyTestCaseWithoutRunTest(unittest.TestCase): def runTest(self): - self.assertEquals('foo', 'foo') + self.assertEqual('foo', 'foo') def test_something(self): pass """) @@ -59,7 +59,7 @@ def test_setup(testdir): def setup_method(self, method): self.foo2 = 1 def test_both(self): - self.assertEquals(1, self.foo) + self.assertEqual(1, self.foo) assert self.foo2 == 1 def teardown_method(self, method): assert 0, "42" @@ -136,7 +136,7 @@ def test_teardown(testdir): self.l.append(None) class Second(unittest.TestCase): def test_check(self): - self.assertEquals(MyTestCase.l, [None]) + self.assertEqual(MyTestCase.l, [None]) """) reprec = testdir.inline_run(testpath) passed, skipped, failed = reprec.countoutcomes() @@ -598,7 +598,7 @@ def test_unittest_not_shown_in_traceback(testdir): class t(unittest.TestCase): def test_hello(self): x = 3 - self.assertEquals(x, 4) + self.assertEqual(x, 4) """) res = testdir.runpytest() assert "failUnlessEqual" not in res.stdout.str() diff --git a/testing/test_warnings.py b/testing/test_warnings.py index 58e6bcf74..1328cc3f2 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -33,6 +33,7 @@ def pyfile_with_warnings(testdir, request): }) +@pytest.mark.filterwarnings('always') def test_normal_flow(testdir, pyfile_with_warnings): """ Check that the warnings section is displayed, containing test node ids followed by @@ -54,6 +55,7 @@ def test_normal_flow(testdir, pyfile_with_warnings): assert result.stdout.str().count('test_normal_flow.py::test_func') == 1 +@pytest.mark.filterwarnings('always') def test_setup_teardown_warnings(testdir, pyfile_with_warnings): testdir.makepyfile(''' import warnings @@ -115,6 +117,7 @@ def test_ignore(testdir, pyfile_with_warnings, method): @pytest.mark.skipif(sys.version_info < (3, 0), reason='warnings message is unicode is ok in python3') +@pytest.mark.filterwarnings('always') def test_unicode(testdir, pyfile_with_warnings): testdir.makepyfile(''' # -*- coding: utf8 -*- @@ -152,6 +155,7 @@ def test_py2_unicode(testdir, pyfile_with_warnings): warnings.warn(u"测试") yield + @pytest.mark.filterwarnings('always') def test_func(fix): pass ''') diff --git a/tox.ini b/tox.ini index da5ae9fec..3dce17b34 100644 --- a/tox.ini +++ b/tox.ini @@ -193,6 +193,7 @@ python_classes = Test Acceptance python_functions = test norecursedirs = .tox ja .hg cx_freeze_source filterwarnings = + error # produced by path.local ignore:bad escape.*:DeprecationWarning:re # produced by path.readlines From bda07d8b277ed4d3027a58e932a0444789670ed3 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 22 Jul 2017 15:04:05 -0300 Subject: [PATCH 51/73] Ignore socket warnings on windows for trial tests --- testing/test_unittest.py | 105 +++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/testing/test_unittest.py b/testing/test_unittest.py index c59e472e0..84f432a54 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -351,61 +351,12 @@ def test_module_level_pytestmark(testdir): reprec.assertoutcome(skipped=1) -def test_trial_testcase_skip_property(testdir): - pytest.importorskip('twisted.trial.unittest') - testpath = testdir.makepyfile(""" - from twisted.trial import unittest - class MyTestCase(unittest.TestCase): - skip = 'dont run' - def test_func(self): - pass - """) - reprec = testdir.inline_run(testpath, "-s") - reprec.assertoutcome(skipped=1) - - -def test_trial_testfunction_skip_property(testdir): - pytest.importorskip('twisted.trial.unittest') - testpath = testdir.makepyfile(""" - from twisted.trial import unittest - class MyTestCase(unittest.TestCase): - def test_func(self): - pass - test_func.skip = 'dont run' - """) - reprec = testdir.inline_run(testpath, "-s") - reprec.assertoutcome(skipped=1) - - -def test_trial_testcase_todo_property(testdir): - pytest.importorskip('twisted.trial.unittest') - testpath = testdir.makepyfile(""" - from twisted.trial import unittest - class MyTestCase(unittest.TestCase): - todo = 'dont run' - def test_func(self): - assert 0 - """) - reprec = testdir.inline_run(testpath, "-s") - reprec.assertoutcome(skipped=1) - - -def test_trial_testfunction_todo_property(testdir): - pytest.importorskip('twisted.trial.unittest') - testpath = testdir.makepyfile(""" - from twisted.trial import unittest - class MyTestCase(unittest.TestCase): - def test_func(self): - assert 0 - test_func.todo = 'dont run' - """) - reprec = testdir.inline_run(testpath, "-s") - reprec.assertoutcome(skipped=1) - - class TestTrialUnittest(object): def setup_class(cls): cls.ut = pytest.importorskip("twisted.trial.unittest") + # on windows trial uses a socket for a reactor and apparently doesn't close it properly + # https://twistedmatrix.com/trac/ticket/9227 + cls.ignore_unclosed_socket_warning = ('-W', 'always') def test_trial_testcase_runtest_not_collected(self, testdir): testdir.makepyfile(""" @@ -415,7 +366,7 @@ class TestTrialUnittest(object): def test_hello(self): pass """) - reprec = testdir.inline_run() + reprec = testdir.inline_run(*self.ignore_unclosed_socket_warning) reprec.assertoutcome(passed=1) testdir.makepyfile(""" from twisted.trial.unittest import TestCase @@ -424,7 +375,7 @@ class TestTrialUnittest(object): def runTest(self): pass """) - reprec = testdir.inline_run() + reprec = testdir.inline_run(*self.ignore_unclosed_socket_warning) reprec.assertoutcome(passed=1) def test_trial_exceptions_with_skips(self, testdir): @@ -462,7 +413,7 @@ class TestTrialUnittest(object): """) from _pytest.compat import _is_unittest_unexpected_success_a_failure should_fail = _is_unittest_unexpected_success_a_failure() - result = testdir.runpytest("-rxs") + result = testdir.runpytest("-rxs", *self.ignore_unclosed_socket_warning) result.stdout.fnmatch_lines_random([ "*XFAIL*test_trial_todo*", "*trialselfskip*", @@ -537,6 +488,50 @@ class TestTrialUnittest(object): child.expect("hellopdb") child.sendeof() + def test_trial_testcase_skip_property(self, testdir): + testpath = testdir.makepyfile(""" + from twisted.trial import unittest + class MyTestCase(unittest.TestCase): + skip = 'dont run' + def test_func(self): + pass + """) + reprec = testdir.inline_run(testpath, "-s") + reprec.assertoutcome(skipped=1) + + def test_trial_testfunction_skip_property(self, testdir): + testpath = testdir.makepyfile(""" + from twisted.trial import unittest + class MyTestCase(unittest.TestCase): + def test_func(self): + pass + test_func.skip = 'dont run' + """) + reprec = testdir.inline_run(testpath, "-s") + reprec.assertoutcome(skipped=1) + + def test_trial_testcase_todo_property(self, testdir): + testpath = testdir.makepyfile(""" + from twisted.trial import unittest + class MyTestCase(unittest.TestCase): + todo = 'dont run' + def test_func(self): + assert 0 + """) + reprec = testdir.inline_run(testpath, "-s") + reprec.assertoutcome(skipped=1) + + def test_trial_testfunction_todo_property(self, testdir): + testpath = testdir.makepyfile(""" + from twisted.trial import unittest + class MyTestCase(unittest.TestCase): + def test_func(self): + assert 0 + test_func.todo = 'dont run' + """) + reprec = testdir.inline_run(testpath, "-s", *self.ignore_unclosed_socket_warning) + reprec.assertoutcome(skipped=1) + def test_djangolike_testcase(testdir): # contributed from Morten Breekevold From d5bb2004f9d1d688a8566547b2e92c409a4e1dbf Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 22 Jul 2017 22:11:51 -0300 Subject: [PATCH 52/73] Fix travis build after change from "precise" to "trusty" Travis recently has changed its dist from "precise" to "trusty", so some Python versions are no longer installed by default --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d3dce9141..6d8d58328 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,9 @@ env: - TOXENV=coveralls # note: please use "tox --listenvs" to populate the build matrix below - TOXENV=linting - - TOXENV=py26 - TOXENV=py27 - - TOXENV=py33 - TOXENV=py34 - TOXENV=py35 - - TOXENV=pypy - TOXENV=py27-pexpect - TOXENV=py27-xdist - TOXENV=py27-trial @@ -32,6 +29,12 @@ env: matrix: include: + - env: TOXENV=py26 + python: '2.6' + - env: TOXENV=py33 + python: '3.3' + - env: TOXENV=pypy + python: 'pypy-5.4' - env: TOXENV=py36 python: '3.6' - env: TOXENV=py37 From d3ab1b9df4b8854b75c5fe88157aae2b5c572357 Mon Sep 17 00:00:00 2001 From: Maik Figura Date: Sat, 15 Jul 2017 14:58:03 +0200 Subject: [PATCH 53/73] Add user documentation The new doc section explains why we raise a `NotImplementedError`. --- _pytest/python_api.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index 176aff590..a96a4f63b 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -1,6 +1,5 @@ import math import sys - import py from _pytest.compat import isclass, izip @@ -32,6 +31,12 @@ class ApproxBase(object): __hash__ = None + def __gt__(self, actual): + raise NotImplementedError + + def __lt__(self, actual): + raise NotImplementedError + def __ne__(self, actual): return not (actual == self) @@ -60,6 +65,12 @@ class ApproxNumpy(ApproxBase): return "approx({0!r})".format(list( self._approx_scalar(x) for x in self.expected)) + def __gt__(self, actual): + raise NotImplementedError + + def __lt__(self, actual): + raise NotImplementedError + def __eq__(self, actual): import numpy as np @@ -358,6 +369,22 @@ def approx(expected, rel=None, abs=None, nan_ok=False): is asymmetric and you can think of ``b`` as the reference value. In the special case that you explicitly specify an absolute tolerance but not a relative tolerance, only the absolute tolerance is considered. + + .. warning:: + + In order to avoid inconsistent behavior, a ``NotImplementedError`` is + raised for ``__lt__`` and ``__gt__`` comparisons. The example below + illustrates the problem:: + + assert approx(0.1) > 0.1 + 1e-10 # calls approx(0.1).__gt__(0.1 + 1e-10) + assert 0.1 + 1e-10 > approx(0.1) # calls approx(0.1).__lt__(0.1 + 1e-10) + + In the second example one expects ``approx(0.1).__le__(0.1 + 1e-10)`` + to be called. But instead, ``approx(0.1).__lt__(0.1 + 1e-10)`` is used to + comparison. This is because the call hierarchy of rich comparisons + follows a fixed behavior. `More information...`__ + + __ https://docs.python.org/3/reference/datamodel.html#object.__ge__ """ from collections import Mapping, Sequence From f0936d42fbb13eb3db4ee4bdf35e981bb7da2a4c Mon Sep 17 00:00:00 2001 From: Maik Figura Date: Sat, 15 Jul 2017 15:03:38 +0200 Subject: [PATCH 54/73] Fix linter errors --- _pytest/python_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index a96a4f63b..fe36372d1 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -372,8 +372,8 @@ def approx(expected, rel=None, abs=None, nan_ok=False): .. warning:: - In order to avoid inconsistent behavior, a ``NotImplementedError`` is - raised for ``__lt__`` and ``__gt__`` comparisons. The example below + In order to avoid inconsistent behavior, a ``NotImplementedError`` is + raised for ``__lt__`` and ``__gt__`` comparisons. The example below illustrates the problem:: assert approx(0.1) > 0.1 + 1e-10 # calls approx(0.1).__gt__(0.1 + 1e-10) From 1851f36beba36a08eb6946a6a187bfc8a0ebcbd4 Mon Sep 17 00:00:00 2001 From: Maik Figura Date: Sat, 15 Jul 2017 15:10:55 +0200 Subject: [PATCH 55/73] Add PR requirements changelog and authors --- AUTHORS | 1 + changelog/2003.feature | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog/2003.feature diff --git a/AUTHORS b/AUTHORS index cd9678b3f..cbcb16639 100644 --- a/AUTHORS +++ b/AUTHORS @@ -99,6 +99,7 @@ Lukas Bednar Luke Murphy Maciek Fijalkowski Maho +Maik Figura Mandeep Bhutani Manuel Krebber Marc Schlaich diff --git a/changelog/2003.feature b/changelog/2003.feature new file mode 100644 index 000000000..b4bf12431 --- /dev/null +++ b/changelog/2003.feature @@ -0,0 +1 @@ +Remove support for `<` and `>` support in approx. From 57a232fc5a275f7a0ae813713afba436c658262a Mon Sep 17 00:00:00 2001 From: Maik Figura Date: Sat, 15 Jul 2017 15:18:03 +0200 Subject: [PATCH 56/73] Remove out of scope change --- _pytest/python_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index fe36372d1..915c4fde2 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -1,5 +1,6 @@ import math import sys + import py from _pytest.compat import isclass, izip From 80f4699572c0ccd304f30a9451d1728066b0db27 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 25 Jul 2017 19:15:21 -0300 Subject: [PATCH 57/73] approx raises TypeError in Python 2 for comparison operators other than != and == --- _pytest/python_api.py | 39 ++++++++++++++++++++++++--------------- changelog/2003.feature | 1 - changelog/2003.removal | 2 ++ testing/python/approx.py | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 16 deletions(-) delete mode 100644 changelog/2003.feature create mode 100644 changelog/2003.removal diff --git a/_pytest/python_api.py b/_pytest/python_api.py index 915c4fde2..c0ff24a8f 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -7,6 +7,19 @@ from _pytest.compat import isclass, izip from _pytest.runner import fail import _pytest._code + +def _cmp_raises_type_error(self, other): + """__cmp__ implementation which raises TypeError. Used + by Approx base classes to implement only == and != and raise a + TypeError for other comparisons. + + Needed in Python 2 only, Python 3 all it takes is not implementing the + other operators at all. + """ + __tracebackhide__ = True + raise TypeError('Comparison operators other than == and != not supported by approx objects') + + # builtin pytest.approx helper @@ -32,15 +45,12 @@ class ApproxBase(object): __hash__ = None - def __gt__(self, actual): - raise NotImplementedError - - def __lt__(self, actual): - raise NotImplementedError - def __ne__(self, actual): return not (actual == self) + if sys.version_info[0] == 2: + __cmp__ = _cmp_raises_type_error + def _approx_scalar(self, x): return ApproxScalar(x, rel=self.rel, abs=self.abs, nan_ok=self.nan_ok) @@ -66,11 +76,8 @@ class ApproxNumpy(ApproxBase): return "approx({0!r})".format(list( self._approx_scalar(x) for x in self.expected)) - def __gt__(self, actual): - raise NotImplementedError - - def __lt__(self, actual): - raise NotImplementedError + if sys.version_info[0] == 2: + __cmp__ = _cmp_raises_type_error def __eq__(self, actual): import numpy as np @@ -370,12 +377,14 @@ def approx(expected, rel=None, abs=None, nan_ok=False): is asymmetric and you can think of ``b`` as the reference value. In the special case that you explicitly specify an absolute tolerance but not a relative tolerance, only the absolute tolerance is considered. - + .. warning:: - In order to avoid inconsistent behavior, a ``NotImplementedError`` is - raised for ``__lt__`` and ``__gt__`` comparisons. The example below - illustrates the problem:: + .. versionchanged:: 3.2 + + In order to avoid inconsistent behavior, ``TypeError`` is + raised for ``>``, ``>=``, ``<`` and ``<=`` comparisons. + The example below illustrates the problem:: assert approx(0.1) > 0.1 + 1e-10 # calls approx(0.1).__gt__(0.1 + 1e-10) assert 0.1 + 1e-10 > approx(0.1) # calls approx(0.1).__lt__(0.1 + 1e-10) diff --git a/changelog/2003.feature b/changelog/2003.feature deleted file mode 100644 index b4bf12431..000000000 --- a/changelog/2003.feature +++ /dev/null @@ -1 +0,0 @@ -Remove support for `<` and `>` support in approx. diff --git a/changelog/2003.removal b/changelog/2003.removal new file mode 100644 index 000000000..d3269bf4e --- /dev/null +++ b/changelog/2003.removal @@ -0,0 +1,2 @@ +``pytest.approx`` no longer supports ``>``, ``>=``, ``<`` and ``<=`` operators to avoid surprising/inconsistent +behavior. See `the docs `_ for more information. diff --git a/testing/python/approx.py b/testing/python/approx.py index 876226e06..d591b8ba5 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -1,4 +1,5 @@ # encoding: utf-8 +import operator import sys import pytest import doctest @@ -382,3 +383,16 @@ class TestApprox(object): '*At index 0 diff: 3 != 4 * {0}'.format(expected), '=* 1 failed in *=', ]) + + @pytest.mark.parametrize('op', [ + pytest.param(operator.le, id='<='), + pytest.param(operator.lt, id='<'), + pytest.param(operator.ge, id='>='), + pytest.param(operator.gt, id='>'), + ]) + def test_comparison_operator_type_error(self, op): + """ + pytest.approx should raise TypeError for operators other than == and != (#2003). + """ + with pytest.raises(TypeError): + op(1, approx(1, rel=1e-6, abs=1e-12)) From b39f957b88b6f44252c0ad2eb289b92fd7937f8e Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 26 Jul 2017 10:08:54 +0100 Subject: [PATCH 58/73] Add test of issue #920 --- testing/python/fixture.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/testing/python/fixture.py b/testing/python/fixture.py index 8dd713416..1801c91ff 100644 --- a/testing/python/fixture.py +++ b/testing/python/fixture.py @@ -2547,6 +2547,40 @@ class TestFixtureMarker(object): '*test_foo*alpha*', '*test_foo*beta*']) + @pytest.mark.issue920 + @pytest.mark.xfail(reason="Fixture reordering not deterministic with hash randomisation") + def test_deterministic_fixture_collection(self, testdir, monkeypatch): + testdir.makepyfile(""" + import pytest + + @pytest.fixture(scope="module", + params=["A", + "B", + "C"]) + def A(request): + return request.param + + @pytest.fixture(scope="module", + params=["DDDDDDDDD", "EEEEEEEEEEEE", "FFFFFFFFFFF", "banansda"]) + def B(request, A): + return request.param + + def test_foo(B): + # Something funky is going on here. + # Despite specified seeds, on what is collected, + # sometimes we get unexpected passes. hashing B seems + # to help? + assert hash(B) or True + """) + monkeypatch.setenv("PYTHONHASHSEED", "1") + out1 = testdir.runpytest_subprocess("-v") + monkeypatch.setenv("PYTHONHASHSEED", "2") + out2 = testdir.runpytest_subprocess("-v") + out1 = [line for line in out1.outlines if line.startswith("test_deterministic_fixture_collection.py::test_foo")] + out2 = [line for line in out2.outlines if line.startswith("test_deterministic_fixture_collection.py::test_foo")] + assert len(out1) == 12 + assert out1 == out2 + class TestRequestScopeAccess(object): pytestmark = pytest.mark.parametrize(("scope", "ok", "error"), [ From a546a612bde02292488b6f9b3185a950d61fbe94 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Tue, 25 Jul 2017 19:54:27 +0100 Subject: [PATCH 59/73] Fix nondeterminism in fixture collection order fixtures.reorder_items is non-deterministic because it reorders based on iteration over an (unordered) set. Change the code to use an OrderedDict instead so that we get deterministic behaviour, fixes #920. --- AUTHORS | 1 + _pytest/fixtures.py | 21 +++++++++++++-------- changelog/920.bugfix | 1 + testing/python/fixture.py | 1 - tox.ini | 1 + 5 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 changelog/920.bugfix diff --git a/AUTHORS b/AUTHORS index cd9678b3f..b6b6b46fa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -91,6 +91,7 @@ Kale Kundert Katarzyna Jachim Kevin Cox Kodi B. Arfer +Lawrence Mitchell Lee Kamentsky Lev Maximov Llandy Riveron Del Risco diff --git a/_pytest/fixtures.py b/_pytest/fixtures.py index 5505fb4e1..d475dfd1b 100644 --- a/_pytest/fixtures.py +++ b/_pytest/fixtures.py @@ -19,6 +19,11 @@ from _pytest.compat import ( from _pytest.runner import fail from _pytest.compat import FuncargnamesCompatAttr +if sys.version_info[:2] == (2, 6): + from ordereddict import OrderedDict +else: + from collections import OrderedDict + def pytest_sessionstart(session): import _pytest.python @@ -136,10 +141,10 @@ def get_parametrized_fixture_keys(item, scopenum): except AttributeError: pass else: - # cs.indictes.items() is random order of argnames but - # then again different functions (items) can change order of - # arguments so it doesn't matter much probably - for argname, param_index in cs.indices.items(): + # cs.indices.items() is random order of argnames. Need to + # sort this so that different calls to + # get_parametrized_fixture_keys will be deterministic. + for argname, param_index in sorted(cs.indices.items()): if cs._arg2scopenum[argname] != scopenum: continue if scopenum == 0: # session @@ -161,7 +166,7 @@ def reorder_items(items): for scopenum in range(0, scopenum_function): argkeys_cache[scopenum] = d = {} for item in items: - keys = set(get_parametrized_fixture_keys(item, scopenum)) + keys = OrderedDict.fromkeys(get_parametrized_fixture_keys(item, scopenum)) if keys: d[item] = keys return reorder_items_atscope(items, set(), argkeys_cache, 0) @@ -196,9 +201,9 @@ def slice_items(items, ignore, scoped_argkeys_cache): for i, item in enumerate(it): argkeys = scoped_argkeys_cache.get(item) if argkeys is not None: - argkeys = argkeys.difference(ignore) - if argkeys: # found a slicing key - slicing_argkey = argkeys.pop() + newargkeys = OrderedDict.fromkeys(k for k in argkeys if k not in ignore) + if newargkeys: # found a slicing key + slicing_argkey, _ = newargkeys.popitem() items_before = items[:i] items_same = [item] items_other = [] diff --git a/changelog/920.bugfix b/changelog/920.bugfix new file mode 100644 index 000000000..efe646a6e --- /dev/null +++ b/changelog/920.bugfix @@ -0,0 +1 @@ +Fix non-determinism in order of fixture collection. diff --git a/testing/python/fixture.py b/testing/python/fixture.py index 1801c91ff..f8aef802f 100644 --- a/testing/python/fixture.py +++ b/testing/python/fixture.py @@ -2548,7 +2548,6 @@ class TestFixtureMarker(object): '*test_foo*beta*']) @pytest.mark.issue920 - @pytest.mark.xfail(reason="Fixture reordering not deterministic with hash randomisation") def test_deterministic_fixture_collection(self, testdir, monkeypatch): testdir.makepyfile(""" import pytest diff --git a/tox.ini b/tox.ini index 3dce17b34..1307cff14 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,7 @@ deps = hypothesis<3.0 nose mock<1.1 + ordereddict [testenv:py27-subprocess] changedir = . From f8bd693f8348dbf5b0536b5529cc5a4884bf06b7 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 26 Jul 2017 10:58:38 +0100 Subject: [PATCH 60/73] Add ordereddict to install_requires for py26 --- setup.py | 3 ++- tox.ini | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 751868c04..55607912b 100644 --- a/setup.py +++ b/setup.py @@ -46,11 +46,12 @@ def main(): install_requires = ['py>=1.4.33', 'setuptools'] # pluggy is vendored in _pytest.vendored_packages extras_require = {} if has_environment_marker_support(): - extras_require[':python_version=="2.6"'] = ['argparse'] + extras_require[':python_version=="2.6"'] = ['argparse', 'ordereddict'] extras_require[':sys_platform=="win32"'] = ['colorama'] else: if sys.version_info < (2, 7): install_requires.append('argparse') + install_requires.append('ordereddict') if sys.platform == 'win32': install_requires.append('colorama') diff --git a/tox.ini b/tox.ini index 1307cff14..3dce17b34 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,6 @@ deps = hypothesis<3.0 nose mock<1.1 - ordereddict [testenv:py27-subprocess] changedir = . From f047e078e2d1c8aba19015e151c1e78c5cbc1cff Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 26 Jul 2017 13:56:17 +0100 Subject: [PATCH 61/73] Mention new (py26) ordereddict dependency in changelog and docs --- changelog/920.bugfix | 2 +- doc/en/getting-started.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog/920.bugfix b/changelog/920.bugfix index efe646a6e..d2dd2be1b 100644 --- a/changelog/920.bugfix +++ b/changelog/920.bugfix @@ -1 +1 @@ -Fix non-determinism in order of fixture collection. +Fix non-determinism in order of fixture collection. Adds new dependency (ordereddict) for Python 2.6. diff --git a/doc/en/getting-started.rst b/doc/en/getting-started.rst index fb863e4e0..1571e4f6b 100644 --- a/doc/en/getting-started.rst +++ b/doc/en/getting-started.rst @@ -9,7 +9,8 @@ Installation and Getting Started **dependencies**: `py `_, `colorama (Windows) `_, -`argparse (py26) `_. +`argparse (py26) `_, +`ordereddict (py26) `_. **documentation as PDF**: `download latest `_ From 17c544e793f95a7e0423114a435ff51c17e8ad5e Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Jul 2017 10:34:49 -0300 Subject: [PATCH 62/73] Introduce new pytest_report_collectionfinish hook Fix #2622 --- _pytest/hookspec.py | 16 +++++++++++++++- _pytest/terminal.py | 10 ++++++---- changelog/2622.feature | 2 ++ doc/en/writing_plugins.rst | 1 + testing/test_terminal.py | 17 +++++++++++++++++ 5 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 changelog/2622.feature diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py index 43667d701..f7b53e892 100644 --- a/_pytest/hookspec.py +++ b/_pytest/hookspec.py @@ -337,7 +337,10 @@ def pytest_assertrepr_compare(config, op, left, right): def pytest_report_header(config, startdir): - """ return a string to be displayed as header info for terminal reporting. + """ return a string or list of strings to be displayed as header info for terminal reporting. + + :param config: the pytest config object. + :param startdir: py.path object with the starting dir .. note:: @@ -347,6 +350,17 @@ def pytest_report_header(config, startdir): """ +def pytest_report_collectionfinish(config, startdir, items): + """ return a string or list of strings to be displayed after collection has finished successfully. + + This strings will be displayed after the standard "collected X items" message. + + :param config: the pytest config object. + :param startdir: py.path object with the starting dir + :param items: list of pytest items that are going to be executed; this list should not be modified. + """ + + @hookspec(firstresult=True) def pytest_report_teststatus(report): """ return result-category, shortletter and verbose word for reporting. diff --git a/_pytest/terminal.py b/_pytest/terminal.py index 10f37c6a1..7dd10924a 100644 --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -323,6 +323,9 @@ class TerminalReporter: self.write_line(msg) lines = self.config.hook.pytest_report_header( config=self.config, startdir=self.startdir) + self._write_report_lines_from_hooks(lines) + + def _write_report_lines_from_hooks(self, lines): lines.reverse() for line in flatten(lines): self.write_line(line) @@ -349,10 +352,9 @@ class TerminalReporter: rep.toterminal(self._tw) return 1 return 0 - if not self.showheader: - return - # for i, testarg in enumerate(self.config.args): - # self.write_line("test path %d: %s" %(i+1, testarg)) + lines = self.config.hook.pytest_report_collectionfinish( + config=self.config, startdir=self.startdir, items=session.items) + self._write_report_lines_from_hooks(lines) def _printcollecteditems(self, items): # to print out items and their parent collectors diff --git a/changelog/2622.feature b/changelog/2622.feature new file mode 100644 index 000000000..298892200 --- /dev/null +++ b/changelog/2622.feature @@ -0,0 +1,2 @@ +New ``pytest_report_collectionfinish`` hook which allows plugins to add messages to the terminal reporting after +collection has been finished successfully. diff --git a/doc/en/writing_plugins.rst b/doc/en/writing_plugins.rst index 26e1a8a69..3eb7d784e 100644 --- a/doc/en/writing_plugins.rst +++ b/doc/en/writing_plugins.rst @@ -644,6 +644,7 @@ Session related reporting hooks: .. autofunction:: pytest_collectreport .. autofunction:: pytest_deselected .. autofunction:: pytest_report_header +.. autofunction:: pytest_report_collectionfinish .. autofunction:: pytest_report_teststatus .. autofunction:: pytest_terminal_summary .. autofunction:: pytest_fixture_setup diff --git a/testing/test_terminal.py b/testing/test_terminal.py index bac3ab8b1..6b20c3a48 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -544,6 +544,23 @@ class TestTerminalFunctional(object): assert "===" not in s assert "passed" not in s + def test_report_collectionfinish_hook(self, testdir): + testdir.makeconftest(""" + def pytest_report_collectionfinish(config, startdir, items): + return ['hello from hook: {0} items'.format(len(items))] + """) + testdir.makepyfile(""" + import pytest + @pytest.mark.parametrize('i', range(3)) + def test(i): + pass + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + "collected 3 items", + "hello from hook: 3 items", + ]) + def test_fail_extra_reporting(testdir): testdir.makepyfile("def test_this(): assert 0") From 62810f61b22341bc2b0a09da30252dfb0f57e027 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 26 Jul 2017 21:06:08 -0300 Subject: [PATCH 63/73] Make cache plugin always remember failed tests --- _pytest/cacheprovider.py | 26 ++++++++----------- changelog/2621.feature | 2 ++ testing/test_cache.py | 55 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 changelog/2621.feature diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py index 5f2c6b062..14fd86f6b 100755 --- a/_pytest/cacheprovider.py +++ b/_pytest/cacheprovider.py @@ -105,27 +105,22 @@ class LFPlugin: self.config = config active_keys = 'lf', 'failedfirst' self.active = any(config.getvalue(key) for key in active_keys) - if self.active: - self.lastfailed = config.cache.get("cache/lastfailed", {}) - else: - self.lastfailed = {} + self.lastfailed = config.cache.get("cache/lastfailed", {}) def pytest_report_header(self): if self.active: if not self.lastfailed: mode = "run all (no recorded failures)" else: - mode = "rerun last %d failures%s" % ( - len(self.lastfailed), + mode = "rerun previous failures%s" % ( " first" if self.config.getvalue("failedfirst") else "") return "run-last-failure: %s" % mode def pytest_runtest_logreport(self, report): - if report.failed and "xfail" not in report.keywords: + if report.passed and report.when == 'call': + self.lastfailed.pop(report.nodeid, None) + elif report.failed: self.lastfailed[report.nodeid] = True - elif not report.failed: - if report.when == "call": - self.lastfailed.pop(report.nodeid, None) def pytest_collectreport(self, report): passed = report.outcome in ('passed', 'skipped') @@ -147,11 +142,11 @@ class LFPlugin: previously_failed.append(item) else: previously_passed.append(item) - if not previously_failed and previously_passed: + if not previously_failed: # running a subset of all tests with recorded failures outside # of the set of tests currently executing - pass - elif self.config.getvalue("lf"): + return + if self.config.getvalue("lf"): items[:] = previously_failed config.hook.pytest_deselected(items=previously_passed) else: @@ -161,8 +156,9 @@ class LFPlugin: config = self.config if config.getvalue("cacheshow") or hasattr(config, "slaveinput"): return - prev_failed = config.cache.get("cache/lastfailed", None) is not None - if (session.testscollected and prev_failed) or self.lastfailed: + + saved_lastfailed = config.cache.get("cache/lastfailed", {}) + if saved_lastfailed != self.lastfailed: config.cache.set("cache/lastfailed", self.lastfailed) diff --git a/changelog/2621.feature b/changelog/2621.feature new file mode 100644 index 000000000..19ca96355 --- /dev/null +++ b/changelog/2621.feature @@ -0,0 +1,2 @@ +``--last-failed`` now remembers forever when a test has failed and only forgets it if it passes again. This makes it +easy to fix a test suite by selectively running files and fixing tests incrementally. diff --git a/testing/test_cache.py b/testing/test_cache.py index 36059ec29..04ce6671c 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -437,3 +437,58 @@ class TestLastFailed(object): testdir.makepyfile(test_errored='def test_error():\n assert False') testdir.runpytest('-q', '--lf') assert os.path.exists('.cache') + + def get_cached_last_failed(self, testdir): + config = testdir.parseconfigure() + return sorted(config.cache.get("cache/lastfailed", {})) + + def test_cache_cumulative(self, testdir): + """ + Test workflow where user fixes errors gradually file by file using --lf. + """ + # 1. initial run + test_bar = testdir.makepyfile(test_bar=""" + def test_bar_1(): + pass + def test_bar_2(): + assert 0 + """) + test_foo = testdir.makepyfile(test_foo=""" + def test_foo_3(): + pass + def test_foo_4(): + assert 0 + """) + testdir.runpytest() + assert self.get_cached_last_failed(testdir) == ['test_bar.py::test_bar_2', 'test_foo.py::test_foo_4'] + + # 2. fix test_bar_2, run only test_bar.py + testdir.makepyfile(test_bar=""" + def test_bar_1(): + pass + def test_bar_2(): + pass + """) + result = testdir.runpytest(test_bar) + result.stdout.fnmatch_lines('*2 passed*') + # ensure cache does not forget that test_foo_4 failed once before + assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4'] + + result = testdir.runpytest('--last-failed') + result.stdout.fnmatch_lines('*1 failed, 3 deselected*') + assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4'] + + # 3. fix test_foo_4, run only test_foo.py + test_foo = testdir.makepyfile(test_foo=""" + def test_foo_3(): + pass + def test_foo_4(): + pass + """) + result = testdir.runpytest(test_foo, '--last-failed') + result.stdout.fnmatch_lines('*1 passed, 1 deselected*') + assert self.get_cached_last_failed(testdir) == [] + + result = testdir.runpytest('--last-failed') + result.stdout.fnmatch_lines('*4 passed*') + assert self.get_cached_last_failed(testdir) == [] From 22212c4d61823e767de2b34da362960e4882113f Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Jul 2017 10:56:04 -0300 Subject: [PATCH 64/73] Add xfail specific tests --- testing/test_cache.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/testing/test_cache.py b/testing/test_cache.py index 04ce6671c..1b2c1499e 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -438,6 +438,28 @@ class TestLastFailed(object): testdir.runpytest('-q', '--lf') assert os.path.exists('.cache') + def test_xfail_not_considered_failure(self, testdir): + testdir.makepyfile(''' + import pytest + @pytest.mark.xfail + def test(): + assert 0 + ''') + result = testdir.runpytest() + result.stdout.fnmatch_lines('*1 xfailed*') + assert self.get_cached_last_failed(testdir) == [] + + def test_xfail_strict_considered_failure(self, testdir): + testdir.makepyfile(''' + import pytest + @pytest.mark.xfail(strict=True) + def test(): + pass + ''') + result = testdir.runpytest() + result.stdout.fnmatch_lines('*1 failed*') + assert self.get_cached_last_failed(testdir) == ['test_xfail_strict_considered_failure.py::test'] + def get_cached_last_failed(self, testdir): config = testdir.parseconfigure() return sorted(config.cache.get("cache/lastfailed", {})) From eb1bd3449ecdb47e6b3502be39aaf699e3b8f09a Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Jul 2017 18:43:04 -0300 Subject: [PATCH 65/73] xfail and skipped tests are removed from the "last-failed" cache This accommodates the case where a failing test is marked as skipped/failed later --- _pytest/cacheprovider.py | 2 +- testing/test_cache.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py index 14fd86f6b..6374453ec 100755 --- a/_pytest/cacheprovider.py +++ b/_pytest/cacheprovider.py @@ -117,7 +117,7 @@ class LFPlugin: return "run-last-failure: %s" % mode def pytest_runtest_logreport(self, report): - if report.passed and report.when == 'call': + if (report.when == 'call' and report.passed) or report.skipped: self.lastfailed.pop(report.nodeid, None) elif report.failed: self.lastfailed[report.nodeid] = True diff --git a/testing/test_cache.py b/testing/test_cache.py index 1b2c1499e..da86a202c 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -460,6 +460,28 @@ class TestLastFailed(object): result.stdout.fnmatch_lines('*1 failed*') assert self.get_cached_last_failed(testdir) == ['test_xfail_strict_considered_failure.py::test'] + @pytest.mark.parametrize('mark', ['mark.xfail', 'mark.skip']) + def test_failed_changed_to_xfail_or_skip(self, testdir, mark): + testdir.makepyfile(''' + import pytest + def test(): + assert 0 + ''') + result = testdir.runpytest() + assert self.get_cached_last_failed(testdir) == ['test_failed_changed_to_xfail_or_skip.py::test'] + assert result.ret == 1 + + testdir.makepyfile(''' + import pytest + @pytest.{mark} + def test(): + assert 0 + '''.format(mark=mark)) + result = testdir.runpytest() + assert result.ret == 0 + assert self.get_cached_last_failed(testdir) == [] + assert result.ret == 0 + def get_cached_last_failed(self, testdir): config = testdir.parseconfigure() return sorted(config.cache.get("cache/lastfailed", {})) From 75e6f7717c12d9fe50a2b0cd0eed86ea0bc4797d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Jul 2017 15:06:35 -0300 Subject: [PATCH 66/73] Use new hook to report accurate tests skipped in --lf and --ff --- _pytest/cacheprovider.py | 13 +++++--- testing/test_cache.py | 67 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py index 6374453ec..c537c1447 100755 --- a/_pytest/cacheprovider.py +++ b/_pytest/cacheprovider.py @@ -106,14 +106,18 @@ class LFPlugin: active_keys = 'lf', 'failedfirst' self.active = any(config.getvalue(key) for key in active_keys) self.lastfailed = config.cache.get("cache/lastfailed", {}) + self._previously_failed_count = None - def pytest_report_header(self): + def pytest_report_collectionfinish(self): if self.active: - if not self.lastfailed: + if not self._previously_failed_count: mode = "run all (no recorded failures)" else: - mode = "rerun previous failures%s" % ( - " first" if self.config.getvalue("failedfirst") else "") + noun = 'failure' if self._previously_failed_count == 1 else 'failures' + suffix = " first" if self.config.getvalue("failedfirst") else "" + mode = "rerun previous {count} {noun}{suffix}".format( + count=self._previously_failed_count, suffix=suffix, noun=noun + ) return "run-last-failure: %s" % mode def pytest_runtest_logreport(self, report): @@ -142,6 +146,7 @@ class LFPlugin: previously_failed.append(item) else: previously_passed.append(item) + self._previously_failed_count = len(previously_failed) if not previously_failed: # running a subset of all tests with recorded failures outside # of the set of tests currently executing diff --git a/testing/test_cache.py b/testing/test_cache.py index da86a202c..a37170cdd 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -340,6 +340,73 @@ class TestLastFailed(object): result = testdir.runpytest() result.stdout.fnmatch_lines('*1 failed in*') + def test_terminal_report_lastfailed(self, testdir): + test_a = testdir.makepyfile(test_a=""" + def test_a1(): + pass + def test_a2(): + pass + """) + test_b = testdir.makepyfile(test_b=""" + def test_b1(): + assert 0 + def test_b2(): + assert 0 + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + 'collected 4 items', + '*2 failed, 2 passed in*', + ]) + + result = testdir.runpytest('--lf') + result.stdout.fnmatch_lines([ + 'collected 4 items', + 'run-last-failure: rerun previous 2 failures', + '*2 failed, 2 deselected in*', + ]) + + result = testdir.runpytest(test_a, '--lf') + result.stdout.fnmatch_lines([ + 'collected 2 items', + 'run-last-failure: run all (no recorded failures)', + '*2 passed in*', + ]) + + result = testdir.runpytest(test_b, '--lf') + result.stdout.fnmatch_lines([ + 'collected 2 items', + 'run-last-failure: rerun previous 2 failures', + '*2 failed in*', + ]) + + result = testdir.runpytest('test_b.py::test_b1', '--lf') + result.stdout.fnmatch_lines([ + 'collected 1 item', + 'run-last-failure: rerun previous 1 failure', + '*1 failed in*', + ]) + + def test_terminal_report_failedfirst(self, testdir): + testdir.makepyfile(test_a=""" + def test_a1(): + assert 0 + def test_a2(): + pass + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + 'collected 2 items', + '*1 failed, 1 passed in*', + ]) + + result = testdir.runpytest('--ff') + result.stdout.fnmatch_lines([ + 'collected 2 items', + 'run-last-failure: rerun previous 1 failure first', + '*1 failed, 1 passed in*', + ]) + def test_lastfailed_collectfailure(self, testdir, monkeypatch): testdir.makepyfile(test_maybe=""" From 5acb64be900e499087bdff8d2e0a37683617b65f Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Jul 2017 15:33:22 -0300 Subject: [PATCH 67/73] Add versionadded tag to pytest_report_collectionfinish hook --- _pytest/hookspec.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py index f7b53e892..1fdc2456e 100644 --- a/_pytest/hookspec.py +++ b/_pytest/hookspec.py @@ -351,7 +351,10 @@ def pytest_report_header(config, startdir): def pytest_report_collectionfinish(config, startdir, items): - """ return a string or list of strings to be displayed after collection has finished successfully. + """ + .. versionadded:: 3.2 + + return a string or list of strings to be displayed after collection has finished successfully. This strings will be displayed after the standard "collected X items" message. From 7a12acb6a1164935ae79030a8220e5be1e999e7d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Jul 2017 16:24:58 -0300 Subject: [PATCH 68/73] Fix linting --- _pytest/hookspec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py index 1fdc2456e..e5c966e58 100644 --- a/_pytest/hookspec.py +++ b/_pytest/hookspec.py @@ -353,7 +353,7 @@ def pytest_report_header(config, startdir): def pytest_report_collectionfinish(config, startdir, items): """ .. versionadded:: 3.2 - + return a string or list of strings to be displayed after collection has finished successfully. This strings will be displayed after the standard "collected X items" message. From 06a49338b2b1be8daed89f779439459a2cb4cbc7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 19 Sep 2015 01:03:05 +0200 Subject: [PATCH 69/73] make Test Outcomes inherit from BaseException instead of exception fixes #580 --- _pytest/fixtures.py | 6 +- _pytest/main.py | 3 +- _pytest/outcomes.py | 141 +++++++++++++++++++++++++++++++++++++++++ _pytest/python.py | 2 +- _pytest/python_api.py | 2 +- _pytest/recwarn.py | 4 +- _pytest/runner.py | 125 +----------------------------------- _pytest/skipping.py | 18 +----- _pytest/unittest.py | 4 +- changelog/580.feature | 1 + doc/en/builtin.rst | 10 +-- pytest.py | 3 +- testing/test_runner.py | 13 +++- 13 files changed, 176 insertions(+), 156 deletions(-) create mode 100644 _pytest/outcomes.py create mode 100644 changelog/580.feature diff --git a/_pytest/fixtures.py b/_pytest/fixtures.py index 5505fb4e1..ab5680201 100644 --- a/_pytest/fixtures.py +++ b/_pytest/fixtures.py @@ -16,7 +16,7 @@ from _pytest.compat import ( getlocation, getfuncargnames, safe_getattr, ) -from _pytest.runner import fail +from _pytest.outcomes import fail, TEST_OUTCOME from _pytest.compat import FuncargnamesCompatAttr @@ -121,7 +121,7 @@ def getfixturemarker(obj): exceptions.""" try: return getattr(obj, "_pytestfixturefunction", None) - except Exception: + except TEST_OUTCOME: # some objects raise errors like request (from flask import request) # we don't expect them to be fixture functions return None @@ -811,7 +811,7 @@ def pytest_fixture_setup(fixturedef, request): my_cache_key = request.param_index try: result = call_fixture_func(fixturefunc, request, kwargs) - except Exception: + except TEST_OUTCOME: fixturedef.cached_result = (None, my_cache_key, sys.exc_info()) raise fixturedef.cached_result = (result, my_cache_key, None) diff --git a/_pytest/main.py b/_pytest/main.py index 274b39782..4bddf1e2d 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -14,7 +14,8 @@ except ImportError: from UserDict import DictMixin as MappingMixin from _pytest.config import directory_arg, UsageError, hookimpl -from _pytest.runner import collect_one_node, exit +from _pytest.runner import collect_one_node +from _pytest.outcomes import exit tracebackcutdir = py.path.local(_pytest.__file__).dirpath() diff --git a/_pytest/outcomes.py b/_pytest/outcomes.py new file mode 100644 index 000000000..57e361260 --- /dev/null +++ b/_pytest/outcomes.py @@ -0,0 +1,141 @@ +""" +exception classes and constants handling test outcomes +as well as functions creating them +""" +from __future__ import absolute_import, division, print_function +import py +import sys + + +class OutcomeException(BaseException): + """ OutcomeException and its subclass instances indicate and + contain info about test and collection outcomes. + """ + def __init__(self, msg=None, pytrace=True): + BaseException.__init__(self, msg) + self.msg = msg + self.pytrace = pytrace + + def __repr__(self): + if self.msg: + val = self.msg + if isinstance(val, bytes): + val = py._builtin._totext(val, errors='replace') + return val + return "<%s instance>" %(self.__class__.__name__,) + __str__ = __repr__ + + +TEST_OUTCOME = (OutcomeException, Exception) + + +class Skipped(OutcomeException): + # XXX hackish: on 3k we fake to live in the builtins + # in order to have Skipped exception printing shorter/nicer + __module__ = 'builtins' + + def __init__(self, msg=None, pytrace=True, allow_module_level=False): + OutcomeException.__init__(self, msg=msg, pytrace=pytrace) + self.allow_module_level = allow_module_level + + +class Failed(OutcomeException): + """ raised from an explicit call to pytest.fail() """ + __module__ = 'builtins' + + +class Exit(KeyboardInterrupt): + """ raised for immediate program exits (no tracebacks/summaries)""" + def __init__(self, msg="unknown reason"): + self.msg = msg + KeyboardInterrupt.__init__(self, msg) + +# exposed helper methods + +def exit(msg): + """ exit testing process as if KeyboardInterrupt was triggered. """ + __tracebackhide__ = True + raise Exit(msg) + + +exit.Exception = Exit + + +def skip(msg=""): + """ skip an executing test with the given message. Note: it's usually + better to use the pytest.mark.skipif marker to declare a test to be + skipped under certain conditions like mismatching platforms or + dependencies. See the pytest_skipping plugin for details. + """ + __tracebackhide__ = True + raise Skipped(msg=msg) + + +skip.Exception = Skipped + + +def fail(msg="", pytrace=True): + """ explicitly fail an currently-executing test with the given Message. + + :arg pytrace: if false the msg represents the full failure information + and no python traceback will be reported. + """ + __tracebackhide__ = True + raise Failed(msg=msg, pytrace=pytrace) + + +fail.Exception = Failed + + + +class XFailed(fail.Exception): + """ raised from an explicit call to pytest.xfail() """ + + +def xfail(reason=""): + """ xfail an executing test or setup functions with the given reason.""" + __tracebackhide__ = True + raise XFailed(reason) + + +xfail.Exception = XFailed + + + +def importorskip(modname, minversion=None): + """ return imported module if it has at least "minversion" as its + __version__ attribute. If no minversion is specified the a skip + is only triggered if the module can not be imported. + """ + import warnings + __tracebackhide__ = True + compile(modname, '', 'eval') # to catch syntaxerrors + should_skip = False + + with warnings.catch_warnings(): + # make sure to ignore ImportWarnings that might happen because + # of existing directories with the same name we're trying to + # import but without a __init__.py file + warnings.simplefilter('ignore') + try: + __import__(modname) + except ImportError: + # Do not raise chained exception here(#1485) + should_skip = True + if should_skip: + raise Skipped("could not import %r" %(modname,), allow_module_level=True) + mod = sys.modules[modname] + if minversion is None: + return mod + verattr = getattr(mod, '__version__', None) + if minversion is not None: + try: + from pkg_resources import parse_version as pv + except ImportError: + raise Skipped("we have a required version for %r but can not import " + "pkg_resources to parse version strings." % (modname,), + allow_module_level=True) + if verattr is None or pv(verattr) < pv(minversion): + raise Skipped("module %r has __version__ %r, required is: %r" %( + modname, verattr, minversion), allow_module_level=True) + return mod diff --git a/_pytest/python.py b/_pytest/python.py index 267372888..e7bb9ad62 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -23,7 +23,7 @@ from _pytest.compat import ( get_real_func, getfslineno, safe_getattr, safe_str, getlocation, enum, ) -from _pytest.runner import fail +from _pytest.outcomes import fail from _pytest.mark import transfer_markers cutdir1 = py.path.local(pluggy.__file__.rstrip("oc")) diff --git a/_pytest/python_api.py b/_pytest/python_api.py index 176aff590..051769398 100644 --- a/_pytest/python_api.py +++ b/_pytest/python_api.py @@ -4,7 +4,7 @@ import sys import py from _pytest.compat import isclass, izip -from _pytest.runner import fail +from _pytest.outcomes import fail import _pytest._code # builtin pytest.approx helper diff --git a/_pytest/recwarn.py b/_pytest/recwarn.py index b9938752c..757b043a1 100644 --- a/_pytest/recwarn.py +++ b/_pytest/recwarn.py @@ -7,8 +7,9 @@ import _pytest._code import py import sys import warnings -from _pytest.fixtures import yield_fixture +from _pytest.fixtures import yield_fixture +from _pytest.outcomes import fail @yield_fixture def recwarn(): @@ -197,7 +198,6 @@ class WarningsChecker(WarningsRecorder): if not any(issubclass(r.category, self.expected_warning) for r in self): __tracebackhide__ = True - from _pytest.runner import fail fail("DID NOT WARN. No warnings of type {0} was emitted. " "The list of emitted warnings is: {1}.".format( self.expected_warning, diff --git a/_pytest/runner.py b/_pytest/runner.py index b5829f46d..0837b01c3 100644 --- a/_pytest/runner.py +++ b/_pytest/runner.py @@ -8,7 +8,7 @@ from time import time import py from _pytest._code.code import TerminalRepr, ExceptionInfo - +from _pytest.outcomes import skip, Skipped, TEST_OUTCOME # # pytest plugin hooks @@ -445,7 +445,7 @@ class SetupState(object): fin = finalizers.pop() try: fin() - except Exception: + except TEST_OUTCOME: # XXX Only first exception will be seen by user, # ideally all should be reported. if exc is None: @@ -492,7 +492,7 @@ class SetupState(object): self.stack.append(col) try: col.setup() - except Exception: + except TEST_OUTCOME: col._prepare_exc = sys.exc_info() raise @@ -507,124 +507,5 @@ def collect_one_node(collector): return rep -# ============================================================= -# Test OutcomeExceptions and helpers for creating them. -class OutcomeException(Exception): - """ OutcomeException and its subclass instances indicate and - contain info about test and collection outcomes. - """ - - def __init__(self, msg=None, pytrace=True): - Exception.__init__(self, msg) - self.msg = msg - self.pytrace = pytrace - - def __repr__(self): - if self.msg: - val = self.msg - if isinstance(val, bytes): - val = py._builtin._totext(val, errors='replace') - return val - return "<%s instance>" % (self.__class__.__name__,) - __str__ = __repr__ - - -class Skipped(OutcomeException): - # XXX hackish: on 3k we fake to live in the builtins - # in order to have Skipped exception printing shorter/nicer - __module__ = 'builtins' - - def __init__(self, msg=None, pytrace=True, allow_module_level=False): - OutcomeException.__init__(self, msg=msg, pytrace=pytrace) - self.allow_module_level = allow_module_level - - -class Failed(OutcomeException): - """ raised from an explicit call to pytest.fail() """ - __module__ = 'builtins' - - -class Exit(KeyboardInterrupt): - """ raised for immediate program exits (no tracebacks/summaries)""" - - def __init__(self, msg="unknown reason"): - self.msg = msg - KeyboardInterrupt.__init__(self, msg) - -# exposed helper methods - - -def exit(msg): - """ exit testing process as if KeyboardInterrupt was triggered. """ - __tracebackhide__ = True - raise Exit(msg) - - -exit.Exception = Exit - - -def skip(msg=""): - """ skip an executing test with the given message. Note: it's usually - better to use the pytest.mark.skipif marker to declare a test to be - skipped under certain conditions like mismatching platforms or - dependencies. See the pytest_skipping plugin for details. - """ - __tracebackhide__ = True - raise Skipped(msg=msg) - - -skip.Exception = Skipped - - -def fail(msg="", pytrace=True): - """ explicitly fail an currently-executing test with the given Message. - - :arg pytrace: if false the msg represents the full failure information - and no python traceback will be reported. - """ - __tracebackhide__ = True - raise Failed(msg=msg, pytrace=pytrace) - - -fail.Exception = Failed - - -def importorskip(modname, minversion=None): - """ return imported module if it has at least "minversion" as its - __version__ attribute. If no minversion is specified the a skip - is only triggered if the module can not be imported. - """ - import warnings - __tracebackhide__ = True - compile(modname, '', 'eval') # to catch syntaxerrors - should_skip = False - - with warnings.catch_warnings(): - # make sure to ignore ImportWarnings that might happen because - # of existing directories with the same name we're trying to - # import but without a __init__.py file - warnings.simplefilter('ignore') - try: - __import__(modname) - except ImportError: - # Do not raise chained exception here(#1485) - should_skip = True - if should_skip: - raise Skipped("could not import %r" % (modname,), allow_module_level=True) - mod = sys.modules[modname] - if minversion is None: - return mod - verattr = getattr(mod, '__version__', None) - if minversion is not None: - try: - from pkg_resources import parse_version as pv - except ImportError: - raise Skipped("we have a required version for %r but can not import " - "pkg_resources to parse version strings." % (modname,), - allow_module_level=True) - if verattr is None or pv(verattr) < pv(minversion): - raise Skipped("module %r has __version__ %r, required is: %r" % ( - modname, verattr, minversion), allow_module_level=True) - return mod diff --git a/_pytest/skipping.py b/_pytest/skipping.py index b11aea801..6954a0555 100644 --- a/_pytest/skipping.py +++ b/_pytest/skipping.py @@ -8,7 +8,7 @@ import traceback import py from _pytest.config import hookimpl from _pytest.mark import MarkInfo, MarkDecorator -from _pytest.runner import fail, skip +from _pytest.outcomes import fail, skip, xfail, TEST_OUTCOME def pytest_addoption(parser): @@ -34,7 +34,7 @@ def pytest_configure(config): def nop(*args, **kwargs): pass - nop.Exception = XFailed + nop.Exception = xfail.Exception setattr(pytest, "xfail", nop) config.addinivalue_line("markers", @@ -60,18 +60,6 @@ def pytest_configure(config): ) -class XFailed(fail.Exception): - """ raised from an explicit call to pytest.xfail() """ - - -def xfail(reason=""): - """ xfail an executing test or setup functions with the given reason.""" - __tracebackhide__ = True - raise XFailed(reason) - - -xfail.Exception = XFailed - class MarkEvaluator: def __init__(self, item, name): @@ -98,7 +86,7 @@ class MarkEvaluator: def istrue(self): try: return self._istrue() - except Exception: + except TEST_OUTCOME: self.exc = sys.exc_info() if isinstance(self.exc[1], SyntaxError): msg = [" " * (self.exc[1].offset + 4) + "^", ] diff --git a/_pytest/unittest.py b/_pytest/unittest.py index 3c473c6bd..585f81472 100644 --- a/_pytest/unittest.py +++ b/_pytest/unittest.py @@ -7,9 +7,9 @@ import traceback # for transferring markers import _pytest._code from _pytest.config import hookimpl -from _pytest.runner import fail, skip +from _pytest.outcomes import fail, skip, xfail from _pytest.python import transfer_markers, Class, Module, Function -from _pytest.skipping import MarkEvaluator, xfail +from _pytest.skipping import MarkEvaluator def pytest_pycollect_makeitem(collector, name, obj): diff --git a/changelog/580.feature b/changelog/580.feature new file mode 100644 index 000000000..5245c7341 --- /dev/null +++ b/changelog/580.feature @@ -0,0 +1 @@ +Exceptions raised by ``pytest.fail``, ``pytest.skip`` and ``pytest.xfail`` now subclass BaseException, making them harder to be caught unintentionally by normal code. \ No newline at end of file diff --git a/doc/en/builtin.rst b/doc/en/builtin.rst index af0dd9a74..a3b75b9b2 100644 --- a/doc/en/builtin.rst +++ b/doc/en/builtin.rst @@ -47,11 +47,11 @@ You can use the following functions in your test, fixture or setup functions to force a certain test outcome. Note that most often you can rather use declarative marks, see :ref:`skipping`. -.. autofunction:: _pytest.runner.fail -.. autofunction:: _pytest.runner.skip -.. autofunction:: _pytest.runner.importorskip -.. autofunction:: _pytest.skipping.xfail -.. autofunction:: _pytest.runner.exit +.. autofunction:: _pytest.outcomes.fail +.. autofunction:: _pytest.outcomes.skip +.. autofunction:: _pytest.outcomes.importorskip +.. autofunction:: _pytest.outcomes.xfail +.. autofunction:: _pytest.outcomes.exit Fixtures and requests ----------------------------------------------------- diff --git a/pytest.py b/pytest.py index da6b64910..1c914a6ed 100644 --- a/pytest.py +++ b/pytest.py @@ -16,9 +16,8 @@ from _pytest.freeze_support import freeze_includes from _pytest import __version__ from _pytest.debugging import pytestPDB as __pytestPDB from _pytest.recwarn import warns, deprecated_call -from _pytest.runner import fail, skip, importorskip, exit +from _pytest.outcomes import fail, skip, importorskip, exit, xfail from _pytest.mark import MARK_GEN as mark, param -from _pytest.skipping import xfail from _pytest.main import Item, Collector, File, Session from _pytest.fixtures import fillfixtures as _fillfuncargs from _pytest.python import ( diff --git a/testing/test_runner.py b/testing/test_runner.py index ae081b4f0..f1fcb86c7 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -6,7 +6,7 @@ import os import py import pytest import sys -from _pytest import runner, main +from _pytest import runner, main, outcomes class TestSetupState(object): @@ -449,10 +449,19 @@ def test_runtest_in_module_ordering(testdir): def test_outcomeexception_exceptionattributes(): - outcome = runner.OutcomeException('test') + outcome = outcomes.OutcomeException('test') assert outcome.args[0] == outcome.msg +def test_outcomeexception_passes_except_Exception(): + with pytest.raises(outcomes.OutcomeException): + try: + raise outcomes.OutcomeException('test') + except Exception: + pass + + + def test_pytest_exit(): try: pytest.exit("hello") From be401bc2f87ee3ab6fb48a2dd4d415581f847a6e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 28 Jul 2017 18:27:59 +0200 Subject: [PATCH 70/73] fix linting issues --- _pytest/outcomes.py | 11 +++++------ _pytest/recwarn.py | 1 + _pytest/runner.py | 5 +---- _pytest/skipping.py | 1 - testing/test_runner.py | 1 - 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/_pytest/outcomes.py b/_pytest/outcomes.py index 57e361260..ff5ef756d 100644 --- a/_pytest/outcomes.py +++ b/_pytest/outcomes.py @@ -22,7 +22,7 @@ class OutcomeException(BaseException): if isinstance(val, bytes): val = py._builtin._totext(val, errors='replace') return val - return "<%s instance>" %(self.__class__.__name__,) + return "<%s instance>" % (self.__class__.__name__,) __str__ = __repr__ @@ -52,6 +52,7 @@ class Exit(KeyboardInterrupt): # exposed helper methods + def exit(msg): """ exit testing process as if KeyboardInterrupt was triggered. """ __tracebackhide__ = True @@ -87,7 +88,6 @@ def fail(msg="", pytrace=True): fail.Exception = Failed - class XFailed(fail.Exception): """ raised from an explicit call to pytest.xfail() """ @@ -101,7 +101,6 @@ def xfail(reason=""): xfail.Exception = XFailed - def importorskip(modname, minversion=None): """ return imported module if it has at least "minversion" as its __version__ attribute. If no minversion is specified the a skip @@ -109,7 +108,7 @@ def importorskip(modname, minversion=None): """ import warnings __tracebackhide__ = True - compile(modname, '', 'eval') # to catch syntaxerrors + compile(modname, '', 'eval') # to catch syntaxerrors should_skip = False with warnings.catch_warnings(): @@ -123,7 +122,7 @@ def importorskip(modname, minversion=None): # Do not raise chained exception here(#1485) should_skip = True if should_skip: - raise Skipped("could not import %r" %(modname,), allow_module_level=True) + raise Skipped("could not import %r" % (modname,), allow_module_level=True) mod = sys.modules[modname] if minversion is None: return mod @@ -136,6 +135,6 @@ def importorskip(modname, minversion=None): "pkg_resources to parse version strings." % (modname,), allow_module_level=True) if verattr is None or pv(verattr) < pv(minversion): - raise Skipped("module %r has __version__ %r, required is: %r" %( + raise Skipped("module %r has __version__ %r, required is: %r" % ( modname, verattr, minversion), allow_module_level=True) return mod diff --git a/_pytest/recwarn.py b/_pytest/recwarn.py index 757b043a1..c9fa872c0 100644 --- a/_pytest/recwarn.py +++ b/_pytest/recwarn.py @@ -11,6 +11,7 @@ import warnings from _pytest.fixtures import yield_fixture from _pytest.outcomes import fail + @yield_fixture def recwarn(): """Return a WarningsRecorder instance that provides these methods: diff --git a/_pytest/runner.py b/_pytest/runner.py index 0837b01c3..a1daa9761 100644 --- a/_pytest/runner.py +++ b/_pytest/runner.py @@ -13,6 +13,7 @@ from _pytest.outcomes import skip, Skipped, TEST_OUTCOME # # pytest plugin hooks + def pytest_addoption(parser): group = parser.getgroup("terminal reporting", "reporting", after="general") group.addoption('--durations', @@ -505,7 +506,3 @@ def collect_one_node(collector): if call and check_interactive_exception(call, rep): ihook.pytest_exception_interact(node=collector, call=call, report=rep) return rep - - - - diff --git a/_pytest/skipping.py b/_pytest/skipping.py index 6954a0555..2fd61448a 100644 --- a/_pytest/skipping.py +++ b/_pytest/skipping.py @@ -60,7 +60,6 @@ def pytest_configure(config): ) - class MarkEvaluator: def __init__(self, item, name): self.item = item diff --git a/testing/test_runner.py b/testing/test_runner.py index f1fcb86c7..1ab449ba3 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -461,7 +461,6 @@ def test_outcomeexception_passes_except_Exception(): pass - def test_pytest_exit(): try: pytest.exit("hello") From 2e61f702c09899bf375dba0c59eec3760fdd8990 Mon Sep 17 00:00:00 2001 From: Jordan Moldow Date: Sat, 29 Jul 2017 02:39:17 -0700 Subject: [PATCH 71/73] Support PEP-415's Exception.__suppress_context__ PEP-415 states that `exception.__context__` should be suppressed in traceback outputs, if `exception.__suppress_context__` is `True`. Now if a ``raise exception from None`` is caught by pytest, pytest will no longer chain the context in the test report. The algorithm in `FormattedExcinfo` now better matches the one in `traceback.TracebackException`. `Exception.__suppress_context__` is available in all of the versions of Python 3 that are supported by pytest. Fixes #2631. --- AUTHORS | 1 + _pytest/_code/code.py | 2 +- changelog/2631.feature | 4 ++++ testing/code/test_excinfo.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 changelog/2631.feature diff --git a/AUTHORS b/AUTHORS index fdcb47690..1e2d34b30 100644 --- a/AUTHORS +++ b/AUTHORS @@ -84,6 +84,7 @@ John Towler Jon Sonesen Jonas Obrist Jordan Guymon +Jordan Moldow Joshua Bronson Jurko Gospodnetić Justyna Janczyszyn diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py index 5750211f2..0230c5660 100644 --- a/_pytest/_code/code.py +++ b/_pytest/_code/code.py @@ -679,7 +679,7 @@ class FormattedExcinfo(object): e = e.__cause__ excinfo = ExceptionInfo((type(e), e, e.__traceback__)) if e.__traceback__ else None descr = 'The above exception was the direct cause of the following exception:' - elif e.__context__ is not None: + elif (e.__context__ is not None and not e.__suppress_context__): e = e.__context__ excinfo = ExceptionInfo((type(e), e, e.__traceback__)) if e.__traceback__ else None descr = 'During handling of the above exception, another exception occurred:' diff --git a/changelog/2631.feature b/changelog/2631.feature new file mode 100644 index 000000000..91b903b17 --- /dev/null +++ b/changelog/2631.feature @@ -0,0 +1,4 @@ +Added support for `PEP-415's `_ +``Exception.__suppress_context__``. Now if a ``raise exception from None`` is +caught by pytest, pytest will no longer chain the context in the test report. +The behavior now matches Python's traceback behavior. diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index 37ceeb423..f8f8a0365 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -1095,6 +1095,36 @@ raise ValueError() assert line.endswith('mod.py') assert tw.lines[47] == ":15: AttributeError" + @pytest.mark.skipif("sys.version_info[0] < 3") + def test_exc_repr_with_raise_from_none_chain_suppression(self, importasmod): + mod = importasmod(""" + def f(): + try: + g() + except Exception: + raise AttributeError() from None + def g(): + raise ValueError() + """) + excinfo = pytest.raises(AttributeError, mod.f) + r = excinfo.getrepr(style="long") + tw = TWMock() + r.toterminal(tw) + for line in tw.lines: + print(line) + assert tw.lines[0] == "" + assert tw.lines[1] == " def f():" + assert tw.lines[2] == " try:" + assert tw.lines[3] == " g()" + assert tw.lines[4] == " except Exception:" + assert tw.lines[5] == "> raise AttributeError() from None" + assert tw.lines[6] == "E AttributeError" + assert tw.lines[7] == "" + line = tw.get_write_msg(8) + assert line.endswith('mod.py') + assert tw.lines[9] == ":6: AttributeError" + assert len(tw.lines) == 10 + @pytest.mark.skipif("sys.version_info[0] < 3") @pytest.mark.parametrize('reason, description', [ ('cause', 'The above exception was the direct cause of the following exception:'), From 07dd1ca7b88f2e1076376270b5cfc4f528de44b3 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sun, 30 Jul 2017 21:37:18 +0000 Subject: [PATCH 72/73] Preparing release version 3.2.0 --- CHANGELOG.rst | 166 ++++++++++++++++++++++++++++++ changelog/1994.feature | 1 - changelog/2003.removal | 2 - changelog/2023.bugfix | 1 - changelog/2147.removal | 1 - changelog/2375.bugfix | 1 - changelog/2427.removal | 1 - changelog/2444.trivial | 1 - changelog/2489.trivial | 1 - changelog/2510.bugfix | 1 - changelog/2516.feature | 1 - changelog/2518.feature | 1 - changelog/2528.feature | 1 - changelog/2533.trivial | 1 - changelog/2539.doc | 1 - changelog/2540.feature | 1 - changelog/2543.feature | 1 - changelog/2546.trivial | 1 - changelog/2548.bugfix | 1 - changelog/2555.bugfix | 1 - changelog/2562.trivial | 1 - changelog/2571.trivial | 1 - changelog/2574.bugfix | 1 - changelog/2581.trivial | 1 - changelog/2582.trivial | 1 - changelog/2583.feature | 2 - changelog/2588.trivial | 1 - changelog/2598.feature | 2 - changelog/2610.bugfix | 1 - changelog/2620.trivial | 1 - changelog/2621.feature | 2 - changelog/2622.feature | 2 - changelog/2631.feature | 4 - changelog/580.feature | 1 - changelog/920.bugfix | 1 - changelog/971.doc | 1 - doc/en/announce/index.rst | 1 + doc/en/announce/release-3.2.0.rst | 48 +++++++++ doc/en/builtin.rst | 16 +-- doc/en/cache.rst | 4 +- doc/en/example/markers.rst | 2 +- doc/en/example/parametrize.rst | 4 +- doc/en/example/reportingdemo.rst | 2 +- doc/en/example/simple.rst | 2 +- doc/en/parametrize.rst | 2 +- 45 files changed, 231 insertions(+), 59 deletions(-) delete mode 100644 changelog/1994.feature delete mode 100644 changelog/2003.removal delete mode 100644 changelog/2023.bugfix delete mode 100644 changelog/2147.removal delete mode 100644 changelog/2375.bugfix delete mode 100644 changelog/2427.removal delete mode 100644 changelog/2444.trivial delete mode 100644 changelog/2489.trivial delete mode 100644 changelog/2510.bugfix delete mode 100644 changelog/2516.feature delete mode 100644 changelog/2518.feature delete mode 100644 changelog/2528.feature delete mode 100644 changelog/2533.trivial delete mode 100644 changelog/2539.doc delete mode 100644 changelog/2540.feature delete mode 100644 changelog/2543.feature delete mode 100644 changelog/2546.trivial delete mode 100644 changelog/2548.bugfix delete mode 100644 changelog/2555.bugfix delete mode 100644 changelog/2562.trivial delete mode 100644 changelog/2571.trivial delete mode 100644 changelog/2574.bugfix delete mode 100644 changelog/2581.trivial delete mode 100644 changelog/2582.trivial delete mode 100644 changelog/2583.feature delete mode 100644 changelog/2588.trivial delete mode 100644 changelog/2598.feature delete mode 100644 changelog/2610.bugfix delete mode 100644 changelog/2620.trivial delete mode 100644 changelog/2621.feature delete mode 100644 changelog/2622.feature delete mode 100644 changelog/2631.feature delete mode 100644 changelog/580.feature delete mode 100644 changelog/920.bugfix delete mode 100644 changelog/971.doc create mode 100644 doc/en/announce/release-3.2.0.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1c1185dc0..e51a8f9d8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,172 @@ .. towncrier release notes start +Pytest 3.2.0 (2017-07-30) +========================= + +Deprecations and Removals +------------------------- + +- ``pytest.approx`` no longer supports ``>``, ``>=``, ``<`` and ``<=`` + operators to avoid surprising/inconsistent behavior. See `the docs + `_ for more + information. (`#2003 `_) + +- All old-style specific behavior in current classes in the pytest's API is + considered deprecated at this point and will be removed in a future release. + This affects Python 2 users only and in rare situations. (`#2147 + `_) + +- introduce deprecation warnings for legacy marks based parametersets (`#2427 + `_) + + +Features +-------- + +- Add support for numpy arrays (and dicts) to approx. (`#1994 + `_) + +- Now test function objects have a ``pytestmark`` attribute containing a list + of marks applied directly to the test function, as opposed to marks inherited + from parent classes or modules. (`#2516 `_) + +- Collection ignores local virtualenvs by default; `--collect-in-virtualenv` + overrides this behavior. (`#2518 `_) + +- Allow class methods decorated as ``@staticmethod`` to be candidates for + collection as a test function. (Only for Python 2.7 and above. Python 2.6 + will still ignore static methods.) (`#2528 `_) + +- Introduce ``mark.with_args`` in order to allow passing functions/classes as + sole argument to marks. (`#2540 `_) + +- New ``cache_dir`` ini option: sets a directory where stores content of cache + plugin. Default directory is ``.cache`` which is created in ``rootdir``. + Directory may be relative or absolute path. If setting relative path, then + directory is created relative to ``rootdir``. Additionally path may contain + environment variables, that will be expanded. (`#2543 + `_) + +- Introduce the ``PYTEST_CURRENT_TEST`` environment variable that is set with + the ``nodeid`` and stage (``setup``, ``call`` and ``teardown``) of the test + being currently executed. See the `documentation + `_ for more info. (`#2583 `_) + +- Introduced ``@pytest.mark.filterwarnings`` mark which allows overwriting the + warnings filter on a per test, class or module level. See the `docs + `_ for more information. (`#2598 `_) + +- ``--last-failed`` now remembers forever when a test has failed and only + forgets it if it passes again. This makes it easy to fix a test suite by + selectively running files and fixing tests incrementally. (`#2621 + `_) + +- New ``pytest_report_collectionfinish`` hook which allows plugins to add + messages to the terminal reporting after collection has been finished + successfully. (`#2622 `_) + +- Added support for `PEP-415's `_ + ``Exception.__suppress_context__``. Now if a ``raise exception from None`` is + caught by pytest, pytest will no longer chain the context in the test report. + The behavior now matches Python's traceback behavior. (`#2631 + `_) + +- Exceptions raised by ``pytest.fail``, ``pytest.skip`` and ``pytest.xfail`` + now subclass BaseException, making them harder to be caught unintentionally + by normal code. (`#580 `_) + + +Bug Fixes +--------- + +- Set ``stdin`` to a closed ``PIPE`` in ``pytester.py.Testdir.popen()`` for + avoid unwanted interactive ``pdb`` (`#2023 `_) + +- Add missing ``encoding`` attribute to ``sys.std*`` streams when using + ``capsys`` capture mode. (`#2375 `_) + +- Fix terminal color changing to black on Windows if ``colorama`` is imported + in a ``conftest.py`` file. (`#2510 `_) + +- Fix line number when reporting summary of skipped tests. (`#2548 + `_) + +- capture: ensure that EncodedFile.name is a string. (`#2555 + `_) + +- The options ```--fixtures`` and ```--fixtures-per-test`` will now keep + indentation within docstrings. (`#2574 `_) + +- doctests line numbers are now reported correctly, fixing `pytest-sugar#122 + `_. (`#2610 + `_) + +- Fix non-determinism in order of fixture collection. Adds new dependency + (ordereddict) for Python 2.6. (`#920 `_) + + +Improved Documentation +---------------------- + +- Clarify ``pytest_configure`` hook call order. (`#2539 + `_) + +- Extend documentation for testing plugin code with the ``pytester`` plugin. + (`#971 `_) + + +Trivial/Internal Changes +------------------------ + +- Update help message for ``--strict`` to make it clear it only deals with + unregistered markers, not warnings. (`#2444 `_) + +- Internal code move: move code for pytest.approx/pytest.raises to own files in + order to cut down the size of python.py (`#2489 `_) + +- Renamed the utility function ``_pytest.compat._escape_strings`` to + ``_ascii_escaped`` to better communicate the function's purpose. (`#2533 + `_) + +- Improve error message for CollectError with skip/skipif. (`#2546 + `_) + +- Emit warning about ``yield`` tests being deprecated only once per generator. + (`#2562 `_) + +- Ensure final collected line doesn't include artifacts of previous write. + (`#2571 `_) + +- Fixed all flake8 errors and warnings. (`#2581 `_) + +- Added ``fix-lint`` tox environment to run automatic pep8 fixes on the code. + (`#2582 `_) + +- Turn warnings into errors in pytest's own test suite in order to catch + regressions due to deprecations more promptly. (`#2588 + `_) + +- Show multiple issue links in CHANGELOG entries. (`#2620 + `_) + + Pytest 3.1.3 (2017-07-03) ========================= diff --git a/changelog/1994.feature b/changelog/1994.feature deleted file mode 100644 index f3c596e63..000000000 --- a/changelog/1994.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for numpy arrays (and dicts) to approx. diff --git a/changelog/2003.removal b/changelog/2003.removal deleted file mode 100644 index d3269bf4e..000000000 --- a/changelog/2003.removal +++ /dev/null @@ -1,2 +0,0 @@ -``pytest.approx`` no longer supports ``>``, ``>=``, ``<`` and ``<=`` operators to avoid surprising/inconsistent -behavior. See `the docs `_ for more information. diff --git a/changelog/2023.bugfix b/changelog/2023.bugfix deleted file mode 100644 index acf4b405b..000000000 --- a/changelog/2023.bugfix +++ /dev/null @@ -1 +0,0 @@ -Set ``stdin`` to a closed ``PIPE`` in ``pytester.py.Testdir.popen()`` for avoid unwanted interactive ``pdb`` diff --git a/changelog/2147.removal b/changelog/2147.removal deleted file mode 100644 index d5f80a108..000000000 --- a/changelog/2147.removal +++ /dev/null @@ -1 +0,0 @@ -All old-style specific behavior in current classes in the pytest's API is considered deprecated at this point and will be removed in a future release. This affects Python 2 users only and in rare situations. diff --git a/changelog/2375.bugfix b/changelog/2375.bugfix deleted file mode 100644 index 3f4fd3c3d..000000000 --- a/changelog/2375.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add missing ``encoding`` attribute to ``sys.std*`` streams when using ``capsys`` capture mode. diff --git a/changelog/2427.removal b/changelog/2427.removal deleted file mode 100644 index c7ed8e17a..000000000 --- a/changelog/2427.removal +++ /dev/null @@ -1 +0,0 @@ -introduce deprecation warnings for legacy marks based parametersets diff --git a/changelog/2444.trivial b/changelog/2444.trivial deleted file mode 100644 index 4d6e2de5b..000000000 --- a/changelog/2444.trivial +++ /dev/null @@ -1 +0,0 @@ -Update help message for ``--strict`` to make it clear it only deals with unregistered markers, not warnings. diff --git a/changelog/2489.trivial b/changelog/2489.trivial deleted file mode 100644 index c997d7e1e..000000000 --- a/changelog/2489.trivial +++ /dev/null @@ -1 +0,0 @@ -Internal code move: move code for pytest.approx/pytest.raises to own files in order to cut down the size of python.py \ No newline at end of file diff --git a/changelog/2510.bugfix b/changelog/2510.bugfix deleted file mode 100644 index e6fcb7c74..000000000 --- a/changelog/2510.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix terminal color changing to black on Windows if ``colorama`` is imported in a ``conftest.py`` file. diff --git a/changelog/2516.feature b/changelog/2516.feature deleted file mode 100644 index 6436de16a..000000000 --- a/changelog/2516.feature +++ /dev/null @@ -1 +0,0 @@ -Now test function objects have a ``pytestmark`` attribute containing a list of marks applied directly to the test function, as opposed to marks inherited from parent classes or modules. \ No newline at end of file diff --git a/changelog/2518.feature b/changelog/2518.feature deleted file mode 100644 index 2f6597a97..000000000 --- a/changelog/2518.feature +++ /dev/null @@ -1 +0,0 @@ -Collection ignores local virtualenvs by default; `--collect-in-virtualenv` overrides this behavior. diff --git a/changelog/2528.feature b/changelog/2528.feature deleted file mode 100644 index c91cdb5d7..000000000 --- a/changelog/2528.feature +++ /dev/null @@ -1 +0,0 @@ -Allow class methods decorated as ``@staticmethod`` to be candidates for collection as a test function. (Only for Python 2.7 and above. Python 2.6 will still ignore static methods.) diff --git a/changelog/2533.trivial b/changelog/2533.trivial deleted file mode 100644 index 930fd4c0d..000000000 --- a/changelog/2533.trivial +++ /dev/null @@ -1 +0,0 @@ -Renamed the utility function ``_pytest.compat._escape_strings`` to ``_ascii_escaped`` to better communicate the function's purpose. diff --git a/changelog/2539.doc b/changelog/2539.doc deleted file mode 100644 index 6d5a9c9db..000000000 --- a/changelog/2539.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify ``pytest_configure`` hook call order. diff --git a/changelog/2540.feature b/changelog/2540.feature deleted file mode 100644 index d65b1ea56..000000000 --- a/changelog/2540.feature +++ /dev/null @@ -1 +0,0 @@ -Introduce ``mark.with_args`` in order to allow passing functions/classes as sole argument to marks. \ No newline at end of file diff --git a/changelog/2543.feature b/changelog/2543.feature deleted file mode 100644 index 6d65a376f..000000000 --- a/changelog/2543.feature +++ /dev/null @@ -1 +0,0 @@ -New ``cache_dir`` ini option: sets a directory where stores content of cache plugin. Default directory is ``.cache`` which is created in ``rootdir``. Directory may be relative or absolute path. If setting relative path, then directory is created relative to ``rootdir``. Additionally path may contain environment variables, that will be expanded. diff --git a/changelog/2546.trivial b/changelog/2546.trivial deleted file mode 100644 index 53e43bc17..000000000 --- a/changelog/2546.trivial +++ /dev/null @@ -1 +0,0 @@ -Improve error message for CollectError with skip/skipif. diff --git a/changelog/2548.bugfix b/changelog/2548.bugfix deleted file mode 100644 index 8201594ed..000000000 --- a/changelog/2548.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix line number when reporting summary of skipped tests. diff --git a/changelog/2555.bugfix b/changelog/2555.bugfix deleted file mode 100644 index 8c20bbc67..000000000 --- a/changelog/2555.bugfix +++ /dev/null @@ -1 +0,0 @@ -capture: ensure that EncodedFile.name is a string. diff --git a/changelog/2562.trivial b/changelog/2562.trivial deleted file mode 100644 index 33e34ff65..000000000 --- a/changelog/2562.trivial +++ /dev/null @@ -1 +0,0 @@ -Emit warning about ``yield`` tests being deprecated only once per generator. diff --git a/changelog/2571.trivial b/changelog/2571.trivial deleted file mode 100644 index 45750f122..000000000 --- a/changelog/2571.trivial +++ /dev/null @@ -1 +0,0 @@ -Ensure final collected line doesn't include artifacts of previous write. diff --git a/changelog/2574.bugfix b/changelog/2574.bugfix deleted file mode 100644 index 13396bc16..000000000 --- a/changelog/2574.bugfix +++ /dev/null @@ -1 +0,0 @@ -The options ```--fixtures`` and ```--fixtures-per-test`` will now keep indentation within docstrings. diff --git a/changelog/2581.trivial b/changelog/2581.trivial deleted file mode 100644 index 341ef337f..000000000 --- a/changelog/2581.trivial +++ /dev/null @@ -1 +0,0 @@ -Fixed all flake8 errors and warnings. diff --git a/changelog/2582.trivial b/changelog/2582.trivial deleted file mode 100644 index a4e0793e4..000000000 --- a/changelog/2582.trivial +++ /dev/null @@ -1 +0,0 @@ -Added ``fix-lint`` tox environment to run automatic pep8 fixes on the code. diff --git a/changelog/2583.feature b/changelog/2583.feature deleted file mode 100644 index 315f2378e..000000000 --- a/changelog/2583.feature +++ /dev/null @@ -1,2 +0,0 @@ -Introduce the ``PYTEST_CURRENT_TEST`` environment variable that is set with the ``nodeid`` and stage (``setup``, ``call`` and -``teardown``) of the test being currently executed. See the `documentation `_ for more info. diff --git a/changelog/2588.trivial b/changelog/2588.trivial deleted file mode 100644 index 44ff69f74..000000000 --- a/changelog/2588.trivial +++ /dev/null @@ -1 +0,0 @@ -Turn warnings into errors in pytest's own test suite in order to catch regressions due to deprecations more promptly. diff --git a/changelog/2598.feature b/changelog/2598.feature deleted file mode 100644 index b811b9120..000000000 --- a/changelog/2598.feature +++ /dev/null @@ -1,2 +0,0 @@ -Introduced ``@pytest.mark.filterwarnings`` mark which allows overwriting the warnings filter on a per test, class or module level. -See the `docs `_ for more information. diff --git a/changelog/2610.bugfix b/changelog/2610.bugfix deleted file mode 100644 index 3757723e0..000000000 --- a/changelog/2610.bugfix +++ /dev/null @@ -1 +0,0 @@ -doctests line numbers are now reported correctly, fixing `pytest-sugar#122 `_. diff --git a/changelog/2620.trivial b/changelog/2620.trivial deleted file mode 100644 index 51c0bd160..000000000 --- a/changelog/2620.trivial +++ /dev/null @@ -1 +0,0 @@ -Show multiple issue links in CHANGELOG entries. diff --git a/changelog/2621.feature b/changelog/2621.feature deleted file mode 100644 index 19ca96355..000000000 --- a/changelog/2621.feature +++ /dev/null @@ -1,2 +0,0 @@ -``--last-failed`` now remembers forever when a test has failed and only forgets it if it passes again. This makes it -easy to fix a test suite by selectively running files and fixing tests incrementally. diff --git a/changelog/2622.feature b/changelog/2622.feature deleted file mode 100644 index 298892200..000000000 --- a/changelog/2622.feature +++ /dev/null @@ -1,2 +0,0 @@ -New ``pytest_report_collectionfinish`` hook which allows plugins to add messages to the terminal reporting after -collection has been finished successfully. diff --git a/changelog/2631.feature b/changelog/2631.feature deleted file mode 100644 index 91b903b17..000000000 --- a/changelog/2631.feature +++ /dev/null @@ -1,4 +0,0 @@ -Added support for `PEP-415's `_ -``Exception.__suppress_context__``. Now if a ``raise exception from None`` is -caught by pytest, pytest will no longer chain the context in the test report. -The behavior now matches Python's traceback behavior. diff --git a/changelog/580.feature b/changelog/580.feature deleted file mode 100644 index 5245c7341..000000000 --- a/changelog/580.feature +++ /dev/null @@ -1 +0,0 @@ -Exceptions raised by ``pytest.fail``, ``pytest.skip`` and ``pytest.xfail`` now subclass BaseException, making them harder to be caught unintentionally by normal code. \ No newline at end of file diff --git a/changelog/920.bugfix b/changelog/920.bugfix deleted file mode 100644 index d2dd2be1b..000000000 --- a/changelog/920.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix non-determinism in order of fixture collection. Adds new dependency (ordereddict) for Python 2.6. diff --git a/changelog/971.doc b/changelog/971.doc deleted file mode 100644 index e182cf8ee..000000000 --- a/changelog/971.doc +++ /dev/null @@ -1 +0,0 @@ -Extend documentation for testing plugin code with the ``pytester`` plugin. diff --git a/doc/en/announce/index.rst b/doc/en/announce/index.rst index 5061f4870..8a2f15d8d 100644 --- a/doc/en/announce/index.rst +++ b/doc/en/announce/index.rst @@ -6,6 +6,7 @@ Release announcements :maxdepth: 2 + release-3.2.0 release-3.1.3 release-3.1.2 release-3.1.1 diff --git a/doc/en/announce/release-3.2.0.rst b/doc/en/announce/release-3.2.0.rst new file mode 100644 index 000000000..4d2830edd --- /dev/null +++ b/doc/en/announce/release-3.2.0.rst @@ -0,0 +1,48 @@ +pytest-3.2.0 +======================================= + +The pytest team is proud to announce the 3.2.0 release! + +pytest is a mature Python testing tool with more than a 1600 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + http://doc.pytest.org/en/latest/changelog.html + +For complete documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Alex Hartoto +* Andras Tim +* Bruno Oliveira +* Daniel Hahler +* Florian Bruhin +* Floris Bruynooghe +* John Still +* Jordan Moldow +* Kale Kundert +* Lawrence Mitchell +* Llandy Riveron Del Risco +* Maik Figura +* Martin Altmayer +* Mihai Capotă +* Nathaniel Waisbrot +* Nguyễn Hồng Quân +* Pauli Virtanen +* Raphael Pierzina +* Ronny Pfannschmidt +* Segev Finer +* V.Kuznetsov + + +Happy testing, +The Pytest Development Team diff --git a/doc/en/builtin.rst b/doc/en/builtin.rst index a3b75b9b2..b59399a79 100644 --- a/doc/en/builtin.rst +++ b/doc/en/builtin.rst @@ -108,14 +108,14 @@ You can ask for available builtin or project-custom The returned ``monkeypatch`` fixture provides these helper methods to modify objects, dictionaries or os.environ:: - monkeypatch.setattr(obj, name, value, raising=True) - monkeypatch.delattr(obj, name, raising=True) - monkeypatch.setitem(mapping, name, value) - monkeypatch.delitem(obj, name, raising=True) - monkeypatch.setenv(name, value, prepend=False) - monkeypatch.delenv(name, value, raising=True) - monkeypatch.syspath_prepend(path) - monkeypatch.chdir(path) + monkeypatch.setattr(obj, name, value, raising=True) + monkeypatch.delattr(obj, name, raising=True) + monkeypatch.setitem(mapping, name, value) + monkeypatch.delitem(obj, name, raising=True) + monkeypatch.setenv(name, value, prepend=False) + monkeypatch.delenv(name, value, raising=True) + monkeypatch.syspath_prepend(path) + monkeypatch.chdir(path) All modifications will be undone after the requesting test function or fixture has finished. The ``raising`` diff --git a/doc/en/cache.rst b/doc/en/cache.rst index ac7a855fc..d5d6b653b 100644 --- a/doc/en/cache.rst +++ b/doc/en/cache.rst @@ -77,9 +77,9 @@ If you then run it with ``--lf``:: $ pytest --lf ======= test session starts ======== platform linux -- Python 3.x.y, pytest-3.x.y, py-1.x.y, pluggy-0.x.y - run-last-failure: rerun last 2 failures rootdir: $REGENDOC_TMPDIR, inifile: collected 50 items + run-last-failure: rerun previous 2 failures test_50.py FF @@ -119,9 +119,9 @@ of ``FF`` and dots):: $ pytest --ff ======= test session starts ======== platform linux -- Python 3.x.y, pytest-3.x.y, py-1.x.y, pluggy-0.x.y - run-last-failure: rerun last 2 failures first rootdir: $REGENDOC_TMPDIR, inifile: collected 50 items + run-last-failure: rerun previous 2 failures first test_50.py FF................................................ diff --git a/doc/en/example/markers.rst b/doc/en/example/markers.rst index d2c38fa81..52627245c 100644 --- a/doc/en/example/markers.rst +++ b/doc/en/example/markers.rst @@ -494,7 +494,7 @@ then you will see two tests skipped and two executed tests as expected:: test_plat.py s.s. ======= short test summary info ======== - SKIP [2] $REGENDOC_TMPDIR/conftest.py:12: cannot run on platform linux + SKIP [2] $REGENDOC_TMPDIR/conftest.py:13: cannot run on platform linux ======= 2 passed, 2 skipped in 0.12 seconds ======== diff --git a/doc/en/example/parametrize.rst b/doc/en/example/parametrize.rst index 2f2a11e6c..b72e8e6de 100644 --- a/doc/en/example/parametrize.rst +++ b/doc/en/example/parametrize.rst @@ -413,7 +413,7 @@ Running it results in some skips if we don't have all the python interpreters in . $ pytest -rs -q multipython.py sssssssssssssss.........sss.........sss......... ======= short test summary info ======== - SKIP [21] $REGENDOC_TMPDIR/CWD/multipython.py:23: 'python2.6' not found + SKIP [21] $REGENDOC_TMPDIR/CWD/multipython.py:24: 'python2.6' not found 27 passed, 21 skipped in 0.12 seconds Indirect parametrization of optional implementations/imports @@ -467,7 +467,7 @@ If you run this with reporting for skips enabled:: test_module.py .s ======= short test summary info ======== - SKIP [1] $REGENDOC_TMPDIR/conftest.py:10: could not import 'opt2' + SKIP [1] $REGENDOC_TMPDIR/conftest.py:11: could not import 'opt2' ======= 1 passed, 1 skipped in 0.12 seconds ======== diff --git a/doc/en/example/reportingdemo.rst b/doc/en/example/reportingdemo.rst index 47c18851d..7508d726f 100644 --- a/doc/en/example/reportingdemo.rst +++ b/doc/en/example/reportingdemo.rst @@ -358,7 +358,7 @@ get on the terminal - we are working on that):: > int(s) E ValueError: invalid literal for int() with base 10: 'qwe' - <0-codegen $PYTHON_PREFIX/lib/python3.5/site-packages/_pytest/python.py:1219>:1: ValueError + <0-codegen $PYTHON_PREFIX/lib/python3.5/site-packages/_pytest/python_api.py:570>:1: ValueError _______ TestRaises.test_raises_doesnt ________ self = diff --git a/doc/en/example/simple.rst b/doc/en/example/simple.rst index 6b5d5a868..f76d39264 100644 --- a/doc/en/example/simple.rst +++ b/doc/en/example/simple.rst @@ -170,7 +170,7 @@ and when running it will see a skipped "slow" test:: test_module.py .s ======= short test summary info ======== - SKIP [1] test_module.py:13: need --runslow option to run + SKIP [1] test_module.py:14: need --runslow option to run ======= 1 passed, 1 skipped in 0.12 seconds ======== diff --git a/doc/en/parametrize.rst b/doc/en/parametrize.rst index 5cde906fa..d1d47c229 100644 --- a/doc/en/parametrize.rst +++ b/doc/en/parametrize.rst @@ -195,7 +195,7 @@ list:: $ pytest -q -rs test_strings.py s ======= short test summary info ======== - SKIP [1] test_strings.py:1: got empty parameter set ['stringinput'], function test_valid_string at $REGENDOC_TMPDIR/test_strings.py:1 + SKIP [1] test_strings.py:2: got empty parameter set ['stringinput'], function test_valid_string at $REGENDOC_TMPDIR/test_strings.py:1 1 skipped in 0.12 seconds For further examples, you might want to look at :ref:`more From d2bca93109a7c85f316e493c5c31ee2322578980 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 1 Aug 2017 18:01:22 -0300 Subject: [PATCH 73/73] Update grammar in changelog as requested --- CHANGELOG.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e51a8f9d8..f033f8cab 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,8 +24,9 @@ Deprecations and Removals This affects Python 2 users only and in rare situations. (`#2147 `_) -- introduce deprecation warnings for legacy marks based parametersets (`#2427 - `_) +- A deprecation warning is now raised when using marks for parameters + in ``pytest.mark.parametrize``. Use ``pytest.param`` to apply marks to + parameters instead. (`#2427 `_) Features @@ -52,12 +53,11 @@ Features sole argument to marks. (`#2540 `_) -- New ``cache_dir`` ini option: sets a directory where stores content of cache - plugin. Default directory is ``.cache`` which is created in ``rootdir``. - Directory may be relative or absolute path. If setting relative path, then - directory is created relative to ``rootdir``. Additionally path may contain - environment variables, that will be expanded. (`#2543 - `_) +- New ``cache_dir`` ini option: sets the directory where the contents of the + cache plugin are stored. Directory may be relative or absolute path: if relative path, then + directory is created relative to ``rootdir``, otherwise it is used as is. + Additionally path may contain environment variables which are expanded during + runtime. (`#2543 `_) - Introduce the ``PYTEST_CURRENT_TEST`` environment variable that is set with the ``nodeid`` and stage (``setup``, ``call`` and ``teardown``) of the test