diff --git a/py/_plugin/pytest_skipping.py b/py/_plugin/pytest_skipping.py index 3f50b16b7..7360c86d8 100644 --- a/py/_plugin/pytest_skipping.py +++ b/py/_plugin/pytest_skipping.py @@ -123,44 +123,83 @@ within test or setup code. Example:: py.test.skip("unsuppored configuration") """ -# XXX py.test.skip, .importorskip and the Skipped class -# should also be defined in this plugin, requires thought/changes import py +class MarkEvaluator: + def __init__(self, item, name): + self.item = item + self.name = name + self.holder = getattr(item.obj, name, None) + + def __bool__(self): + return bool(self.holder) + __nonzero__ = __bool__ + + def istrue(self): + if self.holder: + d = {'os': py.std.os, 'sys': py.std.sys, 'config': self.item.config} + self.result = True + for expr in self.holder.args: + self.expr = expr + if isinstance(expr, str): + result = cached_eval(self.item.config, expr, d) + else: + result = expr + if not result: + self.result = False + self.expr = expr + break + return getattr(self, 'result', False) + + def get(self, attr, default=None): + return self.holder.kwargs.get(attr, default) + + def getexplanation(self): + expl = self.get('reason', None) + if not expl: + if not hasattr(self, 'expr'): + return "condition: True" + else: + return "condition: " + self.expr + return expl + def pytest_runtest_setup(item): if not isinstance(item, py.test.collect.Function): return - expr, result = evalexpression(item, 'skipif') - if result: - py.test.skip(expr) - holder = getattr(item.obj, 'xfail', None) - if holder and not holder.kwargs.get('run', True): - py.test.skip("") + evalskip = MarkEvaluator(item, 'skipif') + if evalskip.istrue(): + py.test.skip(evalskip.getexplanation()) + item._evalxfail = MarkEvaluator(item, 'xfail') + if item._evalxfail.istrue(): + if not item._evalxfail.get('run', True): + py.test.skip("xfail") def pytest_runtest_makereport(__multicall__, item, call): if not isinstance(item, py.test.collect.Function): return - if call.when == "setup": - holder = getattr(item.obj, 'xfail', None) - if holder: - rep = __multicall__.execute() - reason = holder.kwargs.get("reason", "") - rep.keywords['xfail'] = "[not run] " + reason - return rep + evalxfail = getattr(item, '_evalxfail', None) + if not evalxfail: return - elif call.when == "call": - expr, result = evalexpression(item, 'xfail') + if call.when == "setup": rep = __multicall__.execute() - if result: + if rep.skipped and evalxfail.istrue(): + expl = evalxfail.getexplanation() + if not evalxfail.get("run", True): + expl = "[NOTRUN] " + expl + rep.keywords['xfail'] = expl + return rep + elif call.when == "call": + rep = __multicall__.execute() + if evalxfail.istrue(): if call.excinfo: rep.skipped = True rep.failed = rep.passed = False else: rep.skipped = rep.passed = False rep.failed = True - rep.keywords['xfail'] = expr + rep.keywords['xfail'] = evalxfail.getexplanation() else: if 'xfail' in rep.keywords: del rep.keywords['xfail'] @@ -190,43 +229,17 @@ def show_xfailed(terminalreporter): return tr.write_sep("_", "expected failures") for rep in xfailed: - entry = rep.longrepr.reprcrash - modpath = rep.item.getmodpath(includemodule=True) - pos = "%s %s:%d: " %(modpath, entry.path, entry.lineno) - if rep.keywords['xfail']: - reason = rep.keywords['xfail'].strip() - else: - reason = rep.longrepr.reprcrash.message - i = reason.find("\n") - if i != -1: - reason = reason[:i] + pos = terminalreporter.gettestid(rep.item) + reason = rep.keywords['xfail'] tr._tw.line("%s %s" %(pos, reason)) xpassed = terminalreporter.stats.get("xpassed") if xpassed: tr.write_sep("_", "UNEXPECTEDLY PASSING TESTS") for rep in xpassed: - fspath, lineno, modpath = rep.item.reportinfo() - pos = "%s %s:%d: unexpectedly passing" %(modpath, fspath, lineno) - tr._tw.line(pos) - - -def evalexpression(item, keyword): - if isinstance(item, py.test.collect.Function): - markholder = getattr(item.obj, keyword, None) - result = False - if markholder: - d = {'os': py.std.os, 'sys': py.std.sys, 'config': item.config} - expr, result = None, True - for expr in markholder.args: - if isinstance(expr, str): - result = cached_eval(item.config, expr, d) - else: - result = expr - if not result: - break - return expr, result - return None, False + pos = terminalreporter.gettestid(rep.item) + reason = rep.keywords['xfail'] + tr._tw.line("%s %s" %(pos, reason)) def cached_eval(config, expr, d): if not hasattr(config, '_evalcache'): diff --git a/testing/plugin/test_pytest_skipping.py b/testing/plugin/test_pytest_skipping.py index 467d01517..c6b654376 100644 --- a/testing/plugin/test_pytest_skipping.py +++ b/testing/plugin/test_pytest_skipping.py @@ -1,40 +1,185 @@ import py -def test_xfail_not_report_default(testdir): - p = testdir.makepyfile(test_one=""" - import py - @py.test.mark.xfail - def test_this(): - assert 0 - """) - result = testdir.runpytest(p, '-v') - result.stdout.fnmatch_lines([ - "*1 expected failures*--report=xfailed*", - ]) +from py._plugin.pytest_skipping import MarkEvaluator +from py._plugin.pytest_skipping import pytest_runtest_setup +from py._plugin.pytest_runner import runtestprotocol -def test_xfail_not_run(testdir): - p = testdir.makepyfile(test_one=""" - import py - @py.test.mark.xfail(run=False, reason="noway") - def test_this(): - assert 0 - @py.test.mark.xfail("True", run=False, reason="noway") - def test_this_true(): - assert 0 - @py.test.mark.xfail("False", run=True, reason="huh") - def test_this_false(): - assert 1 - """) - result = testdir.runpytest(p, '-v') - result.stdout.fnmatch_lines([ - "*2 expected failures*--report=xfailed*", - "*1 passed*", - ]) - result = testdir.runpytest(p, '--report=xfailed', ) - result.stdout.fnmatch_lines([ - "*test_one*test_this*not run*noway", - "*test_one*test_this_true*not run*noway", - ]) +class TestEvaluator: + def test_no_marker(self, testdir): + item = testdir.getitem("def test_func(): pass") + evalskipif = MarkEvaluator(item, 'skipif') + assert not evalskipif + assert not evalskipif.istrue() + + def test_marked_no_args(self, testdir): + item = testdir.getitem(""" + import py + @py.test.mark.xyz + def test_func(): + pass + """) + ev = MarkEvaluator(item, 'xyz') + assert ev + assert ev.istrue() + expl = ev.getexplanation() + assert expl == "condition: True" + assert not ev.get("run", False) + + def test_marked_one_arg(self, testdir): + item = testdir.getitem(""" + import py + @py.test.mark.xyz("hasattr(os, 'sep')") + def test_func(): + pass + """) + ev = MarkEvaluator(item, 'xyz') + assert ev + assert ev.istrue() + expl = ev.getexplanation() + assert expl == "condition: hasattr(os, 'sep')" + + def test_marked_one_arg_with_reason(self, testdir): + item = testdir.getitem(""" + import py + @py.test.mark.xyz("hasattr(os, 'sep')", attr=2, reason="hello world") + def test_func(): + pass + """) + ev = MarkEvaluator(item, 'xyz') + assert ev + assert ev.istrue() + expl = ev.getexplanation() + assert expl == "hello world" + assert ev.get("attr") == 2 + + def test_skipif_class(self, testdir): + item, = testdir.getitems(""" + import py + class TestClass: + pytestmark = py.test.mark.skipif("config._hackxyz") + def test_func(self): + pass + """) + item.config._hackxyz = 3 + ev = MarkEvaluator(item, 'skipif') + assert ev.istrue() + expl = ev.getexplanation() + assert expl == "condition: config._hackxyz" + + +class TestXFail: + def test_xfail_simple(self, testdir): + item = testdir.getitem(""" + import py + @py.test.mark.xfail + def test_func(): + assert 0 + """) + reports = runtestprotocol(item, log=False) + assert len(reports) == 3 + callreport = reports[1] + assert callreport.skipped + expl = callreport.keywords['xfail'] + assert expl == "condition: True" + + def test_xfail_xpassed(self, testdir): + item = testdir.getitem(""" + import py + @py.test.mark.xfail + def test_func(): + assert 1 + """) + reports = runtestprotocol(item, log=False) + assert len(reports) == 3 + callreport = reports[1] + assert callreport.failed + expl = callreport.keywords['xfail'] + assert expl == "condition: True" + + def test_xfail_evalfalse_but_fails(self, testdir): + item = testdir.getitem(""" + import py + @py.test.mark.xfail('False') + def test_func(): + assert 0 + """) + reports = runtestprotocol(item, log=False) + callreport = reports[1] + assert callreport.failed + assert 'xfail' not in callreport.keywords + + def test_xfail_not_report_default(self, testdir): + p = testdir.makepyfile(test_one=""" + import py + @py.test.mark.xfail + def test_this(): + assert 0 + """) + result = testdir.runpytest(p, '-v') + result.stdout.fnmatch_lines([ + "*1 expected failures*--report=xfailed*", + ]) + + def test_xfail_not_run_xfail_reporting(self, testdir): + p = testdir.makepyfile(test_one=""" + import py + @py.test.mark.xfail(run=False, reason="noway") + def test_this(): + assert 0 + @py.test.mark.xfail("True", run=False) + def test_this_true(): + assert 0 + @py.test.mark.xfail("False", run=False, reason="huh") + def test_this_false(): + assert 1 + """) + result = testdir.runpytest(p, '--report=xfailed', ) + result.stdout.fnmatch_lines([ + "*test_one*test_this*NOTRUN*noway", + "*test_one*test_this_true*NOTRUN*condition:*True*", + "*1 passed*", + ]) + + def test_xfail_xpass(self, testdir): + p = testdir.makepyfile(test_one=""" + import py + @py.test.mark.xfail + def test_that(): + assert 1 + """) + result = testdir.runpytest(p, '--report=xfailed') + result.stdout.fnmatch_lines([ + "*UNEXPECTEDLY PASSING*", + "*test_that*", + "*1 xpassed*" + ]) + assert result.ret == 1 + +class TestSkipif: + def test_skipif_conditional(self, testdir): + item = testdir.getitem(""" + import py + @py.test.mark.skipif("hasattr(os, 'sep')") + def test_func(): + pass + """) + x = py.test.raises(py.test.skip.Exception, "pytest_runtest_setup(item)") + assert x.value.msg == "condition: hasattr(os, 'sep')" + + + def test_skipif_reporting(self, testdir): + p = testdir.makepyfile(""" + import py + @py.test.mark.skipif("hasattr(sys, 'platform')") + def test_that(): + assert 0 + """) + result = testdir.runpytest(p, '-s', '--report=skipped') + result.stdout.fnmatch_lines([ + "*Skipped*platform*", + "*1 skipped*" + ]) + assert result.ret == 0 def test_skip_not_report_default(testdir): p = testdir.makepyfile(test_one=""" @@ -47,69 +192,6 @@ def test_skip_not_report_default(testdir): "*1 skipped*--report=skipped*", ]) -def test_xfail_decorator(testdir): - p = testdir.makepyfile(test_one=""" - import py - @py.test.mark.xfail - def test_this(): - assert 0 - - @py.test.mark.xfail - def test_that(): - assert 1 - """) - result = testdir.runpytest(p, '--report=xfailed') - result.stdout.fnmatch_lines([ - "*expected failures*", - "*test_one.test_this*test_one.py:4*", - "*UNEXPECTEDLY PASSING*", - "*test_that*", - "*1 xfailed*" - ]) - assert result.ret == 1 - -def test_xfail_at_module(testdir): - p = testdir.makepyfile(""" - import py - pytestmark = py.test.mark.xfail('True') - def test_intentional_xfail(): - assert 0 - """) - result = testdir.runpytest(p, '--report=xfailed') - result.stdout.fnmatch_lines([ - "*expected failures*", - "*test_intentional_xfail*:4*", - "*1 xfailed*" - ]) - assert result.ret == 0 - -def test_xfail_evalfalse_but_fails(testdir): - p = testdir.makepyfile(""" - import py - @py.test.mark.xfail('False') - def test_fail(): - assert 0 - """) - result = testdir.runpytest(p, '--report=xfailed') - result.stdout.fnmatch_lines([ - "*test_xfail_evalfalse_but_fails*:4*", - "*1 failed*" - ]) - assert result.ret == 1 - -def test_skipif_decorator(testdir): - p = testdir.makepyfile(""" - import py - @py.test.mark.skipif("hasattr(sys, 'platform')") - def test_that(): - assert 0 - """) - result = testdir.runpytest(p, '--report=skipped') - result.stdout.fnmatch_lines([ - "*Skipped*platform*", - "*1 skipped*" - ]) - assert result.ret == 0 def test_skipif_class(testdir): p = testdir.makepyfile(""" @@ -127,19 +209,6 @@ def test_skipif_class(testdir): "*2 skipped*" ]) -def test_evalexpression_cls_config_example(testdir): - from py._plugin.pytest_skipping import evalexpression - item, = testdir.getitems(""" - import py - class TestClass: - pytestmark = py.test.mark.skipif("config._hackxyz") - def test_func(self): - pass - """) - item.config._hackxyz = 3 - x, y = evalexpression(item, 'skipif') - assert x == 'config._hackxyz' - assert y == 3 def test_skip_reasons_folding(): from py._plugin import pytest_runner as runner