add unit-tests for xfail and refine xfail handling and reporting
--HG-- branch : trunk
This commit is contained in:
holger krekel
@@ -1,40 +1,185 @@
|
||||
import py
|
||||
|
||||
def test_xfail_not_report_default(testdir):
|
||||
p = testdir.makepyfile(test_one="""
|
||||
import py
|
||||
@py.test.mark.xfail
|
||||
def test_this():
|
||||
assert 0
|
||||
""")
|
||||
result = testdir.runpytest(p, '-v')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*1 expected failures*--report=xfailed*",
|
||||
])
|
||||
from py._plugin.pytest_skipping import MarkEvaluator
|
||||
from py._plugin.pytest_skipping import pytest_runtest_setup
|
||||
from py._plugin.pytest_runner import runtestprotocol
|
||||
|
||||
def test_xfail_not_run(testdir):
|
||||
p = testdir.makepyfile(test_one="""
|
||||
import py
|
||||
@py.test.mark.xfail(run=False, reason="noway")
|
||||
def test_this():
|
||||
assert 0
|
||||
@py.test.mark.xfail("True", run=False, reason="noway")
|
||||
def test_this_true():
|
||||
assert 0
|
||||
@py.test.mark.xfail("False", run=True, reason="huh")
|
||||
def test_this_false():
|
||||
assert 1
|
||||
""")
|
||||
result = testdir.runpytest(p, '-v')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*2 expected failures*--report=xfailed*",
|
||||
"*1 passed*",
|
||||
])
|
||||
result = testdir.runpytest(p, '--report=xfailed', )
|
||||
result.stdout.fnmatch_lines([
|
||||
"*test_one*test_this*not run*noway",
|
||||
"*test_one*test_this_true*not run*noway",
|
||||
])
|
||||
class TestEvaluator:
|
||||
def test_no_marker(self, testdir):
|
||||
item = testdir.getitem("def test_func(): pass")
|
||||
evalskipif = MarkEvaluator(item, 'skipif')
|
||||
assert not evalskipif
|
||||
assert not evalskipif.istrue()
|
||||
|
||||
def test_marked_no_args(self, testdir):
|
||||
item = testdir.getitem("""
|
||||
import py
|
||||
@py.test.mark.xyz
|
||||
def test_func():
|
||||
pass
|
||||
""")
|
||||
ev = MarkEvaluator(item, 'xyz')
|
||||
assert ev
|
||||
assert ev.istrue()
|
||||
expl = ev.getexplanation()
|
||||
assert expl == "condition: True"
|
||||
assert not ev.get("run", False)
|
||||
|
||||
def test_marked_one_arg(self, testdir):
|
||||
item = testdir.getitem("""
|
||||
import py
|
||||
@py.test.mark.xyz("hasattr(os, 'sep')")
|
||||
def test_func():
|
||||
pass
|
||||
""")
|
||||
ev = MarkEvaluator(item, 'xyz')
|
||||
assert ev
|
||||
assert ev.istrue()
|
||||
expl = ev.getexplanation()
|
||||
assert expl == "condition: hasattr(os, 'sep')"
|
||||
|
||||
def test_marked_one_arg_with_reason(self, testdir):
|
||||
item = testdir.getitem("""
|
||||
import py
|
||||
@py.test.mark.xyz("hasattr(os, 'sep')", attr=2, reason="hello world")
|
||||
def test_func():
|
||||
pass
|
||||
""")
|
||||
ev = MarkEvaluator(item, 'xyz')
|
||||
assert ev
|
||||
assert ev.istrue()
|
||||
expl = ev.getexplanation()
|
||||
assert expl == "hello world"
|
||||
assert ev.get("attr") == 2
|
||||
|
||||
def test_skipif_class(self, testdir):
|
||||
item, = testdir.getitems("""
|
||||
import py
|
||||
class TestClass:
|
||||
pytestmark = py.test.mark.skipif("config._hackxyz")
|
||||
def test_func(self):
|
||||
pass
|
||||
""")
|
||||
item.config._hackxyz = 3
|
||||
ev = MarkEvaluator(item, 'skipif')
|
||||
assert ev.istrue()
|
||||
expl = ev.getexplanation()
|
||||
assert expl == "condition: config._hackxyz"
|
||||
|
||||
|
||||
class TestXFail:
|
||||
def test_xfail_simple(self, testdir):
|
||||
item = testdir.getitem("""
|
||||
import py
|
||||
@py.test.mark.xfail
|
||||
def test_func():
|
||||
assert 0
|
||||
""")
|
||||
reports = runtestprotocol(item, log=False)
|
||||
assert len(reports) == 3
|
||||
callreport = reports[1]
|
||||
assert callreport.skipped
|
||||
expl = callreport.keywords['xfail']
|
||||
assert expl == "condition: True"
|
||||
|
||||
def test_xfail_xpassed(self, testdir):
|
||||
item = testdir.getitem("""
|
||||
import py
|
||||
@py.test.mark.xfail
|
||||
def test_func():
|
||||
assert 1
|
||||
""")
|
||||
reports = runtestprotocol(item, log=False)
|
||||
assert len(reports) == 3
|
||||
callreport = reports[1]
|
||||
assert callreport.failed
|
||||
expl = callreport.keywords['xfail']
|
||||
assert expl == "condition: True"
|
||||
|
||||
def test_xfail_evalfalse_but_fails(self, testdir):
|
||||
item = testdir.getitem("""
|
||||
import py
|
||||
@py.test.mark.xfail('False')
|
||||
def test_func():
|
||||
assert 0
|
||||
""")
|
||||
reports = runtestprotocol(item, log=False)
|
||||
callreport = reports[1]
|
||||
assert callreport.failed
|
||||
assert 'xfail' not in callreport.keywords
|
||||
|
||||
def test_xfail_not_report_default(self, testdir):
|
||||
p = testdir.makepyfile(test_one="""
|
||||
import py
|
||||
@py.test.mark.xfail
|
||||
def test_this():
|
||||
assert 0
|
||||
""")
|
||||
result = testdir.runpytest(p, '-v')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*1 expected failures*--report=xfailed*",
|
||||
])
|
||||
|
||||
def test_xfail_not_run_xfail_reporting(self, testdir):
|
||||
p = testdir.makepyfile(test_one="""
|
||||
import py
|
||||
@py.test.mark.xfail(run=False, reason="noway")
|
||||
def test_this():
|
||||
assert 0
|
||||
@py.test.mark.xfail("True", run=False)
|
||||
def test_this_true():
|
||||
assert 0
|
||||
@py.test.mark.xfail("False", run=False, reason="huh")
|
||||
def test_this_false():
|
||||
assert 1
|
||||
""")
|
||||
result = testdir.runpytest(p, '--report=xfailed', )
|
||||
result.stdout.fnmatch_lines([
|
||||
"*test_one*test_this*NOTRUN*noway",
|
||||
"*test_one*test_this_true*NOTRUN*condition:*True*",
|
||||
"*1 passed*",
|
||||
])
|
||||
|
||||
def test_xfail_xpass(self, testdir):
|
||||
p = testdir.makepyfile(test_one="""
|
||||
import py
|
||||
@py.test.mark.xfail
|
||||
def test_that():
|
||||
assert 1
|
||||
""")
|
||||
result = testdir.runpytest(p, '--report=xfailed')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*UNEXPECTEDLY PASSING*",
|
||||
"*test_that*",
|
||||
"*1 xpassed*"
|
||||
])
|
||||
assert result.ret == 1
|
||||
|
||||
class TestSkipif:
|
||||
def test_skipif_conditional(self, testdir):
|
||||
item = testdir.getitem("""
|
||||
import py
|
||||
@py.test.mark.skipif("hasattr(os, 'sep')")
|
||||
def test_func():
|
||||
pass
|
||||
""")
|
||||
x = py.test.raises(py.test.skip.Exception, "pytest_runtest_setup(item)")
|
||||
assert x.value.msg == "condition: hasattr(os, 'sep')"
|
||||
|
||||
|
||||
def test_skipif_reporting(self, testdir):
|
||||
p = testdir.makepyfile("""
|
||||
import py
|
||||
@py.test.mark.skipif("hasattr(sys, 'platform')")
|
||||
def test_that():
|
||||
assert 0
|
||||
""")
|
||||
result = testdir.runpytest(p, '-s', '--report=skipped')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*Skipped*platform*",
|
||||
"*1 skipped*"
|
||||
])
|
||||
assert result.ret == 0
|
||||
|
||||
def test_skip_not_report_default(testdir):
|
||||
p = testdir.makepyfile(test_one="""
|
||||
@@ -47,69 +192,6 @@ def test_skip_not_report_default(testdir):
|
||||
"*1 skipped*--report=skipped*",
|
||||
])
|
||||
|
||||
def test_xfail_decorator(testdir):
|
||||
p = testdir.makepyfile(test_one="""
|
||||
import py
|
||||
@py.test.mark.xfail
|
||||
def test_this():
|
||||
assert 0
|
||||
|
||||
@py.test.mark.xfail
|
||||
def test_that():
|
||||
assert 1
|
||||
""")
|
||||
result = testdir.runpytest(p, '--report=xfailed')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*expected failures*",
|
||||
"*test_one.test_this*test_one.py:4*",
|
||||
"*UNEXPECTEDLY PASSING*",
|
||||
"*test_that*",
|
||||
"*1 xfailed*"
|
||||
])
|
||||
assert result.ret == 1
|
||||
|
||||
def test_xfail_at_module(testdir):
|
||||
p = testdir.makepyfile("""
|
||||
import py
|
||||
pytestmark = py.test.mark.xfail('True')
|
||||
def test_intentional_xfail():
|
||||
assert 0
|
||||
""")
|
||||
result = testdir.runpytest(p, '--report=xfailed')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*expected failures*",
|
||||
"*test_intentional_xfail*:4*",
|
||||
"*1 xfailed*"
|
||||
])
|
||||
assert result.ret == 0
|
||||
|
||||
def test_xfail_evalfalse_but_fails(testdir):
|
||||
p = testdir.makepyfile("""
|
||||
import py
|
||||
@py.test.mark.xfail('False')
|
||||
def test_fail():
|
||||
assert 0
|
||||
""")
|
||||
result = testdir.runpytest(p, '--report=xfailed')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*test_xfail_evalfalse_but_fails*:4*",
|
||||
"*1 failed*"
|
||||
])
|
||||
assert result.ret == 1
|
||||
|
||||
def test_skipif_decorator(testdir):
|
||||
p = testdir.makepyfile("""
|
||||
import py
|
||||
@py.test.mark.skipif("hasattr(sys, 'platform')")
|
||||
def test_that():
|
||||
assert 0
|
||||
""")
|
||||
result = testdir.runpytest(p, '--report=skipped')
|
||||
result.stdout.fnmatch_lines([
|
||||
"*Skipped*platform*",
|
||||
"*1 skipped*"
|
||||
])
|
||||
assert result.ret == 0
|
||||
|
||||
def test_skipif_class(testdir):
|
||||
p = testdir.makepyfile("""
|
||||
@@ -127,19 +209,6 @@ def test_skipif_class(testdir):
|
||||
"*2 skipped*"
|
||||
])
|
||||
|
||||
def test_evalexpression_cls_config_example(testdir):
|
||||
from py._plugin.pytest_skipping import evalexpression
|
||||
item, = testdir.getitems("""
|
||||
import py
|
||||
class TestClass:
|
||||
pytestmark = py.test.mark.skipif("config._hackxyz")
|
||||
def test_func(self):
|
||||
pass
|
||||
""")
|
||||
item.config._hackxyz = 3
|
||||
x, y = evalexpression(item, 'skipif')
|
||||
assert x == 'config._hackxyz'
|
||||
assert y == 3
|
||||
|
||||
def test_skip_reasons_folding():
|
||||
from py._plugin import pytest_runner as runner
|
||||
|
||||
Reference in New Issue
Block a user