pytest2/testing/test_terminal.py

"""
terminal reporting of the full testing process.
"""
import pytest
import py
import pluggy
import sys

from _pytest.terminal import TerminalReporter, repr_pythonversion, getreportopt
from _pytest import runner

def basic_run_report(item):
    runner.call_and_report(item, "setup", log=False)
    return runner.call_and_report(item, "call", log=False)

class Option:
    def __init__(self, verbose=False, fulltrace=False):
        self.verbose = verbose
        self.fulltrace = fulltrace

    @property
    def args(self):
        l = []
        if self.verbose:
            l.append('-v')
        if self.fulltrace:
            l.append('--fulltrace')
        return l

def pytest_generate_tests(metafunc):
    if "option" in metafunc.fixturenames:
        metafunc.addcall(id="default",
                         funcargs={'option': Option(verbose=False)})
        metafunc.addcall(id="verbose",
                         funcargs={'option': Option(verbose=True)})
        metafunc.addcall(id="quiet",
                         funcargs={'option': Option(verbose= -1)})
        metafunc.addcall(id="fulltrace",
                         funcargs={'option': Option(fulltrace=True)})


class TestTerminal:
    def test_pass_skip_fail(self, testdir, option):
        testdir.makepyfile("""
            import pytest
            def test_ok():
                pass
            def test_skip():
                pytest.skip("xx")
            def test_func():
                assert 0
        """)
        result = testdir.runpytest(*option.args)
        if option.verbose:
            result.stdout.fnmatch_lines([
                "*test_pass_skip_fail.py::test_ok PASS*",
                "*test_pass_skip_fail.py::test_skip SKIP*",
                "*test_pass_skip_fail.py::test_func FAIL*",
            ])
        else:
            result.stdout.fnmatch_lines([
            "*test_pass_skip_fail.py .sF"
        ])
        result.stdout.fnmatch_lines([
            "    def test_func():",
            ">       assert 0",
            "E       assert 0",
        ])

    def test_internalerror(self, testdir, linecomp):
        modcol = testdir.getmodulecol("def test_one(): pass")
        rep = TerminalReporter(modcol.config, file=linecomp.stringio)
        excinfo = pytest.raises(ValueError, "raise ValueError('hello')")
        rep.pytest_internalerror(excinfo.getrepr())
        linecomp.assert_contains_lines([
            "INTERNALERROR> *ValueError*hello*"
        ])

    def test_writeline(self, testdir, linecomp):
        modcol = testdir.getmodulecol("def test_one(): pass")
        rep = TerminalReporter(modcol.config, file=linecomp.stringio)
        rep.write_fspath_result(modcol.nodeid, ".")
        rep.write_line("hello world")
        lines = linecomp.stringio.getvalue().split('\n')
        assert not lines[0]
        assert lines[1].endswith(modcol.name + " .")
        assert lines[2] == "hello world"

    def test_show_runtest_logstart(self, testdir, linecomp):
        item = testdir.getitem("def test_func(): pass")
        tr = TerminalReporter(item.config, file=linecomp.stringio)
        item.config.pluginmanager.register(tr)
        location = item.reportinfo()
        tr.config.hook.pytest_runtest_logstart(nodeid=item.nodeid,
            location=location, fspath=str(item.fspath))
        linecomp.assert_contains_lines([
            "*test_show_runtest_logstart.py*"
        ])

    def test_runtest_location_shown_before_test_starts(self, testdir):
        testdir.makepyfile("""
            def test_1():
                import time
                time.sleep(20)
        """)
        child = testdir.spawn_pytest("")
        child.expect(".*test_runtest_location.*py")
        child.sendeof()
        child.kill(15)

    def test_itemreport_subclasses_show_subclassed_file(self, testdir):
        testdir.makepyfile(test_p1="""
            class BaseTests:
                def test_p1(self):
                    pass
            class TestClass(BaseTests):
                pass
        """)
        p2 = testdir.makepyfile(test_p2="""
            from test_p1 import BaseTests
            class TestMore(BaseTests):
                pass
        """)
        result = testdir.runpytest(p2)
        result.stdout.fnmatch_lines([
            "*test_p2.py .",
            "*1 passed*",
        ])
        result = testdir.runpytest("-v", p2)
        result.stdout.fnmatch_lines([
            "*test_p2.py::TestMore::test_p1* <- *test_p1.py*PASSED",
        ])

    def test_itemreport_directclasses_not_shown_as_subclasses(self, testdir):
        a = testdir.mkpydir("a123")
        a.join("test_hello123.py").write(py.code.Source("""
            class TestClass:
                def test_method(self):
                    pass
        """))
        result = testdir.runpytest("-v")
        assert result.ret == 0
        result.stdout.fnmatch_lines([
            "*a123/test_hello123.py*PASS*",
        ])
        assert " <- " not in result.stdout.str()

    def test_keyboard_interrupt(self, testdir, option):
        testdir.makepyfile("""
            def test_foobar():
                assert 0
            def test_spamegg():
                import py; pytest.skip('skip me please!')
            def test_interrupt_me():
                raise KeyboardInterrupt   # simulating the user
        """)

        result = testdir.runpytest(*option.args, no_reraise_ctrlc=True)
        result.stdout.fnmatch_lines([
            "    def test_foobar():",
            ">       assert 0",
            "E       assert 0",
            "*_keyboard_interrupt.py:6: KeyboardInterrupt*",
        ])
        if option.fulltrace:
            result.stdout.fnmatch_lines([
                "*raise KeyboardInterrupt   # simulating the user*",
            ])
        result.stdout.fnmatch_lines(['*KeyboardInterrupt*'])

    def test_keyboard_in_sessionstart(self, testdir):
        testdir.makeconftest("""
            def pytest_sessionstart():
                raise KeyboardInterrupt
        """)
        testdir.makepyfile("""
            def test_foobar():
                pass
        """)

        result = testdir.runpytest(no_reraise_ctrlc=True)
        assert result.ret == 2
        result.stdout.fnmatch_lines(['*KeyboardInterrupt*'])


class TestCollectonly:
    def test_collectonly_basic(self, testdir):
        testdir.makepyfile("""
            def test_func():
                pass
        """)
        result = testdir.runpytest("--collect-only",)
        result.stdout.fnmatch_lines([
           "<Module 'test_collectonly_basic.py'>",
           "  <Function 'test_func'>",
        ])

    def test_collectonly_skipped_module(self, testdir):
        testdir.makepyfile("""
            import pytest
            pytest.skip("hello")
        """)
        result = testdir.runpytest("--collect-only", "-rs")
        result.stdout.fnmatch_lines([
            "SKIP*hello*",
            "*1 skip*",
        ])

    def test_collectonly_failed_module(self, testdir):
        testdir.makepyfile("""raise ValueError(0)""")
        result = testdir.runpytest("--collect-only")
        result.stdout.fnmatch_lines([
            "*raise ValueError*",
            "*1 error*",
        ])

    def test_collectonly_fatal(self, testdir):
        testdir.makeconftest("""
            def pytest_collectstart(collector):
                assert 0, "urgs"
        """)
        result = testdir.runpytest("--collect-only")
        result.stdout.fnmatch_lines([
            "*INTERNAL*args*"
        ])
        assert result.ret == 3

    def test_collectonly_simple(self, testdir):
        p = testdir.makepyfile("""
            def test_func1():
                pass
            class TestClass:
                def test_method(self):
                    pass
        """)
        result = testdir.runpytest("--collect-only", p)
        #assert stderr.startswith("inserting into sys.path")
        assert result.ret == 0
        result.stdout.fnmatch_lines([
            "*<Module '*.py'>",
            "* <Function 'test_func1'*>",
            "* <Class 'TestClass'>",
            #"*  <Instance '()'>",
            "*   <Function 'test_method'*>",
        ])

    def test_collectonly_error(self, testdir):
        p = testdir.makepyfile("import Errlkjqweqwe")
        result = testdir.runpytest("--collect-only", p)
        assert result.ret == 1
        result.stdout.fnmatch_lines(py.code.Source("""
            *ERROR*
            *import Errlk*
            *ImportError*
            *1 error*
        """).strip())

    def test_collectonly_missing_path(self, testdir):
        """this checks issue 115,
            failure in parseargs will cause session
            not to have the items attribute
        """
        result = testdir.runpytest("--collect-only", "uhm_missing_path")
        assert result.ret == 4
        result.stderr.fnmatch_lines([
            '*ERROR: file not found*',
        ])

    def test_collectonly_quiet(self, testdir):
        testdir.makepyfile("def test_foo(): pass")
        result = testdir.runpytest("--collect-only", "-q")
        result.stdout.fnmatch_lines([
            '*test_foo*',
        ])

    def test_collectonly_more_quiet(self, testdir):
        testdir.makepyfile(test_fun="def test_foo(): pass")
        result = testdir.runpytest("--collect-only", "-qq")
        result.stdout.fnmatch_lines([
            '*test_fun.py: 1*',
        ])


def test_repr_python_version(monkeypatch):
    try:
        monkeypatch.setattr(sys, 'version_info', (2, 5, 1, 'final', 0))
        assert repr_pythonversion() == "2.5.1-final-0"
        py.std.sys.version_info = x = (2, 3)
        assert repr_pythonversion() == str(x)
    finally:
        monkeypatch.undo() # do this early as pytest can get confused

class TestFixtureReporting:
    def test_setup_fixture_error(self, testdir):
        testdir.makepyfile("""
            def setup_function(function):
                print ("setup func")
                assert 0
            def test_nada():
                pass
        """)
        result = testdir.runpytest()
        result.stdout.fnmatch_lines([
            "*ERROR at setup of test_nada*",
            "*setup_function(function):*",
            "*setup func*",
            "*assert 0*",
            "*1 error*",
        ])
        assert result.ret != 0

    def test_teardown_fixture_error(self, testdir):
        testdir.makepyfile("""
            def test_nada():
                pass
            def teardown_function(function):
                print ("teardown func")
                assert 0
        """)
        result = testdir.runpytest()
        result.stdout.fnmatch_lines([
            "*ERROR at teardown*",
            "*teardown_function(function):*",
            "*assert 0*",
            "*Captured stdout*",
            "*teardown func*",
            "*1 passed*1 error*",
        ])

    def test_teardown_fixture_error_and_test_failure(self, testdir):
        testdir.makepyfile("""
            def test_fail():
                assert 0, "failingfunc"

            def teardown_function(function):
                print ("teardown func")
                assert False
        """)
        result = testdir.runpytest()
        result.stdout.fnmatch_lines([
            "*ERROR at teardown of test_fail*",
            "*teardown_function(function):*",
            "*assert False*",
            "*Captured stdout*",
            "*teardown func*",

            "*test_fail*",
            "*def test_fail():",
            "*failingfunc*",
            "*1 failed*1 error*",
         ])

class TestTerminalFunctional:
    def test_deselected(self, testdir):
        testpath = testdir.makepyfile("""
                def test_one():
                    pass
                def test_two():
                    pass
                def test_three():
                    pass
           """
        )
        result = testdir.runpytest("-k", "test_two:", testpath)
        result.stdout.fnmatch_lines([
            "*test_deselected.py ..",
            "=* 1 test*deselected by*test_two:*=",
        ])
        assert result.ret == 0

    def test_no_skip_summary_if_failure(self, testdir):
        testdir.makepyfile("""
            import pytest
            def test_ok():
                pass
            def test_fail():
                assert 0
            def test_skip():
                pytest.skip("dontshow")
        """)
        result = testdir.runpytest()
        assert result.stdout.str().find("skip test summary") == -1
        assert result.ret == 1

    def test_passes(self, testdir):
        p1 = testdir.makepyfile("""
            def test_passes():
                pass
            class TestClass:
                def test_method(self):
                    pass
        """)
        old = p1.dirpath().chdir()
        try:
            result = testdir.runpytest()
        finally:
            old.chdir()
        result.stdout.fnmatch_lines([
            "test_passes.py ..",
            "* 2 pass*",
        ])
        assert result.ret == 0

    def test_header_trailer_info(self, testdir):
        testdir.makepyfile("""
            def test_passes():
                pass
        """)
        result = testdir.runpytest()
        verinfo = ".".join(map(str, py.std.sys.version_info[:3]))
        result.stdout.fnmatch_lines([
            "*===== test session starts ====*",
            "platform %s -- Python %s*pytest-%s*py-%s*pluggy-%s" % (
                py.std.sys.platform, verinfo,
                pytest.__version__, py.__version__, pluggy.__version__),
            "*test_header_trailer_info.py .",
            "=* 1 passed*in *.[0-9][0-9] seconds *=",
        ])
        if pytest.config.pluginmanager.list_plugin_distinfo():
            result.stdout.fnmatch_lines([
                "plugins: *",
            ])

    def test_showlocals(self, testdir):
        p1 = testdir.makepyfile("""
            def test_showlocals():
                x = 3
                y = "x" * 5000
                assert 0
        """)
        result = testdir.runpytest(p1, '-l')
        result.stdout.fnmatch_lines([
            #"_ _ * Locals *",
            "x* = 3",
            "y* = 'xxxxxx*"
        ])

    def test_verbose_reporting(self, testdir, pytestconfig):
        p1 = testdir.makepyfile("""
            import pytest
            def test_fail():
                raise ValueError()
            def test_pass():
                pass
            class TestClass:
                def test_skip(self):
                    pytest.skip("hello")
            def test_gen():
                def check(x):
                    assert x == 1
                yield check, 0
        """)
        result = testdir.runpytest(p1, '-v')
        result.stdout.fnmatch_lines([
            "*test_verbose_reporting.py::test_fail *FAIL*",
            "*test_verbose_reporting.py::test_pass *PASS*",
            "*test_verbose_reporting.py::TestClass::test_skip *SKIP*",
            "*test_verbose_reporting.py::test_gen*0* *FAIL*",
        ])
        assert result.ret == 1

        if not pytestconfig.pluginmanager.get_plugin("xdist"):
            pytest.skip("xdist plugin not installed")

        result = testdir.runpytest(p1, '-v', '-n 1')
        result.stdout.fnmatch_lines([
            "*FAIL*test_verbose_reporting.py::test_fail*",
        ])
        assert result.ret == 1

    def test_quiet_reporting(self, testdir):
        p1 = testdir.makepyfile("def test_pass(): pass")
        result = testdir.runpytest(p1, '-q')
        s = result.stdout.str()
        assert 'test session starts' not in s
        assert p1.basename not in s
        assert "===" not in s
        assert "passed" in s

    def test_more_quiet_reporting(self, testdir):
        p1 = testdir.makepyfile("def test_pass(): pass")
        result = testdir.runpytest(p1, '-qq')
        s = result.stdout.str()
        assert 'test session starts' not in s
        assert p1.basename not in s
        assert "===" not in s
        assert "passed" not in s


def test_fail_extra_reporting(testdir):
    testdir.makepyfile("def test_this(): assert 0")
    result = testdir.runpytest()
    assert 'short test summary' not in result.stdout.str()
    result = testdir.runpytest('-rf')
    result.stdout.fnmatch_lines([
        "*test summary*",
        "FAIL*test_fail_extra_reporting*",
    ])

def test_fail_reporting_on_pass(testdir):
    testdir.makepyfile("def test_this(): assert 1")
    result = testdir.runpytest('-rf')
    assert 'short test summary' not in result.stdout.str()

def test_color_yes(testdir):
    testdir.makepyfile("def test_this(): assert 1")
    result = testdir.runpytest('--color=yes')
    assert 'test session starts' in result.stdout.str()
    assert '\x1b[1m' in result.stdout.str()

def test_color_no(testdir):
    testdir.makepyfile("def test_this(): assert 1")
    result = testdir.runpytest('--color=no')
    assert 'test session starts' in result.stdout.str()
    assert '\x1b[1m' not in result.stdout.str()

def test_getreportopt():
    class config:
        class option:
            reportchars = ""
    config.option.report = "xfailed"
    assert getreportopt(config) == "x"

    config.option.report = "xfailed,skipped"
    assert getreportopt(config) == "xs"

    config.option.report = "skipped,xfailed"
    assert getreportopt(config) == "sx"

    config.option.report = "skipped"
    config.option.reportchars = "sf"
    assert getreportopt(config) == "sf"

    config.option.reportchars = "sfx"
    assert getreportopt(config) == "sfx"

def test_terminalreporter_reportopt_addopts(testdir):
    testdir.makeini("[pytest]\naddopts=-rs")
    testdir.makepyfile("""
        def pytest_funcarg__tr(request):
            tr = request.config.pluginmanager.getplugin("terminalreporter")
            return tr
        def test_opt(tr):
            assert tr.hasopt('skipped')
            assert not tr.hasopt('qwe')
    """)
    result = testdir.runpytest()
    result.stdout.fnmatch_lines([
        "*1 passed*"
    ])

def test_tbstyle_short(testdir):
    p = testdir.makepyfile("""
        def pytest_funcarg__arg(request):
            return 42
        def test_opt(arg):
            x = 0
            assert x
    """)
    result = testdir.runpytest("--tb=short")
    s = result.stdout.str()
    assert 'arg = 42' not in s
    assert 'x = 0' not in s
    result.stdout.fnmatch_lines([
        "*%s:5*" % p.basename,
        "    assert x",
        "E   assert*",
    ])
    result = testdir.runpytest()
    s = result.stdout.str()
    assert 'x = 0' in s
    assert 'assert x' in s

def test_traceconfig(testdir, monkeypatch):
    result = testdir.runpytest("--traceconfig")
    result.stdout.fnmatch_lines([
        "*active plugins*"
    ])
    assert result.ret == 0


class TestGenericReporting:
    """ this test class can be subclassed with a different option
        provider to run e.g. distributed tests.
    """
    def test_collect_fail(self, testdir, option):
        testdir.makepyfile("import xyz\n")
        result = testdir.runpytest(*option.args)
        result.stdout.fnmatch_lines([
            "?   import xyz",
            "E   ImportError: No module named *xyz*",
            "*1 error*",
        ])

    def test_maxfailures(self, testdir, option):
        testdir.makepyfile("""
            def test_1():
                assert 0
            def test_2():
                assert 0
            def test_3():
                assert 0
        """)
        result = testdir.runpytest("--maxfail=2", *option.args)
        result.stdout.fnmatch_lines([
            "*def test_1():*",
            "*def test_2():*",
            "*!! Interrupted: stopping after 2 failures*!!*",
            "*2 failed*",
        ])


    def test_tb_option(self, testdir, option):
        testdir.makepyfile("""
            import pytest
            def g():
                raise IndexError
            def test_func():
                print (6*7)
                g()  # --calling--
        """)
        for tbopt in ["long", "short", "no"]:
            print('testing --tb=%s...' % tbopt)
            result = testdir.runpytest('--tb=%s' % tbopt)
            s = result.stdout.str()
            if tbopt == "long":
                assert 'print (6*7)' in s
            else:
                assert 'print (6*7)' not in s
            if tbopt != "no":
                assert '--calling--' in s
                assert 'IndexError' in s
            else:
                assert 'FAILURES' not in s
                assert '--calling--' not in s
                assert 'IndexError' not in s

    def test_tb_crashline(self, testdir, option):
        p = testdir.makepyfile("""
            import pytest
            def g():
                raise IndexError
            def test_func1():
                print (6*7)
                g()  # --calling--
            def test_func2():
                assert 0, "hello"
        """)
        result = testdir.runpytest("--tb=line")
        bn = p.basename
        result.stdout.fnmatch_lines([
            "*%s:3: IndexError*" % bn,
            "*%s:8: AssertionError: hello*" % bn,
        ])
        s = result.stdout.str()
        assert "def test_func2" not in s

    def test_pytest_report_header(self, testdir, option):
        testdir.makeconftest("""
            def pytest_sessionstart(session):
                session.config._somevalue = 42
            def pytest_report_header(config):
                return "hello: %s" % config._somevalue
        """)
        testdir.mkdir("a").join("conftest.py").write("""
def pytest_report_header(config, startdir):
    return ["line1", str(startdir)]
""")
        result = testdir.runpytest("a")
        result.stdout.fnmatch_lines([
            "*hello: 42*",
            "line1",
            str(testdir.tmpdir),
        ])

@pytest.mark.xfail("not hasattr(os, 'dup')")
def test_fdopen_kept_alive_issue124(testdir):
    testdir.makepyfile("""
        import os, sys
        k = []
        def test_open_file_and_keep_alive(capfd):
            stdout = os.fdopen(1, 'w', 1)
            k.append(stdout)

        def test_close_kept_alive_file():
            stdout = k.pop()
            stdout.close()
    """)
    result = testdir.runpytest()
    result.stdout.fnmatch_lines([
        "*2 passed*"
    ])

def test_tbstyle_native_setup_error(testdir):
    testdir.makepyfile("""
        import pytest
        @pytest.fixture
        def setup_error_fixture():
            raise Exception("error in exception")

        def test_error_fixture(setup_error_fixture):
            pass
    """)
    result = testdir.runpytest("--tb=native")
    result.stdout.fnmatch_lines([
            '*File *test_tbstyle_native_setup_error.py", line *, in setup_error_fixture*'
            ])

def test_terminal_summary(testdir):
    testdir.makeconftest("""
        def pytest_terminal_summary(terminalreporter):
            w = terminalreporter
            w.section("hello")
            w.line("world")
    """)
    result = testdir.runpytest()
    result.stdout.fnmatch_lines("""
        *==== hello ====*
        world
    """)