Merge branch 'master' into term-color

Conflicts:
	src/_pytest/terminal.py
	testing/test_debugging.py
	testing/test_terminal.py

testing/acceptance_test.py

@@ -8,7 +8,7 @@ import py
 
 import pytest
 from _pytest.compat import importlib_metadata
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 
 
 def prepend_pythonpath(*dirs):
@@ -104,6 +104,8 @@ class TestGeneralUsage:
 
     @pytest.mark.parametrize("load_cov_early", [True, False])
     def test_early_load_setuptools_name(self, testdir, monkeypatch, load_cov_early):
+        monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
+
         testdir.makepyfile(mytestplugin1_module="")
         testdir.makepyfile(mytestplugin2_module="")
         testdir.makepyfile(mycov_module="")
@@ -188,10 +190,10 @@ class TestGeneralUsage:
         )
 
     @pytest.mark.filterwarnings("default")
-    def test_better_reporting_on_conftest_load_failure(self, testdir, request):
+    def test_better_reporting_on_conftest_load_failure(self, testdir):
         """Show a user-friendly traceback on conftest import failures (#486, #3332)"""
         testdir.makepyfile("")
-        testdir.makeconftest(
+        conftest = testdir.makeconftest(
             """
             def foo():
                 import qwerty
@@ -206,22 +208,18 @@ class TestGeneralUsage:
         """
         )
         result = testdir.runpytest()
-        dirname = request.node.name + "0"
         exc_name = (
             "ModuleNotFoundError" if sys.version_info >= (3, 6) else "ImportError"
         )
-        result.stderr.fnmatch_lines(
-            [
-                "ImportError while loading conftest '*{sep}{dirname}{sep}conftest.py'.".format(
-                    dirname=dirname, sep=os.sep
-                ),
-                "conftest.py:3: in <module>",
-                "    foo()",
-                "conftest.py:2: in foo",
-                "    import qwerty",
-                "E   {}: No module named 'qwerty'".format(exc_name),
-            ]
-        )
+        assert result.stdout.lines == []
+        assert result.stderr.lines == [
+            "ImportError while loading conftest '{}'.".format(conftest),
+            "conftest.py:3: in <module>",
+            "    foo()",
+            "conftest.py:2: in foo",
+            "    import qwerty",
+            "E   {}: No module named 'qwerty'".format(exc_name),
+        ]
 
     def test_early_skip(self, testdir):
         testdir.mkdir("xyz")
@@ -412,7 +410,7 @@ class TestGeneralUsage:
     def test_report_all_failed_collections_initargs(self, testdir):
         testdir.makeconftest(
             """
-            from _pytest.main import ExitCode
+            from _pytest.config import ExitCode
 
             def pytest_sessionfinish(exitstatus):
                 assert exitstatus == ExitCode.USAGE_ERROR
@@ -1277,11 +1275,39 @@ def test_pdb_can_be_rewritten(testdir):
             "    def check():",
             ">       assert 1 == 2",
             "E       assert 1 == 2",
-            "E         -1",
-            "E         +2",
+            "E         +1",
+            "E         -2",
             "",
             "pdb.py:2: AssertionError",
             "*= 1 failed in *",
         ]
     )
     assert result.ret == 1
+
+
+def test_tee_stdio_captures_and_live_prints(testdir):
+    testpath = testdir.makepyfile(
+        """
+        import sys
+        def test_simple():
+            print ("@this is stdout@")
+            print ("@this is stderr@", file=sys.stderr)
+    """
+    )
+    result = testdir.runpytest_subprocess(
+        testpath,
+        "--capture=tee-sys",
+        "--junitxml=output.xml",
+        "-o",
+        "junit_logging=all",
+    )
+
+    # ensure stdout/stderr were 'live printed'
+    result.stdout.fnmatch_lines(["*@this is stdout@*"])
+    result.stderr.fnmatch_lines(["*@this is stderr@*"])
+
+    # now ensure the output is in the junitxml
+    with open(os.path.join(testdir.tmpdir.strpath, "output.xml"), "r") as f:
+        fullXml = f.read()
+    assert "@this is stdout@\n" in fullXml
+    assert "@this is stderr@\n" in fullXml

testing/code/test_code.py

@@ -2,14 +2,17 @@ import sys
 from types import FrameType
 from unittest import mock
 
-import _pytest._code
 import pytest
+from _pytest._code import Code
+from _pytest._code import ExceptionInfo
+from _pytest._code import Frame
+from _pytest._code.code import ReprFuncArgs
 
 
 def test_ne() -> None:
-    code1 = _pytest._code.Code(compile('foo = "bar"', "", "exec"))
+    code1 = Code(compile('foo = "bar"', "", "exec"))
     assert code1 == code1
-    code2 = _pytest._code.Code(compile('foo = "baz"', "", "exec"))
+    code2 = Code(compile('foo = "baz"', "", "exec"))
     assert code2 != code1
 
 
@@ -17,7 +20,7 @@ def test_code_gives_back_name_for_not_existing_file() -> None:
     name = "abc-123"
     co_code = compile("pass\n", name, "exec")
     assert co_code.co_filename == name
-    code = _pytest._code.Code(co_code)
+    code = Code(co_code)
     assert str(code.path) == name
     assert code.fullsource is None
 
@@ -26,7 +29,7 @@ def test_code_with_class() -> None:
     class A:
         pass
 
-    pytest.raises(TypeError, _pytest._code.Code, A)
+    pytest.raises(TypeError, Code, A)
 
 
 def x() -> None:
@@ -34,13 +37,13 @@ def x() -> None:
 
 
 def test_code_fullsource() -> None:
-    code = _pytest._code.Code(x)
+    code = Code(x)
     full = code.fullsource
     assert "test_code_fullsource()" in str(full)
 
 
 def test_code_source() -> None:
-    code = _pytest._code.Code(x)
+    code = Code(x)
     src = code.source()
     expected = """def x() -> None:
     raise NotImplementedError()"""
@@ -51,7 +54,7 @@ def test_frame_getsourcelineno_myself() -> None:
     def func() -> FrameType:
         return sys._getframe(0)
 
-    f = _pytest._code.Frame(func())
+    f = Frame(func())
     source, lineno = f.code.fullsource, f.lineno
     assert source is not None
     assert source[lineno].startswith("        return sys._getframe(0)")
@@ -61,13 +64,13 @@ def test_getstatement_empty_fullsource() -> None:
     def func() -> FrameType:
         return sys._getframe(0)
 
-    f = _pytest._code.Frame(func())
+    f = Frame(func())
     with mock.patch.object(f.code.__class__, "fullsource", None):
         assert f.statement == ""
 
 
 def test_code_from_func() -> None:
-    co = _pytest._code.Code(test_frame_getsourcelineno_myself)
+    co = Code(test_frame_getsourcelineno_myself)
     assert co.firstlineno
     assert co.path
 
@@ -86,25 +89,25 @@ def test_code_getargs() -> None:
     def f1(x):
         raise NotImplementedError()
 
-    c1 = _pytest._code.Code(f1)
+    c1 = Code(f1)
     assert c1.getargs(var=True) == ("x",)
 
     def f2(x, *y):
         raise NotImplementedError()
 
-    c2 = _pytest._code.Code(f2)
+    c2 = Code(f2)
     assert c2.getargs(var=True) == ("x", "y")
 
     def f3(x, **z):
         raise NotImplementedError()
 
-    c3 = _pytest._code.Code(f3)
+    c3 = Code(f3)
     assert c3.getargs(var=True) == ("x", "z")
 
     def f4(x, *y, **z):
         raise NotImplementedError()
 
-    c4 = _pytest._code.Code(f4)
+    c4 = Code(f4)
     assert c4.getargs(var=True) == ("x", "y", "z")
 
 
@@ -112,25 +115,25 @@ def test_frame_getargs() -> None:
     def f1(x) -> FrameType:
         return sys._getframe(0)
 
-    fr1 = _pytest._code.Frame(f1("a"))
+    fr1 = Frame(f1("a"))
     assert fr1.getargs(var=True) == [("x", "a")]
 
     def f2(x, *y) -> FrameType:
         return sys._getframe(0)
 
-    fr2 = _pytest._code.Frame(f2("a", "b", "c"))
+    fr2 = Frame(f2("a", "b", "c"))
     assert fr2.getargs(var=True) == [("x", "a"), ("y", ("b", "c"))]
 
     def f3(x, **z) -> FrameType:
         return sys._getframe(0)
 
-    fr3 = _pytest._code.Frame(f3("a", b="c"))
+    fr3 = Frame(f3("a", b="c"))
     assert fr3.getargs(var=True) == [("x", "a"), ("z", {"b": "c"})]
 
     def f4(x, *y, **z) -> FrameType:
         return sys._getframe(0)
 
-    fr4 = _pytest._code.Frame(f4("a", "b", c="d"))
+    fr4 = Frame(f4("a", "b", c="d"))
     assert fr4.getargs(var=True) == [("x", "a"), ("y", ("b",)), ("z", {"c": "d"})]
 
 
@@ -142,12 +145,12 @@ class TestExceptionInfo:
             else:
                 assert False
         except AssertionError:
-            exci = _pytest._code.ExceptionInfo.from_current()
+            exci = ExceptionInfo.from_current()
         assert exci.getrepr()
 
     def test_from_current_with_missing(self) -> None:
         with pytest.raises(AssertionError, match="no current exception"):
-            _pytest._code.ExceptionInfo.from_current()
+            ExceptionInfo.from_current()
 
 
 class TestTracebackEntry:
@@ -158,7 +161,7 @@ class TestTracebackEntry:
             else:
                 assert False
         except AssertionError:
-            exci = _pytest._code.ExceptionInfo.from_current()
+            exci = ExceptionInfo.from_current()
         entry = exci.traceback[0]
         source = entry.getsource()
         assert source is not None
@@ -168,8 +171,6 @@ class TestTracebackEntry:
 
 class TestReprFuncArgs:
     def test_not_raise_exception_with_mixed_encoding(self, tw_mock) -> None:
-        from _pytest._code.code import ReprFuncArgs
-
         args = [("unicode_string", "São Paulo"), ("utf8_string", b"S\xc3\xa3o Paulo")]
 
         r = ReprFuncArgs(args)

testing/code/test_source.py

@@ -9,10 +9,11 @@ from typing import Any
 from typing import Dict
 from typing import Optional
 
-import py
+import py.path
 
 import _pytest._code
 import pytest
+from _pytest._code import getfslineno
 from _pytest._code import Source
 
 
@@ -496,10 +497,8 @@ def test_findsource() -> None:
 
 
 def test_getfslineno() -> None:
-    from _pytest._code import getfslineno
-
     def f(x) -> None:
-        pass
+        raise NotImplementedError()
 
     fspath, lineno = getfslineno(f)
 
@@ -513,6 +512,7 @@ def test_getfslineno() -> None:
     fspath, lineno = getfslineno(A)
 
     _, A_lineno = inspect.findsource(A)
+    assert isinstance(fspath, py.path.local)
     assert fspath.basename == "test_source.py"
     assert lineno == A_lineno
 
@@ -524,6 +524,14 @@ def test_getfslineno() -> None:
     B.__name__ = "B2"
     assert getfslineno(B)[1] == -1
 
+    co = compile("...", "", "eval")
+    assert co.co_filename == ""
+
+    if hasattr(sys, "pypy_version_info"):
+        assert getfslineno(co) == ("", -1)
+    else:
+        assert getfslineno(co) == ("", 0)
+
 
 def test_code_of_object_instance_with_call() -> None:
     class A:

testing/code/test_terminal_writer.py

@@ -0,0 +1,28 @@
+import re
+from io import StringIO
+
+import pytest
+from _pytest._io import TerminalWriter
+
+
+@pytest.mark.parametrize(
+    "has_markup, expected",
+    [
+        pytest.param(
+            True, "{kw}assert{hl-reset} {number}0{hl-reset}\n", id="with markup"
+        ),
+        pytest.param(False, "assert 0\n", id="no markup"),
+    ],
+)
+def test_code_highlight(has_markup, expected, color_mapping):
+    f = StringIO()
+    tw = TerminalWriter(f)
+    tw.hasmarkup = has_markup
+    tw._write_source(["assert 0"])
+    assert f.getvalue().splitlines(keepends=True) == color_mapping.format([expected])
+
+    with pytest.raises(
+        ValueError,
+        match=re.escape("indents size (2) should have same size as lines (1)"),
+    ):
+        tw._write_source(["assert 0"], [" ", " "])

testing/conftest.py

@@ -1,6 +1,10 @@
+import re
 import sys
+from typing import List
 
 import pytest
+from _pytest.pytester import RunResult
+from _pytest.pytester import Testdir
 
 if sys.gettrace():
 
@@ -77,6 +81,12 @@ def tw_mock():
         def write(self, msg, **kw):
             self.lines.append((TWMock.WRITE, msg))
 
+        def _write_source(self, lines, indents=()):
+            if not indents:
+                indents = [""] * len(lines)
+            for indent, line in zip(indents, lines):
+                self.line(indent + line)
+
         def line(self, line, **kw):
             self.lines.append(line)
 
@@ -118,3 +128,70 @@ def dummy_yaml_custom_test(testdir):
     """
     )
     testdir.makefile(".yaml", test1="")
+
+
+@pytest.fixture
+def testdir(testdir: Testdir) -> Testdir:
+    testdir.monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1")
+    return testdir
+
+
+@pytest.fixture(scope="session")
+def color_mapping():
+    """Returns a utility class which can replace keys in strings in the form "{NAME}"
+    by their equivalent ASCII codes in the terminal.
+
+    Used by tests which check the actual colors output by pytest.
+    """
+
+    class ColorMapping:
+        COLORS = {
+            "red": "\x1b[31m",
+            "green": "\x1b[32m",
+            "yellow": "\x1b[33m",
+            "bold": "\x1b[1m",
+            "reset": "\x1b[0m",
+            "kw": "\x1b[94m",
+            "hl-reset": "\x1b[39;49;00m",
+            "function": "\x1b[92m",
+            "number": "\x1b[94m",
+            "str": "\x1b[33m",
+            "print": "\x1b[96m",
+        }
+        RE_COLORS = {k: re.escape(v) for k, v in COLORS.items()}
+
+        @classmethod
+        def format(cls, lines: List[str]) -> List[str]:
+            """Straightforward replacement of color names to their ASCII codes."""
+            return [line.format(**cls.COLORS) for line in lines]
+
+        @classmethod
+        def format_for_fnmatch(cls, lines: List[str]) -> List[str]:
+            """Replace color names for use with LineMatcher.fnmatch_lines"""
+            return [line.format(**cls.COLORS).replace("[", "[[]") for line in lines]
+
+        @classmethod
+        def format_for_rematch(cls, lines: List[str]) -> List[str]:
+            """Replace color names for use with LineMatcher.re_match_lines"""
+            return [line.format(**cls.RE_COLORS) for line in lines]
+
+        @classmethod
+        def requires_ordered_markup(cls, result: RunResult):
+            """Should be called if a test expects markup to appear in the output
+            in the order they were passed, for example:
+
+                tw.write(line, bold=True, red=True)
+
+            In Python 3.5 there's no guarantee that the generated markup will appear
+            in the order called, so we do some limited color testing and skip the rest of
+            the test.
+            """
+            if sys.version_info < (3, 6):
+                # terminal writer.write accepts keyword arguments, so
+                # py36+ is required so the markup appears in the expected order
+                output = result.stdout.str()
+                assert "test session starts" in output
+                assert "\x1b[1m" in output
+                pytest.skip("doing limited testing because lacking ordered markup")
+
+    return ColorMapping

testing/deprecated_test.py

@@ -1,5 +1,8 @@
+import inspect
+
 import pytest
 from _pytest import deprecated
+from _pytest import nodes
 
 
 @pytest.mark.filterwarnings("default")
@@ -73,3 +76,58 @@ def test_warn_about_imminent_junit_family_default_change(testdir, junit_family):
         result.stdout.no_fnmatch_line(warning_msg)
     else:
         result.stdout.fnmatch_lines([warning_msg])
+
+
+def test_node_direct_ctor_warning():
+    class MockConfig:
+        pass
+
+    ms = MockConfig()
+    with pytest.warns(
+        DeprecationWarning,
+        match="direct construction of .* has been deprecated, please use .*.from_parent",
+    ) as w:
+        nodes.Node(name="test", config=ms, session=ms, nodeid="None")
+    assert w[0].lineno == inspect.currentframe().f_lineno - 1
+    assert w[0].filename == __file__
+
+
+def assert_no_print_logs(testdir, args):
+    result = testdir.runpytest(*args)
+    result.stdout.fnmatch_lines(
+        [
+            "*--no-print-logs is deprecated and scheduled for removal in pytest 6.0*",
+            "*Please use --show-capture instead.*",
+        ]
+    )
+
+
+@pytest.mark.filterwarnings("default")
+def test_noprintlogs_is_deprecated_cmdline(testdir):
+    testdir.makepyfile(
+        """
+        def test_foo():
+            pass
+        """
+    )
+
+    assert_no_print_logs(testdir, ("--no-print-logs",))
+
+
+@pytest.mark.filterwarnings("default")
+def test_noprintlogs_is_deprecated_ini(testdir):
+    testdir.makeini(
+        """
+        [pytest]
+        log_print=False
+        """
+    )
+
+    testdir.makepyfile(
+        """
+        def test_foo():
+            pass
+        """
+    )
+
+    assert_no_print_logs(testdir, ())

testing/io/test_saferepr.py

@@ -1,4 +1,5 @@
 import pytest
+from _pytest._io.saferepr import _pformat_dispatch
 from _pytest._io.saferepr import saferepr
 
 
@@ -147,3 +148,9 @@ def test_unicode():
     val = "£€"
     reprval = "'£€'"
     assert saferepr(val) == reprval
+
+
+def test_pformat_dispatch():
+    assert _pformat_dispatch("a") == "'a'"
+    assert _pformat_dispatch("a" * 10, width=5) == "'aaaaaaaaaa'"
+    assert _pformat_dispatch("foo bar", width=5) == "('foo '\n 'bar')"

testing/logging/test_reporting.py

@@ -946,7 +946,7 @@ def test_collection_collect_only_live_logging(testdir, verbose):
         expected_lines.extend(
             [
                 "*test_collection_collect_only_live_logging.py::test_simple*",
-                "no tests ran in 0.[0-9][0-9]s",
+                "no tests ran in [0-1].[0-9][0-9]s",
             ]
         )
     elif verbose == "-qq":

testing/python/approx.py

@@ -452,7 +452,7 @@ class TestApprox:
         expected = "4.0e-06"
         result = testdir.runpytest()
         result.stdout.fnmatch_lines(
-            ["*At index 0 diff: 3 != 4 * {}".format(expected), "=* 1 failed in *="]
+            ["*At index 0 diff: 3 != 4 ± {}".format(expected), "=* 1 failed in *="]
         )
 
     @pytest.mark.parametrize(

testing/python/collect.py

@@ -4,7 +4,7 @@ import textwrap
 
 import _pytest._code
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 from _pytest.nodes import Collector
 
 
@@ -281,10 +281,10 @@ class TestFunction:
         from _pytest.fixtures import FixtureManager
 
         config = testdir.parseconfigure()
-        session = testdir.Session(config)
+        session = testdir.Session.from_config(config)
         session._fixturemanager = FixtureManager(session)
 
-        return pytest.Function(config=config, parent=session, **kwargs)
+        return pytest.Function.from_parent(parent=session, **kwargs)
 
     def test_function_equality(self, testdir):
         def func1():
@@ -463,7 +463,7 @@ class TestFunction:
                return '3'
 
             @pytest.mark.parametrize('fix2', ['2'])
-            def test_it(fix1):
+            def test_it(fix1, fix2):
                assert fix1 == '21'
                assert not fix3_instantiated
         """
@@ -492,6 +492,19 @@ class TestFunction:
         )
         assert "foo" in keywords[1] and "bar" in keywords[1] and "baz" in keywords[1]
 
+    def test_parametrize_with_empty_string_arguments(self, testdir):
+        items = testdir.getitems(
+            """\
+            import pytest
+
+            @pytest.mark.parametrize('v', ('', ' '))
+            @pytest.mark.parametrize('w', ('', ' '))
+            def test(v, w): ...
+            """
+        )
+        names = {item.name for item in items}
+        assert names == {"test[-]", "test[ -]", "test[- ]", "test[ - ]"}
+
     def test_function_equality_with_callspec(self, testdir):
         items = testdir.getitems(
             """
@@ -1024,7 +1037,7 @@ class TestReportInfo:
                     return "ABCDE", 42, "custom"
             def pytest_pycollect_makeitem(collector, name, obj):
                 if name == "test_func":
-                    return MyFunction(name, parent=collector)
+                    return MyFunction.from_parent(name=name, parent=collector)
         """
         )
         item = testdir.getitem("def test_func(): pass")
@@ -1210,6 +1223,28 @@ def test_syntax_error_with_non_ascii_chars(testdir):
     result.stdout.fnmatch_lines(["*ERROR collecting*", "*SyntaxError*", "*1 error in*"])
 
 
+def test_collecterror_with_fulltrace(testdir):
+    testdir.makepyfile("assert 0")
+    result = testdir.runpytest("--fulltrace")
+    result.stdout.fnmatch_lines(
+        [
+            "collected 0 items / 1 error",
+            "",
+            "*= ERRORS =*",
+            "*_ ERROR collecting test_collecterror_with_fulltrace.py _*",
+            "",
+            "*/_pytest/python.py:*: ",
+            "_ _ _ _ _ _ _ _ *",
+            "",
+            ">   assert 0",
+            "E   assert 0",
+            "",
+            "test_collecterror_with_fulltrace.py:1: AssertionError",
+            "*! Interrupted: 1 error during collection !*",
+        ]
+    )
+
+
 def test_skip_duplicates_by_default(testdir):
     """Test for issue https://github.com/pytest-dev/pytest/issues/1609 (#1609)
 

testing/python/fixtures.py

@@ -1102,6 +1102,38 @@ class TestFixtureUsages:
             "*Fixture 'badscope' from test_invalid_scope.py got an unexpected scope value 'functions'"
         )
 
+    @pytest.mark.parametrize("scope", ["function", "session"])
+    def test_parameters_without_eq_semantics(self, scope, testdir):
+        testdir.makepyfile(
+            """
+            class NoEq1:  # fails on `a == b` statement
+                def __eq__(self, _):
+                    raise RuntimeError
+
+            class NoEq2:  # fails on `if a == b:` statement
+                def __eq__(self, _):
+                    class NoBool:
+                        def __bool__(self):
+                            raise RuntimeError
+                    return NoBool()
+
+            import pytest
+            @pytest.fixture(params=[NoEq1(), NoEq2()], scope={scope!r})
+            def no_eq(request):
+                return request.param
+
+            def test1(no_eq):
+                pass
+
+            def test2(no_eq):
+                pass
+        """.format(
+                scope=scope
+            )
+        )
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines(["*4 passed*"])
+
     def test_funcarg_parametrized_and_used_twice(self, testdir):
         testdir.makepyfile(
             """
@@ -3662,13 +3694,30 @@ class TestParameterizedSubRequest:
                 "    test_foos.py::test_foo",
                 "",
                 "Requested fixture 'fix_with_param' defined in:",
-                "*fix.py:4",
+                "{}:4".format(fixfile),
                 "Requested here:",
                 "test_foos.py:4",
                 "*1 failed*",
             ]
         )
 
+        # With non-overlapping rootdir, passing tests_dir.
+        rootdir = testdir.mkdir("rootdir")
+        rootdir.chdir()
+        result = testdir.runpytest("--rootdir", rootdir, tests_dir)
+        result.stdout.fnmatch_lines(
+            [
+                "The requested fixture has no parameter defined for test:",
+                "    test_foos.py::test_foo",
+                "",
+                "Requested fixture 'fix_with_param' defined in:",
+                "{}:4".format(fixfile),
+                "Requested here:",
+                "{}:4".format(testfile),
+                "*1 failed*",
+            ]
+        )
+
 
 def test_pytest_fixture_setup_and_post_finalizer_hook(testdir):
     testdir.makeconftest(

testing/python/integration.py

@@ -10,7 +10,7 @@ class TestOEJSKITSpecials:
             import pytest
             def pytest_pycollect_makeitem(collector, name, obj):
                 if name == "MyClass":
-                    return MyCollector(name, parent=collector)
+                    return MyCollector.from_parent(collector, name=name)
             class MyCollector(pytest.Collector):
                 def reportinfo(self):
                     return self.fspath, 3, "xyz"
@@ -40,7 +40,7 @@ class TestOEJSKITSpecials:
             import pytest
             def pytest_pycollect_makeitem(collector, name, obj):
                 if name == "MyClass":
-                    return MyCollector(name, parent=collector)
+                    return MyCollector.from_parent(collector, name=name)
             class MyCollector(pytest.Collector):
                 def reportinfo(self):
                     return self.fspath, 3, "xyz"

testing/python/metafunc.py

@@ -9,6 +9,8 @@ from hypothesis import strategies
 import pytest
 from _pytest import fixtures
 from _pytest import python
+from _pytest.outcomes import fail
+from _pytest.python import _idval
 
 
 class TestMetafunc:
@@ -26,9 +28,12 @@ class TestMetafunc:
         class DefinitionMock(python.FunctionDefinition):
             obj = attr.ib()
 
+            def listchain(self):
+                return []
+
         names = fixtures.getfuncargnames(func)
         fixtureinfo = FixtureInfo(names)
-        definition = DefinitionMock(func)
+        definition = DefinitionMock._create(func)
         return python.Metafunc(definition, fixtureinfo, config)
 
     def test_no_funcargs(self):
@@ -61,6 +66,39 @@ class TestMetafunc:
         pytest.raises(ValueError, lambda: metafunc.parametrize("y", [5, 6]))
         pytest.raises(ValueError, lambda: metafunc.parametrize("y", [5, 6]))
 
+        with pytest.raises(
+            TypeError, match="^ids must be a callable, sequence or generator$"
+        ):
+            metafunc.parametrize("y", [5, 6], ids=42)
+
+    def test_parametrize_error_iterator(self):
+        def func(x):
+            raise NotImplementedError()
+
+        class Exc(Exception):
+            def __repr__(self):
+                return "Exc(from_gen)"
+
+        def gen():
+            yield 0
+            yield None
+            yield Exc()
+
+        metafunc = self.Metafunc(func)
+        metafunc.parametrize("x", [1, 2], ids=gen())
+        assert [(x.funcargs, x.id) for x in metafunc._calls] == [
+            ({"x": 1}, "0"),
+            ({"x": 2}, "2"),
+        ]
+        with pytest.raises(
+            fail.Exception,
+            match=(
+                r"In func: ids must be list of string/float/int/bool, found:"
+                r" Exc\(from_gen\) \(type: <class .*Exc'>\) at index 2"
+            ),
+        ):
+            metafunc.parametrize("x", [1, 2, 3], ids=gen())
+
     def test_parametrize_bad_scope(self):
         def func(x):
             pass
@@ -167,6 +205,26 @@ class TestMetafunc:
                 ("x", "y"), [("abc", "def"), ("ghi", "jkl")], ids=["one"]
             )
 
+    def test_parametrize_ids_iterator_without_mark(self):
+        import itertools
+
+        def func(x, y):
+            pass
+
+        it = itertools.count()
+
+        metafunc = self.Metafunc(func)
+        metafunc.parametrize("x", [1, 2], ids=it)
+        metafunc.parametrize("y", [3, 4], ids=it)
+        ids = [x.id for x in metafunc._calls]
+        assert ids == ["0-2", "0-3", "1-2", "1-3"]
+
+        metafunc = self.Metafunc(func)
+        metafunc.parametrize("x", [1, 2], ids=it)
+        metafunc.parametrize("y", [3, 4], ids=it)
+        ids = [x.id for x in metafunc._calls]
+        assert ids == ["4-6", "4-7", "5-6", "5-7"]
+
     def test_parametrize_empty_list(self):
         """#510"""
 
@@ -209,8 +267,6 @@ class TestMetafunc:
         deadline=400.0
     )  # very close to std deadline and CI boxes are not reliable in CPU power
     def test_idval_hypothesis(self, value):
-        from _pytest.python import _idval
-
         escaped = _idval(value, "a", 6, None, item=None, config=None)
         assert isinstance(escaped, str)
         escaped.encode("ascii")
@@ -221,8 +277,6 @@ class TestMetafunc:
         escapes if they're not.
 
         """
-        from _pytest.python import _idval
-
         values = [
             ("", ""),
             ("ascii", "ascii"),
@@ -242,7 +296,6 @@ class TestMetafunc:
         disable_test_id_escaping_and_forfeit_all_rights_to_community_support
         option. (#5294)
         """
-        from _pytest.python import _idval
 
         class MockConfig:
             def __init__(self, config):
@@ -274,8 +327,6 @@ class TestMetafunc:
         "binary escape", where any byte < 127 is escaped into its hex form.
         - python3: bytes objects are always escaped using "binary escape".
         """
-        from _pytest.python import _idval
-
         values = [
             (b"", ""),
             (b"\xc3\xb4\xff\xe4", "\\xc3\\xb4\\xff\\xe4"),
@@ -289,7 +340,6 @@ class TestMetafunc:
         """unittest for the expected behavior to obtain ids for parametrized
         values that are classes or functions: their __name__.
         """
-        from _pytest.python import _idval
 
         class TestClass:
             pass
@@ -534,9 +584,22 @@ class TestMetafunc:
             @pytest.mark.parametrize("arg", ({1: 2}, {3, 4}), ids=ids)
             def test(arg):
                 assert arg
+
+            @pytest.mark.parametrize("arg", (1, 2.0, True), ids=ids)
+            def test_int(arg):
+                assert arg
             """
         )
-        assert testdir.runpytest().ret == 0
+        result = testdir.runpytest("-vv", "-s")
+        result.stdout.fnmatch_lines(
+            [
+                "test_parametrize_ids_returns_non_string.py::test[arg0] PASSED",
+                "test_parametrize_ids_returns_non_string.py::test[arg1] PASSED",
+                "test_parametrize_ids_returns_non_string.py::test_int[1] PASSED",
+                "test_parametrize_ids_returns_non_string.py::test_int[2.0] PASSED",
+                "test_parametrize_ids_returns_non_string.py::test_int[True] PASSED",
+            ]
+        )
 
     def test_idmaker_with_ids(self):
         from _pytest.python import idmaker
@@ -1186,12 +1249,12 @@ class TestMetafuncFunctional:
         result.stdout.fnmatch_lines(["* 1 skipped *"])
 
     def test_parametrized_ids_invalid_type(self, testdir):
-        """Tests parametrized with ids as non-strings (#1857)."""
+        """Test error with non-strings/non-ints, without generator (#1857)."""
         testdir.makepyfile(
             """
             import pytest
 
-            @pytest.mark.parametrize("x, expected", [(10, 20), (40, 80)], ids=(None, 2))
+            @pytest.mark.parametrize("x, expected", [(1, 2), (3, 4), (5, 6)], ids=(None, 2, type))
             def test_ids_numbers(x,expected):
                 assert x * 2 == expected
         """
@@ -1199,7 +1262,8 @@ class TestMetafuncFunctional:
         result = testdir.runpytest()
         result.stdout.fnmatch_lines(
             [
-                "*In test_ids_numbers: ids must be list of strings, found: 2 (type: *'int'>)*"
+                "In test_ids_numbers: ids must be list of string/float/int/bool,"
+                " found: <class 'type'> (type: <class 'type'>) at index 2"
             ]
         )
 
@@ -1780,3 +1844,87 @@ class TestMarkersWithParametrization:
         )
         result = testdir.runpytest()
         result.assert_outcomes(passed=1)
+
+    def test_parametrize_iterator(self, testdir):
+        testdir.makepyfile(
+            """
+            import itertools
+            import pytest
+
+            id_parametrize = pytest.mark.parametrize(
+                ids=("param%d" % i for i in itertools.count())
+            )
+
+            @id_parametrize('y', ['a', 'b'])
+            def test1(y):
+                pass
+
+            @id_parametrize('y', ['a', 'b'])
+            def test2(y):
+                pass
+
+            @pytest.mark.parametrize("a, b", [(1, 2), (3, 4)], ids=itertools.count())
+            def test_converted_to_str(a, b):
+                pass
+        """
+        )
+        result = testdir.runpytest("-vv", "-s")
+        result.stdout.fnmatch_lines(
+            [
+                "test_parametrize_iterator.py::test1[param0] PASSED",
+                "test_parametrize_iterator.py::test1[param1] PASSED",
+                "test_parametrize_iterator.py::test2[param0] PASSED",
+                "test_parametrize_iterator.py::test2[param1] PASSED",
+                "test_parametrize_iterator.py::test_converted_to_str[0] PASSED",
+                "test_parametrize_iterator.py::test_converted_to_str[1] PASSED",
+                "*= 6 passed in *",
+            ]
+        )
+
+    def test_parametrize_explicit_parameters_func(self, testdir):
+        testdir.makepyfile(
+            """
+            import pytest
+
+
+            @pytest.fixture
+            def fixture(arg):
+                return arg
+
+            @pytest.mark.parametrize("arg", ["baz"])
+            def test_without_arg(fixture):
+                assert "baz" == fixture
+        """
+        )
+        result = testdir.runpytest()
+        result.assert_outcomes(error=1)
+        result.stdout.fnmatch_lines(
+            [
+                '*In function "test_without_arg"*',
+                '*Parameter "arg" should be declared explicitly via indirect or in function itself*',
+            ]
+        )
+
+    def test_parametrize_explicit_parameters_method(self, testdir):
+        testdir.makepyfile(
+            """
+            import pytest
+
+            class Test:
+                @pytest.fixture
+                def test_fixture(self, argument):
+                    return argument
+
+                @pytest.mark.parametrize("argument", ["foobar"])
+                def test_without_argument(self, test_fixture):
+                    assert "foobar" == test_fixture
+        """
+        )
+        result = testdir.runpytest()
+        result.assert_outcomes(error=1)
+        result.stdout.fnmatch_lines(
+            [
+                '*In function "test_without_argument"*',
+                '*Parameter "argument" should be declared explicitly via indirect or in function itself*',
+            ]
+        )

testing/test_assertion.py

@@ -72,10 +72,19 @@ class TestImportHookInstallation:
         result = testdir.runpytest_subprocess()
         result.stdout.fnmatch_lines(
             [
-                "E * AssertionError: ([[][]], [[][]], [[]<TestReport *>[]])*",
-                "E * assert"
-                " {'failed': 1, 'passed': 0, 'skipped': 0} =="
-                " {'failed': 0, 'passed': 1, 'skipped': 0}",
+                ">       r.assertoutcome(passed=1)",
+                "E       AssertionError: ([[][]], [[][]], [[]<TestReport *>[]])*",
+                "E       assert {'failed': 1,... 'skipped': 0} == {'failed': 0,... 'skipped': 0}",
+                "E         Omitting 1 identical items, use -vv to show",
+                "E         Differing items:",
+                "E         Use -v to get the full diff",
+            ]
+        )
+        # XXX: unstable output.
+        result.stdout.fnmatch_lines_random(
+            [
+                "E         {'failed': 1} != {'failed': 0}",
+                "E         {'passed': 0} != {'passed': 1}",
             ]
         )
 
@@ -316,8 +325,8 @@ class TestAssert_reprcompare:
 
     def test_text_diff(self):
         diff = callequal("spam", "eggs")[1:]
-        assert "- spam" in diff
-        assert "+ eggs" in diff
+        assert "- eggs" in diff
+        assert "+ spam" in diff
 
     def test_text_skipping(self):
         lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs")
@@ -327,15 +336,15 @@ class TestAssert_reprcompare:
 
     def test_text_skipping_verbose(self):
         lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs", verbose=1)
-        assert "- " + "a" * 50 + "spam" in lines
-        assert "+ " + "a" * 50 + "eggs" in lines
+        assert "- " + "a" * 50 + "eggs" in lines
+        assert "+ " + "a" * 50 + "spam" in lines
 
     def test_multiline_text_diff(self):
         left = "foo\nspam\nbar"
         right = "foo\neggs\nbar"
         diff = callequal(left, right)
-        assert "- spam" in diff
-        assert "+ eggs" in diff
+        assert "- eggs" in diff
+        assert "+ spam" in diff
 
     def test_bytes_diff_normal(self):
         """Check special handling for bytes diff (#5260)"""
@@ -354,8 +363,8 @@ class TestAssert_reprcompare:
             "b'spam' == b'eggs'",
             "At index 0 diff: b's' != b'e'",
             "Full diff:",
-            "- b'spam'",
-            "+ b'eggs'",
+            "- b'eggs'",
+            "+ b'spam'",
         ]
 
     def test_list(self):
@@ -370,9 +379,9 @@ class TestAssert_reprcompare:
                 [0, 2],
                 """
                 Full diff:
-                - [0, 1]
+                - [0, 2]
                 ?     ^
-                + [0, 2]
+                + [0, 1]
                 ?     ^
             """,
                 id="lists",
@@ -382,9 +391,9 @@ class TestAssert_reprcompare:
                 {0: 2},
                 """
                 Full diff:
-                - {0: 1}
+                - {0: 2}
                 ?     ^
-                + {0: 2}
+                + {0: 1}
                 ?     ^
             """,
                 id="dicts",
@@ -394,9 +403,9 @@ class TestAssert_reprcompare:
                 {0, 2},
                 """
                 Full diff:
-                - {0, 1}
+                - {0, 2}
                 ?     ^
-                + {0, 2}
+                + {0, 1}
                 ?     ^
             """,
                 id="sets",
@@ -433,7 +442,7 @@ class TestAssert_reprcompare:
             "   'a',",
             "   'b',",
             "   'c',",
-            "+  '" + long_d + "',",
+            "-  '" + long_d + "',",
             "  ]",
         ]
 
@@ -446,7 +455,7 @@ class TestAssert_reprcompare:
             "   'a',",
             "   'b',",
             "   'c',",
-            "-  '" + long_d + "',",
+            "+  '" + long_d + "',",
             "  ]",
         ]
 
@@ -462,10 +471,10 @@ class TestAssert_reprcompare:
             "At index 0 diff: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' != 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'",
             "Full diff:",
             "  [",
-            "-  'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',",
+            "+  'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',",
             "   'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',",
             "   'cccccccccccccccccccccccccccccc',",
-            "+  'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',",
+            "-  'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',",
             "  ]",
         ]
 
@@ -480,28 +489,28 @@ class TestAssert_reprcompare:
             "Left contains 7 more items, first extra item: 'aaaaaaaaaa'",
             "Full diff:",
             "  [",
-            "+  'should not get wrapped',",
-            "-  'a',",
-            "-  'aaaaaaaaaa',",
-            "-  'aaaaaaaaaa',",
-            "-  'aaaaaaaaaa',",
-            "-  'aaaaaaaaaa',",
-            "-  'aaaaaaaaaa',",
-            "-  'aaaaaaaaaa',",
-            "-  'aaaaaaaaaa',",
+            "-  'should not get wrapped',",
+            "+  'a',",
+            "+  'aaaaaaaaaa',",
+            "+  'aaaaaaaaaa',",
+            "+  'aaaaaaaaaa',",
+            "+  'aaaaaaaaaa',",
+            "+  'aaaaaaaaaa',",
+            "+  'aaaaaaaaaa',",
+            "+  'aaaaaaaaaa',",
             "  ]",
         ]
 
     def test_dict_wrap(self):
-        d1 = {"common": 1, "env": {"env1": 1}}
-        d2 = {"common": 1, "env": {"env1": 1, "env2": 2}}
+        d1 = {"common": 1, "env": {"env1": 1, "env2": 2}}
+        d2 = {"common": 1, "env": {"env1": 1}}
 
         diff = callequal(d1, d2, verbose=True)
         assert diff == [
-            "{'common': 1,...: {'env1': 1}} == {'common': 1,...1, 'env2': 2}}",
+            "{'common': 1,...1, 'env2': 2}} == {'common': 1,...: {'env1': 1}}",
             "Omitting 1 identical items, use -vv to show",
             "Differing items:",
-            "{'env': {'env1': 1}} != {'env': {'env1': 1, 'env2': 2}}",
+            "{'env': {'env1': 1, 'env2': 2}} != {'env': {'env1': 1}}",
             "Full diff:",
             "- {'common': 1, 'env': {'env1': 1}}",
             "+ {'common': 1, 'env': {'env1': 1, 'env2': 2}}",
@@ -523,7 +532,7 @@ class TestAssert_reprcompare:
             "   'env': {'sub': {'long_a': '" + long_a + "',",
             "                   'sub1': {'long_a': 'substring that gets wrapped substring '",
             "                                      'that gets wrapped '}}},",
-            "+  'new': 1,",
+            "-  'new': 1,",
             "  }",
         ]
 
@@ -561,8 +570,8 @@ class TestAssert_reprcompare:
             "Right contains 2 more items:",
             "{'b': 1, 'c': 2}",
             "Full diff:",
-            "- {'a': 0}",
-            "+ {'b': 1, 'c': 2}",
+            "- {'b': 1, 'c': 2}",
+            "+ {'a': 0}",
         ]
         lines = callequal({"b": 1, "c": 2}, {"a": 0}, verbose=2)
         assert lines == [
@@ -572,8 +581,8 @@ class TestAssert_reprcompare:
             "Right contains 1 more item:",
             "{'a': 0}",
             "Full diff:",
-            "- {'b': 1, 'c': 2}",
-            "+ {'a': 0}",
+            "- {'a': 0}",
+            "+ {'b': 1, 'c': 2}",
         ]
 
     def test_sequence_different_items(self):
@@ -583,8 +592,8 @@ class TestAssert_reprcompare:
             "At index 0 diff: 1 != 3",
             "Right contains one more item: 5",
             "Full diff:",
-            "- (1, 2)",
-            "+ (3, 4, 5)",
+            "- (3, 4, 5)",
+            "+ (1, 2)",
         ]
         lines = callequal((1, 2, 3), (4,), verbose=2)
         assert lines == [
@@ -592,8 +601,8 @@ class TestAssert_reprcompare:
             "At index 0 diff: 1 != 4",
             "Left contains 2 more items, first extra item: 2",
             "Full diff:",
-            "- (1, 2, 3)",
-            "+ (4,)",
+            "- (4,)",
+            "+ (1, 2, 3)",
         ]
 
     def test_set(self):
@@ -654,12 +663,12 @@ class TestAssert_reprcompare:
         assert callequal(nums_x, nums_y) is None
 
         expl = callequal(nums_x, nums_y, verbose=1)
-        assert "-" + repr(nums_x) in expl
-        assert "+" + repr(nums_y) in expl
+        assert "+" + repr(nums_x) in expl
+        assert "-" + repr(nums_y) in expl
 
         expl = callequal(nums_x, nums_y, verbose=2)
-        assert "-" + repr(nums_x) in expl
-        assert "+" + repr(nums_y) in expl
+        assert "+" + repr(nums_x) in expl
+        assert "-" + repr(nums_y) in expl
 
     def test_list_bad_repr(self):
         class A:
@@ -668,8 +677,16 @@ class TestAssert_reprcompare:
 
         expl = callequal([], [A()])
         assert "ValueError" in "".join(expl)
-        expl = callequal({}, {"1": A()})
-        assert "faulty" in "".join(expl)
+        expl = callequal({}, {"1": A()}, verbose=2)
+        assert expl[0].startswith("{} == <[ValueError")
+        assert "raised in repr" in expl[0]
+        assert expl[1:] == [
+            "(pytest_assertion plugin: representation of details failed:"
+            " {}:{}: ValueError: 42.".format(
+                __file__, A.__repr__.__code__.co_firstlineno + 1
+            ),
+            " Probably an object has a faulty __repr__.)",
+        ]
 
     def test_one_repr_empty(self):
         """
@@ -693,8 +710,8 @@ class TestAssert_reprcompare:
         right = "£"
         expl = callequal(left, right)
         assert expl[0] == "'£€' == '£'"
-        assert expl[1] == "- £€"
-        assert expl[2] == "+ £"
+        assert expl[1] == "- £"
+        assert expl[2] == "+ £€"
 
     def test_nonascii_text(self):
         """
@@ -707,7 +724,7 @@ class TestAssert_reprcompare:
                 return "\xff"
 
         expl = callequal(A(), "1")
-        assert expl == ["ÿ == '1'", "+ 1"]
+        assert expl == ["ÿ == '1'", "- 1"]
 
     def test_format_nonascii_explanation(self):
         assert util.format_explanation("λ")
@@ -1007,9 +1024,9 @@ class TestTruncateExplanation:
         # without -vv, truncate the message showing a few diff lines only
         result.stdout.fnmatch_lines(
             [
-                "*- 1*",
-                "*- 3*",
-                "*- 5*",
+                "*+ 1*",
+                "*+ 3*",
+                "*+ 5*",
                 "*truncated (%d lines hidden)*use*-vv*" % expected_truncated_lines,
             ]
         )
@@ -1062,9 +1079,9 @@ def test_reprcompare_whitespaces():
     assert detail == [
         r"'\r\n' == '\n'",
         r"Strings contain only whitespace, escaping them using repr()",
-        r"- '\r\n'",
-        r"?  --",
-        r"+ '\n'",
+        r"- '\n'",
+        r"+ '\r\n'",
+        r"?  ++",
     ]
 
 
@@ -1312,8 +1329,8 @@ def test_diff_newline_at_end(testdir):
         r"""
         *assert 'asdf' == 'asdf\n'
         *  - asdf
+        *  ?     -
         *  + asdf
-        *  ?     +
     """
     )
 

testing/test_assertrewrite.py

@@ -21,7 +21,7 @@ from _pytest.assertion.rewrite import get_cache_dir
 from _pytest.assertion.rewrite import PYC_TAIL
 from _pytest.assertion.rewrite import PYTEST_TAG
 from _pytest.assertion.rewrite import rewrite_asserts
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 from _pytest.pathlib import Path
 
 
@@ -180,8 +180,8 @@ class TestAssertionRewrite:
         if verbose > 0:
             assert msg == (
                 "assert <module 'sys' (built-in)> == 42\n"
-                "  -<module 'sys' (built-in)>\n"
-                "  +42"
+                "  +<module 'sys' (built-in)>\n"
+                "  -42"
             )
         else:
             assert msg == "assert sys == 42"
@@ -194,12 +194,12 @@ class TestAssertionRewrite:
 
         msg = getmsg(f, {"cls": X}).splitlines()
         if verbose > 1:
-            assert msg == ["assert {!r} == 42".format(X), "  -{!r}".format(X), "  +42"]
+            assert msg == ["assert {!r} == 42".format(X), "  +{!r}".format(X), "  -42"]
         elif verbose > 0:
             assert msg == [
                 "assert <class 'test_...e.<locals>.X'> == 42",
-                "  -{!r}".format(X),
-                "  +42",
+                "  +{!r}".format(X),
+                "  -42",
             ]
         else:
             assert msg == ["assert cls == 42"]
@@ -241,7 +241,7 @@ class TestAssertionRewrite:
         # XXX: looks like the "where" should also be there in verbose mode?!
         message = getmsg(f, {"cls": Y}).splitlines()
         if request.config.getoption("verbose") > 0:
-            assert message == ["assert 3 == 2", "  -3", "  +2"]
+            assert message == ["assert 3 == 2", "  +3", "  -2"]
         else:
             assert message == [
                 "assert 3 == 2",
@@ -625,7 +625,7 @@ class TestAssertionRewrite:
 
         msg = getmsg(f)
         if request.config.getoption("verbose") > 0:
-            assert msg == "assert 10 == 11\n  -10\n  +11"
+            assert msg == "assert 10 == 11\n  +10\n  -11"
         else:
             assert msg == "assert 10 == 11\n +  where 10 = len([0, 1, 2, 3, 4, 5, ...])"
 
@@ -688,7 +688,7 @@ class TestAssertionRewrite:
 
         lines = util._format_lines([getmsg(f)])
         if request.config.getoption("verbose") > 0:
-            assert lines == ["assert 0 == 1\n  -0\n  +1"]
+            assert lines == ["assert 0 == 1\n  +0\n  -1"]
         else:
             assert lines == ["assert 0 == 1\n +  where 1 = \\n{ \\n~ \\n}.a"]
 

testing/test_cacheprovider.py

@@ -6,7 +6,7 @@ import sys
 import py
 
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 
 pytest_plugins = ("pytester",)
 
@@ -56,9 +56,7 @@ class TestNewAPI:
             testdir.tmpdir.ensure_dir(".pytest_cache").chmod(mode)
 
     @pytest.mark.skipif(sys.platform.startswith("win"), reason="no chmod on windows")
-    @pytest.mark.filterwarnings(
-        "ignore:could not create cache path:pytest.PytestWarning"
-    )
+    @pytest.mark.filterwarnings("default")
     def test_cache_failure_warns(self, testdir, monkeypatch):
         monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1")
         cache_dir = str(testdir.tmpdir.ensure_dir(".pytest_cache"))
@@ -70,7 +68,15 @@ class TestNewAPI:
             assert result.ret == 1
             # warnings from nodeids, lastfailed, and stepwise
             result.stdout.fnmatch_lines(
-                ["*could not create cache path*", "*3 warnings*"]
+                [
+                    # Validate location/stacklevel of warning from cacheprovider.
+                    "*= warnings summary =*",
+                    "*/cacheprovider.py:314",
+                    "  */cacheprovider.py:314: PytestCacheWarning: could not create cache path "
+                    "{}/v/cache/nodeids".format(cache_dir),
+                    '    config.cache.set("cache/nodeids", self.cached_nodeids)',
+                    "*1 failed, 3 warnings in*",
+                ]
             )
         finally:
             testdir.tmpdir.ensure_dir(".pytest_cache").chmod(mode)
@@ -683,11 +689,28 @@ class TestLastFailed:
         result.stdout.fnmatch_lines(["*2 passed*"])
         result = testdir.runpytest("--lf", "--lfnf", "all")
         result.stdout.fnmatch_lines(["*2 passed*"])
+
+        # Ensure the list passed to pytest_deselected is a copy,
+        # and not a reference which is cleared right after.
+        testdir.makeconftest(
+            """
+            deselected = []
+
+            def pytest_deselected(items):
+                global deselected
+                deselected = items
+
+            def pytest_sessionfinish():
+                print("\\ndeselected={}".format(len(deselected)))
+        """
+        )
+
         result = testdir.runpytest("--lf", "--lfnf", "none")
         result.stdout.fnmatch_lines(
             [
                 "collected 2 items / 2 deselected",
                 "run-last-failure: no previously failed tests, deselecting all items.",
+                "deselected=2",
                 "* 2 deselected in *",
             ]
         )

testing/test_capture.py

@@ -7,13 +7,15 @@ import sys
 import textwrap
 from io import StringIO
 from io import UnsupportedOperation
+from typing import BinaryIO
+from typing import Generator
 from typing import List
 from typing import TextIO
 
 import pytest
 from _pytest import capture
 from _pytest.capture import CaptureManager
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 
 # note: py.io capture tests where copied from
 # pylib 1.4.20.dev2 (rev 13d9af95547e)
@@ -32,6 +34,10 @@ def StdCapture(out=True, err=True, in_=True):
     return capture.MultiCapture(out, err, in_, Capture=capture.SysCapture)
 
 
+def TeeStdCapture(out=True, err=True, in_=True):
+    return capture.MultiCapture(out, err, in_, Capture=capture.TeeSysCapture)
+
+
 class TestCaptureManager:
     def test_getmethod_default_no_fd(self, monkeypatch):
         from _pytest.capture import pytest_addoption
@@ -471,9 +477,9 @@ class TestCaptureFixture:
         result.stdout.fnmatch_lines(
             [
                 "*test_one*",
-                "*capsys*capfd*same*time*",
+                "E * cannot use capfd and capsys at the same time",
                 "*test_two*",
-                "*capfd*capsys*same*time*",
+                "E * cannot use capsys and capfd at the same time",
                 "*2 failed in*",
             ]
         )
@@ -816,6 +822,25 @@ class TestCaptureIO:
         assert f.getvalue() == "foo\r\n"
 
 
+class TestCaptureAndPassthroughIO(TestCaptureIO):
+    def test_text(self):
+        sio = io.StringIO()
+        f = capture.CaptureAndPassthroughIO(sio)
+        f.write("hello")
+        s1 = f.getvalue()
+        assert s1 == "hello"
+        s2 = sio.getvalue()
+        assert s2 == s1
+        f.close()
+        sio.close()
+
+    def test_unicode_and_str_mixture(self):
+        sio = io.StringIO()
+        f = capture.CaptureAndPassthroughIO(sio)
+        f.write("\u00f6")
+        pytest.raises(TypeError, f.write, b"hello")
+
+
 def test_dontreadfrominput():
     from _pytest.capture import DontReadFromInput
 
@@ -831,7 +856,7 @@ def test_dontreadfrominput():
 
 
 @pytest.fixture
-def tmpfile(testdir):
+def tmpfile(testdir) -> Generator[BinaryIO, None, None]:
     f = testdir.makepyfile("").open("wb+")
     yield f
     if not f.closed:
@@ -1112,6 +1137,23 @@ class TestStdCapture:
             pytest.raises(IOError, sys.stdin.read)
 
 
+class TestTeeStdCapture(TestStdCapture):
+    captureclass = staticmethod(TeeStdCapture)
+
+    def test_capturing_error_recursive(self):
+        """ for TeeStdCapture since we passthrough stderr/stdout, cap1
+        should get all output, while cap2 should only get "cap2\n" """
+
+        with self.getcapture() as cap1:
+            print("cap1")
+            with self.getcapture() as cap2:
+                print("cap2")
+                out2, err2 = cap2.readouterr()
+                out1, err1 = cap1.readouterr()
+        assert out1 == "cap1\ncap2\n"
+        assert out2 == "cap2\n"
+
+
 class TestStdCaptureFD(TestStdCapture):
     pytestmark = needsosdup
     captureclass = staticmethod(StdCaptureFD)
@@ -1252,7 +1294,7 @@ def test_close_and_capture_again(testdir):
     )
 
 
-@pytest.mark.parametrize("method", ["SysCapture", "FDCapture"])
+@pytest.mark.parametrize("method", ["SysCapture", "FDCapture", "TeeSysCapture"])
 def test_capturing_and_logging_fundamentals(testdir, method):
     if method == "StdCaptureFD" and not hasattr(os, "dup"):
         pytest.skip("need os.dup")
@@ -1497,3 +1539,15 @@ def test_typeerror_encodedfile_write(testdir):
 def test_stderr_write_returns_len(capsys):
     """Write on Encoded files, namely captured stderr, should return number of characters written."""
     assert sys.stderr.write("Foo") == 3
+
+
+def test_encodedfile_writelines(tmpfile: BinaryIO) -> None:
+    ef = capture.EncodedFile(tmpfile, "utf-8")
+    with pytest.raises(AttributeError):
+        ef.writelines([b"line1", b"line2"])  # type: ignore[list-item]  # noqa: F821
+    assert ef.writelines(["line1", "line2"]) is None  # type: ignore[func-returns-value]  # noqa: F821
+    tmpfile.seek(0)
+    assert tmpfile.read() == b"line1line2"
+    tmpfile.close()
+    with pytest.raises(ValueError):
+        ef.read()

testing/test_collection.py

@@ -6,9 +6,10 @@ import textwrap
 import py
 
 import pytest
+from _pytest.config import ExitCode
 from _pytest.main import _in_venv
-from _pytest.main import ExitCode
 from _pytest.main import Session
+from _pytest.pytester import Testdir
 
 
 class TestCollector:
@@ -18,7 +19,7 @@ class TestCollector:
         assert not issubclass(Collector, Item)
         assert not issubclass(Item, Collector)
 
-    def test_check_equality(self, testdir):
+    def test_check_equality(self, testdir: Testdir) -> None:
         modcol = testdir.getmodulecol(
             """
             def test_pass(): pass
@@ -40,12 +41,15 @@ class TestCollector:
         assert fn1 != fn3
 
         for fn in fn1, fn2, fn3:
-            assert fn != 3
+            assert isinstance(fn, pytest.Function)
+            assert fn != 3  # type: ignore[comparison-overlap]  # noqa: F821
             assert fn != modcol
-            assert fn != [1, 2, 3]
-            assert [1, 2, 3] != fn
+            assert fn != [1, 2, 3]  # type: ignore[comparison-overlap]  # noqa: F821
+            assert [1, 2, 3] != fn  # type: ignore[comparison-overlap]  # noqa: F821
             assert modcol != fn
 
+        assert testdir.collect_by_name(modcol, "doesnotexist") is None
+
     def test_getparent(self, testdir):
         modcol = testdir.getmodulecol(
             """
@@ -75,7 +79,7 @@ class TestCollector:
                 pass
             def pytest_collect_file(path, parent):
                 if path.ext == ".xxx":
-                    return CustomFile(path, parent=parent)
+                    return CustomFile.from_parent(fspath=path, parent=parent)
         """
         )
         node = testdir.getpathnode(hello)
@@ -438,7 +442,7 @@ class TestCustomConftests:
 
 
 class TestSession:
-    def test_parsearg(self, testdir):
+    def test_parsearg(self, testdir) -> None:
         p = testdir.makepyfile("def test_func(): pass")
         subdir = testdir.mkdir("sub")
         subdir.ensure("__init__.py")
@@ -446,16 +450,16 @@ class TestSession:
         p.move(target)
         subdir.chdir()
         config = testdir.parseconfig(p.basename)
-        rcol = Session(config=config)
+        rcol = Session.from_config(config)
         assert rcol.fspath == subdir
-        parts = rcol._parsearg(p.basename)
+        fspath, parts = rcol._parsearg(p.basename)
 
-        assert parts[0] == target
+        assert fspath == target
+        assert len(parts) == 0
+        fspath, parts = rcol._parsearg(p.basename + "::test_func")
+        assert fspath == target
+        assert parts[0] == "test_func"
         assert len(parts) == 1
-        parts = rcol._parsearg(p.basename + "::test_func")
-        assert parts[0] == target
-        assert parts[1] == "test_func"
-        assert len(parts) == 2
 
     def test_collect_topdir(self, testdir):
         p = testdir.makepyfile("def test_func(): pass")
@@ -463,7 +467,7 @@ class TestSession:
         # XXX migrate to collectonly? (see below)
         config = testdir.parseconfig(id)
         topdir = testdir.tmpdir
-        rcol = Session(config)
+        rcol = Session.from_config(config)
         assert topdir == rcol.fspath
         # rootid = rcol.nodeid
         # root2 = rcol.perform_collect([rcol.nodeid], genitems=False)[0]
@@ -809,6 +813,43 @@ class TestNodekeywords:
         reprec = testdir.inline_run("-k repr")
         reprec.assertoutcome(passed=1, failed=0)
 
+    def test_keyword_matching_is_case_insensitive_by_default(self, testdir):
+        """Check that selection via -k EXPRESSION is case-insensitive.
+
+        Since markers are also added to the node keywords, they too can
+        be matched without having to think about case sensitivity.
+
+        """
+        testdir.makepyfile(
+            """
+            import pytest
+
+            def test_sPeCiFiCToPiC_1():
+                assert True
+
+            class TestSpecificTopic_2:
+                def test(self):
+                    assert True
+
+            @pytest.mark.sPeCiFiCToPic_3
+            def test():
+                assert True
+
+            @pytest.mark.sPeCiFiCToPic_4
+            class Test:
+                def test(self):
+                    assert True
+
+            def test_failing_5():
+                assert False, "This should not match"
+
+        """
+        )
+        num_matching_tests = 4
+        for expression in ("specifictopic", "SPECIFICTOPIC", "SpecificTopic"):
+            reprec = testdir.inline_run("-k " + expression)
+            reprec.assertoutcome(passed=num_matching_tests, failed=0)
+
 
 COLLECTION_ERROR_PY_FILES = dict(
     test_01_failure="""

testing/test_config.py

@@ -7,11 +7,11 @@ import pytest
 from _pytest.compat import importlib_metadata
 from _pytest.config import _iter_rewritable_modules
 from _pytest.config import Config
+from _pytest.config import ExitCode
 from _pytest.config.exceptions import UsageError
 from _pytest.config.findpaths import determine_setup
 from _pytest.config.findpaths import get_common_ancestor
 from _pytest.config.findpaths import getcfg
-from _pytest.main import ExitCode
 from _pytest.pathlib import Path
 
 
@@ -659,6 +659,13 @@ def test_disable_plugin_autoload(testdir, monkeypatch, parse_args, should_load):
     class PseudoPlugin:
         x = 42
 
+        attrs_used = []
+
+        def __getattr__(self, name):
+            assert name == "__loader__"
+            self.attrs_used.append(name)
+            return object()
+
     def distributions():
         return (Distribution(),)
 
@@ -668,6 +675,36 @@ def test_disable_plugin_autoload(testdir, monkeypatch, parse_args, should_load):
     config = testdir.parseconfig(*parse_args)
     has_loaded = config.pluginmanager.get_plugin("mytestplugin") is not None
     assert has_loaded == should_load
+    if should_load:
+        assert PseudoPlugin.attrs_used == ["__loader__"]
+    else:
+        assert PseudoPlugin.attrs_used == []
+
+
+def test_plugin_loading_order(testdir):
+    """Test order of plugin loading with `-p`."""
+    p1 = testdir.makepyfile(
+        """
+        def test_terminal_plugin(request):
+            import myplugin
+            assert myplugin.terminal_plugin == [False, True]
+        """,
+        **{
+            "myplugin": """
+            terminal_plugin = []
+
+            def pytest_configure(config):
+                terminal_plugin.append(bool(config.pluginmanager.get_plugin("terminalreporter")))
+
+            def pytest_sessionstart(session):
+                config = session.config
+                terminal_plugin.append(bool(config.pluginmanager.get_plugin("terminalreporter")))
+            """
+        }
+    )
+    testdir.syspathinsert()
+    result = testdir.runpytest("-p", "myplugin", str(p1))
+    assert result.ret == 0
 
 
 def test_cmdline_processargs_simple(testdir):
@@ -802,7 +839,7 @@ def test_load_initial_conftest_last_ordering(_config_for_test):
     pm.register(m)
     hc = pm.hook.pytest_load_initial_conftests
     values = hc._nonwrappers + hc._wrappers
-    expected = ["_pytest.config", "test_config", "_pytest.capture"]
+    expected = ["_pytest.config", m.__module__, "_pytest.capture"]
     assert [x.function.__module__ for x in values] == expected
 
 
@@ -1097,7 +1134,7 @@ class TestOverrideIniArgs:
                 % (testdir.request.config._parser.optparser.prog,)
             ]
         )
-        assert result.ret == _pytest.main.ExitCode.USAGE_ERROR
+        assert result.ret == _pytest.config.ExitCode.USAGE_ERROR
 
     def test_override_ini_does_not_contain_paths(self, _config_for_test, _sys_snapshot):
         """Check that -o no longer swallows all options after it (#3103)"""

testing/test_conftest.py

@@ -4,8 +4,8 @@ import textwrap
 import py
 
 import pytest
+from _pytest.config import ExitCode
 from _pytest.config import PytestPluginManager
-from _pytest.main import ExitCode
 from _pytest.pathlib import Path
 
 

testing/test_debugging.py

@@ -22,7 +22,7 @@ def pdb_env(request):
     if "testdir" in request.fixturenames:
         # Disable pdb++ with inner tests.
         testdir = request.getfixturevalue("testdir")
-        testdir._env_run_update["PDBPP_HIJACK_PDB"] = "0"
+        testdir.monkeypatch.setenv("PDBPP_HIJACK_PDB", "0")
 
 
 def runpdb_and_get_report(testdir, source):
@@ -193,7 +193,7 @@ class TestPDB:
         )
         child = testdir.spawn_pytest("-rs --pdb %s" % p1)
         child.expect("Skipping also with pdb active")
-        child.expect_exact("= \x1b[33m\x1b[1m1 skipped\x1b[0m\x1b[33m in")
+        child.expect_exact("= 1 skipped in")
         child.sendeof()
         self.flush(child)
 
@@ -221,7 +221,7 @@ class TestPDB:
         child.sendeof()
         rest = child.read().decode("utf8")
         assert "Exit: Quitting debugger" in rest
-        assert "= \x1b[31m\x1b[1m1 failed\x1b[0m\x1b[31m in" in rest
+        assert "= 1 failed in" in rest
         assert "def test_1" not in rest
         assert "get rekt" not in rest
         self.flush(child)
@@ -506,7 +506,7 @@ class TestPDB:
         rest = child.read().decode("utf8")
 
         assert "! _pytest.outcomes.Exit: Quitting debugger !" in rest
-        assert "= \x1b[33mno tests ran\x1b[0m\x1b[32m in" in rest
+        assert "= no tests ran in" in rest
         assert "BdbQuit" not in rest
         assert "UNEXPECTED EXCEPTION" not in rest
 
@@ -725,7 +725,7 @@ class TestPDB:
             assert "> PDB continue (IO-capturing resumed) >" in rest
         else:
             assert "> PDB continue >" in rest
-        assert "= \x1b[32m\x1b[1m1 passed\x1b[0m\x1b[32m in" in rest
+        assert "= 1 passed in" in rest
 
     def test_pdb_used_outside_test(self, testdir):
         p1 = testdir.makepyfile(
@@ -1041,7 +1041,7 @@ class TestTraceOption:
         child.sendline("q")
         child.expect_exact("Exit: Quitting debugger")
         rest = child.read().decode("utf8")
-        assert "= \x1b[32m\x1b[1m2 passed\x1b[0m\x1b[32m in" in rest
+        assert "= 2 passed in" in rest
         assert "reading from stdin while output" not in rest
         # Only printed once - not on stderr.
         assert "Exit: Quitting debugger" not in child.before.decode("utf8")
@@ -1086,7 +1086,7 @@ class TestTraceOption:
             child.sendline("c")
             child.expect_exact("> PDB continue (IO-capturing resumed) >")
         rest = child.read().decode("utf8")
-        assert "= \x1b[32m\x1b[1m6 passed\x1b[0m\x1b[32m in" in rest
+        assert "= 6 passed in" in rest
         assert "reading from stdin while output" not in rest
         # Only printed once - not on stderr.
         assert "Exit: Quitting debugger" not in child.before.decode("utf8")
@@ -1197,7 +1197,7 @@ def test_pdb_suspends_fixture_capturing(testdir, fixture):
 
     TestPDB.flush(child)
     assert child.exitstatus == 0
-    assert "= \x1b[32m\x1b[1m1 passed\x1b[0m\x1b[32m in" in rest
+    assert "= 1 passed in" in rest
     assert "> PDB continue (IO-capturing resumed for fixture %s) >" % (fixture) in rest
 
 

testing/test_error_diffs.py

@@ -0,0 +1,274 @@
+"""
+Tests and examples for correct "+/-" usage in error diffs.
+
+See https://github.com/pytest-dev/pytest/issues/3333 for details.
+
+"""
+import sys
+
+import pytest
+
+
+TESTCASES = [
+    pytest.param(
+        """
+        def test_this():
+            result =   [1, 4, 3]
+            expected = [1, 2, 3]
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       assert [1, 4, 3] == [1, 2, 3]
+        E         At index 1 diff: 4 != 2
+        E         Full diff:
+        E         - [1, 2, 3]
+        E         ?     ^
+        E         + [1, 4, 3]
+        E         ?     ^
+        """,
+        id="Compare lists, one item differs",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   [1, 2, 3]
+            expected = [1, 2]
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       assert [1, 2, 3] == [1, 2]
+        E         Left contains one more item: 3
+        E         Full diff:
+        E         - [1, 2]
+        E         + [1, 2, 3]
+        E         ?      +++
+        """,
+        id="Compare lists, one extra item",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   [1, 3]
+            expected = [1, 2, 3]
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       assert [1, 3] == [1, 2, 3]
+        E         At index 1 diff: 3 != 2
+        E         Right contains one more item: 3
+        E         Full diff:
+        E         - [1, 2, 3]
+        E         ?     ---
+        E         + [1, 3]
+        """,
+        id="Compare lists, one item missing",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   (1, 4, 3)
+            expected = (1, 2, 3)
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       assert (1, 4, 3) == (1, 2, 3)
+        E         At index 1 diff: 4 != 2
+        E         Full diff:
+        E         - (1, 2, 3)
+        E         ?     ^
+        E         + (1, 4, 3)
+        E         ?     ^
+        """,
+        id="Compare tuples",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   {1, 3, 4}
+            expected = {1, 2, 3}
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       assert {1, 3, 4} == {1, 2, 3}
+        E         Extra items in the left set:
+        E         4
+        E         Extra items in the right set:
+        E         2
+        E         Full diff:
+        E         - {1, 2, 3}
+        E         ?     ^  ^
+        E         + {1, 3, 4}
+        E         ?     ^  ^
+        """,
+        id="Compare sets",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   {1: 'spam', 3: 'eggs'}
+            expected = {1: 'spam', 2: 'eggs'}
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       AssertionError: assert {1: 'spam', 3: 'eggs'} == {1: 'spam', 2: 'eggs'}
+        E         Common items:
+        E         {1: 'spam'}
+        E         Left contains 1 more item:
+        E         {3: 'eggs'}
+        E         Right contains 1 more item:
+        E         {2: 'eggs'}
+        E         Full diff:
+        E         - {1: 'spam', 2: 'eggs'}
+        E         ?             ^
+        E         + {1: 'spam', 3: 'eggs'}
+        E         ?             ^
+        """,
+        id="Compare dicts with differing keys",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   {1: 'spam', 2: 'eggs'}
+            expected = {1: 'spam', 2: 'bacon'}
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       AssertionError: assert {1: 'spam', 2: 'eggs'} == {1: 'spam', 2: 'bacon'}
+        E         Common items:
+        E         {1: 'spam'}
+        E         Differing items:
+        E         {2: 'eggs'} != {2: 'bacon'}
+        E         Full diff:
+        E         - {1: 'spam', 2: 'bacon'}
+        E         ?                 ^^^^^
+        E         + {1: 'spam', 2: 'eggs'}
+        E         ?                 ^^^^
+        """,
+        id="Compare dicts with differing values",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   {1: 'spam', 2: 'eggs'}
+            expected = {1: 'spam', 3: 'bacon'}
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       AssertionError: assert {1: 'spam', 2: 'eggs'} == {1: 'spam', 3: 'bacon'}
+        E         Common items:
+        E         {1: 'spam'}
+        E         Left contains 1 more item:
+        E         {2: 'eggs'}
+        E         Right contains 1 more item:
+        E         {3: 'bacon'}
+        E         Full diff:
+        E         - {1: 'spam', 3: 'bacon'}
+        E         ?             ^   ^^^^^
+        E         + {1: 'spam', 2: 'eggs'}
+        E         ?             ^   ^^^^
+        """,
+        id="Compare dicts with differing items",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   "spmaeggs"
+            expected = "spameggs"
+            assert result == expected
+        """,
+        """
+        >       assert result == expected
+        E       AssertionError: assert 'spmaeggs' == 'spameggs'
+        E         - spameggs
+        E         ?    -
+        E         + spmaeggs
+        E         ?   +
+        """,
+        id="Compare strings",
+    ),
+    pytest.param(
+        """
+        def test_this():
+            result =   "spam bacon eggs"
+            assert "bacon" not in result
+        """,
+        """
+        >       assert "bacon" not in result
+        E       AssertionError: assert 'bacon' not in 'spam bacon eggs'
+        E         'bacon' is contained here:
+        E           spam bacon eggs
+        E         ?      +++++
+        """,
+        id='Test "not in" string',
+    ),
+]
+if sys.version_info[:2] >= (3, 7):
+    TESTCASES.extend(
+        [
+            pytest.param(
+                """
+                from dataclasses import dataclass
+
+                @dataclass
+                class A:
+                    a: int
+                    b: str
+
+                def test_this():
+                    result =   A(1, 'spam')
+                    expected = A(2, 'spam')
+                    assert result == expected
+                """,
+                """
+                >       assert result == expected
+                E       AssertionError: assert A(a=1, b='spam') == A(a=2, b='spam')
+                E         Matching attributes:
+                E         ['b']
+                E         Differing attributes:
+                E         a: 1 != 2
+                """,
+                id="Compare data classes",
+            ),
+            pytest.param(
+                """
+                import attr
+
+                @attr.s(auto_attribs=True)
+                class A:
+                    a: int
+                    b: str
+
+                def test_this():
+                    result =   A(1, 'spam')
+                    expected = A(1, 'eggs')
+                    assert result == expected
+                """,
+                """
+                >       assert result == expected
+                E       AssertionError: assert A(a=1, b='spam') == A(a=1, b='eggs')
+                E         Matching attributes:
+                E         ['a']
+                E         Differing attributes:
+                E         b: 'spam' != 'eggs'
+                """,
+                id="Compare attrs classes",
+            ),
+        ]
+    )
+
+
+@pytest.mark.parametrize("code, expected", TESTCASES)
+def test_error_diff(code, expected, testdir):
+    expected = [l.lstrip() for l in expected.splitlines()]
+    p = testdir.makepyfile(code)
+    result = testdir.runpytest(p, "-vv")
+    result.stdout.fnmatch_lines(expected)
+    assert result.ret == 1

testing/test_faulthandler.py

@@ -88,7 +88,7 @@ def test_cancel_timeout_on_hook(monkeypatch, hook_name):
     exception (pytest-dev/pytest-faulthandler#14).
     """
     import faulthandler
-    from _pytest import faulthandler as plugin_module
+    from _pytest.faulthandler import FaultHandlerHooks
 
     called = []
 
@@ -98,6 +98,35 @@ def test_cancel_timeout_on_hook(monkeypatch, hook_name):
 
     # call our hook explicitly, we can trust that pytest will call the hook
     # for us at the appropriate moment
-    hook_func = getattr(plugin_module, hook_name)
-    hook_func()
+    hook_func = getattr(FaultHandlerHooks, hook_name)
+    hook_func(self=None)
     assert called == [1]
+
+
+@pytest.mark.parametrize("faulthandler_timeout", [0, 2])
+def test_already_initialized(faulthandler_timeout, testdir):
+    """Test for faulthandler being initialized earlier than pytest (#6575)"""
+    testdir.makepyfile(
+        """
+        def test():
+            import faulthandler
+            assert faulthandler.is_enabled()
+    """
+    )
+    result = testdir.run(
+        sys.executable,
+        "-X",
+        "faulthandler",
+        "-mpytest",
+        testdir.tmpdir,
+        "-o",
+        "faulthandler_timeout={}".format(faulthandler_timeout),
+    )
+    # ensure warning is emitted if faulthandler_timeout is configured
+    warning_line = "*faulthandler.py*faulthandler module enabled before*"
+    if faulthandler_timeout > 0:
+        result.stdout.fnmatch_lines(warning_line)
+    else:
+        result.stdout.no_fnmatch_line(warning_line)
+    result.stdout.fnmatch_lines("*1 passed*")
+    assert result.ret == 0

testing/test_helpconfig.py

@@ -1,8 +1,9 @@
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 
 
 def test_version(testdir, pytestconfig):
+    testdir.monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
     result = testdir.runpytest("--version")
     assert result.ret == 0
     # p = py.path.local(py.__file__).dirpath()

testing/test_junitxml.py

@@ -445,7 +445,9 @@ class TestPython:
         fnode.assert_attr(message="internal error")
         assert "Division" in fnode.toxml()
 
-    @pytest.mark.parametrize("junit_logging", ["no", "system-out", "system-err"])
+    @pytest.mark.parametrize(
+        "junit_logging", ["no", "log", "system-out", "system-err", "out-err", "all"]
+    )
     @parametrize_families
     def test_failure_function(
         self, testdir, junit_logging, run_and_parse, xunit_family
@@ -467,35 +469,48 @@ class TestPython:
         result, dom = run_and_parse(
             "-o", "junit_logging=%s" % junit_logging, family=xunit_family
         )
-        assert result.ret
+        assert result.ret, "Expected ret > 0"
         node = dom.find_first_by_tag("testsuite")
         node.assert_attr(failures=1, tests=1)
         tnode = node.find_first_by_tag("testcase")
         tnode.assert_attr(classname="test_failure_function", name="test_fail")
         fnode = tnode.find_first_by_tag("failure")
         fnode.assert_attr(message="ValueError: 42")
-        assert "ValueError" in fnode.toxml()
-        systemout = fnode.next_sibling
-        assert systemout.tag == "system-out"
-        systemout_xml = systemout.toxml()
-        assert "hello-stdout" in systemout_xml
-        assert "info msg" not in systemout_xml
-        systemerr = systemout.next_sibling
-        assert systemerr.tag == "system-err"
-        systemerr_xml = systemerr.toxml()
-        assert "hello-stderr" in systemerr_xml
-        assert "info msg" not in systemerr_xml
+        assert "ValueError" in fnode.toxml(), "ValueError not included"
 
-        if junit_logging == "system-out":
-            assert "warning msg" in systemout_xml
-            assert "warning msg" not in systemerr_xml
-        elif junit_logging == "system-err":
-            assert "warning msg" not in systemout_xml
-            assert "warning msg" in systemerr_xml
-        else:
-            assert junit_logging == "no"
-            assert "warning msg" not in systemout_xml
-            assert "warning msg" not in systemerr_xml
+        if junit_logging in ["log", "all"]:
+            logdata = tnode.find_first_by_tag("system-out")
+            log_xml = logdata.toxml()
+            assert logdata.tag == "system-out", "Expected tag: system-out"
+            assert "info msg" not in log_xml, "Unexpected INFO message"
+            assert "warning msg" in log_xml, "Missing WARN message"
+        if junit_logging in ["system-out", "out-err", "all"]:
+            systemout = tnode.find_first_by_tag("system-out")
+            systemout_xml = systemout.toxml()
+            assert systemout.tag == "system-out", "Expected tag: system-out"
+            assert "info msg" not in systemout_xml, "INFO message found in system-out"
+            assert (
+                "hello-stdout" in systemout_xml
+            ), "Missing 'hello-stdout' in system-out"
+        if junit_logging in ["system-err", "out-err", "all"]:
+            systemerr = tnode.find_first_by_tag("system-err")
+            systemerr_xml = systemerr.toxml()
+            assert systemerr.tag == "system-err", "Expected tag: system-err"
+            assert "info msg" not in systemerr_xml, "INFO message found in system-err"
+            assert (
+                "hello-stderr" in systemerr_xml
+            ), "Missing 'hello-stderr' in system-err"
+            assert (
+                "warning msg" not in systemerr_xml
+            ), "WARN message found in system-err"
+        if junit_logging == "no":
+            assert not tnode.find_by_tag("log"), "Found unexpected content: log"
+            assert not tnode.find_by_tag(
+                "system-out"
+            ), "Found unexpected content: system-out"
+            assert not tnode.find_by_tag(
+                "system-err"
+            ), "Found unexpected content: system-err"
 
     @parametrize_families
     def test_failure_verbose_message(self, testdir, run_and_parse, xunit_family):
@@ -523,7 +538,9 @@ class TestPython:
                 assert 0
         """
         )
-        result, dom = run_and_parse(family=xunit_family)
+        result, dom = run_and_parse(
+            "-o", "junit_logging=system-out", family=xunit_family
+        )
         assert result.ret
         node = dom.find_first_by_tag("testsuite")
         node.assert_attr(failures=3, tests=3)
@@ -536,7 +553,7 @@ class TestPython:
             )
             sysout = tnode.find_first_by_tag("system-out")
             text = sysout.text
-            assert text == "%s\n" % char
+            assert "%s\n" % char in text
 
     @parametrize_families
     def test_junit_prefixing(self, testdir, run_and_parse, xunit_family):
@@ -597,7 +614,10 @@ class TestPython:
         fnode = tnode.find_first_by_tag("skipped")
         fnode.assert_attr(type="pytest.xfail", message="42")
 
-    def test_xfail_captures_output_once(self, testdir, run_and_parse):
+    @pytest.mark.parametrize(
+        "junit_logging", ["no", "log", "system-out", "system-err", "out-err", "all"]
+    )
+    def test_xfail_captures_output_once(self, testdir, junit_logging, run_and_parse):
         testdir.makepyfile(
             """
             import sys
@@ -610,11 +630,18 @@ class TestPython:
                 assert 0
         """
         )
-        result, dom = run_and_parse()
+        result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging)
         node = dom.find_first_by_tag("testsuite")
         tnode = node.find_first_by_tag("testcase")
-        assert len(tnode.find_by_tag("system-err")) == 1
-        assert len(tnode.find_by_tag("system-out")) == 1
+        if junit_logging in ["system-err", "out-err", "all"]:
+            assert len(tnode.find_by_tag("system-err")) == 1
+        else:
+            assert len(tnode.find_by_tag("system-err")) == 0
+
+        if junit_logging in ["log", "system-out", "out-err", "all"]:
+            assert len(tnode.find_by_tag("system-out")) == 1
+        else:
+            assert len(tnode.find_by_tag("system-out")) == 0
 
     @parametrize_families
     def test_xfailure_xpass(self, testdir, run_and_parse, xunit_family):
@@ -696,20 +723,29 @@ class TestPython:
         result, dom = run_and_parse()
         print(dom.toxml())
 
-    def test_pass_captures_stdout(self, testdir, run_and_parse):
+    @pytest.mark.parametrize("junit_logging", ["no", "system-out"])
+    def test_pass_captures_stdout(self, testdir, run_and_parse, junit_logging):
         testdir.makepyfile(
             """
             def test_pass():
                 print('hello-stdout')
         """
         )
-        result, dom = run_and_parse()
+        result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging)
         node = dom.find_first_by_tag("testsuite")
         pnode = node.find_first_by_tag("testcase")
-        systemout = pnode.find_first_by_tag("system-out")
-        assert "hello-stdout" in systemout.toxml()
+        if junit_logging == "no":
+            assert not node.find_by_tag(
+                "system-out"
+            ), "system-out should not be generated"
+        if junit_logging == "system-out":
+            systemout = pnode.find_first_by_tag("system-out")
+            assert (
+                "hello-stdout" in systemout.toxml()
+            ), "'hello-stdout' should be in system-out"
 
-    def test_pass_captures_stderr(self, testdir, run_and_parse):
+    @pytest.mark.parametrize("junit_logging", ["no", "system-err"])
+    def test_pass_captures_stderr(self, testdir, run_and_parse, junit_logging):
         testdir.makepyfile(
             """
             import sys
@@ -717,13 +753,21 @@ class TestPython:
                 sys.stderr.write('hello-stderr')
         """
         )
-        result, dom = run_and_parse()
+        result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging)
         node = dom.find_first_by_tag("testsuite")
         pnode = node.find_first_by_tag("testcase")
-        systemout = pnode.find_first_by_tag("system-err")
-        assert "hello-stderr" in systemout.toxml()
+        if junit_logging == "no":
+            assert not node.find_by_tag(
+                "system-err"
+            ), "system-err should not be generated"
+        if junit_logging == "system-err":
+            systemerr = pnode.find_first_by_tag("system-err")
+            assert (
+                "hello-stderr" in systemerr.toxml()
+            ), "'hello-stderr' should be in system-err"
 
-    def test_setup_error_captures_stdout(self, testdir, run_and_parse):
+    @pytest.mark.parametrize("junit_logging", ["no", "system-out"])
+    def test_setup_error_captures_stdout(self, testdir, run_and_parse, junit_logging):
         testdir.makepyfile(
             """
             import pytest
@@ -736,13 +780,21 @@ class TestPython:
                 pass
         """
         )
-        result, dom = run_and_parse()
+        result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging)
         node = dom.find_first_by_tag("testsuite")
         pnode = node.find_first_by_tag("testcase")
-        systemout = pnode.find_first_by_tag("system-out")
-        assert "hello-stdout" in systemout.toxml()
+        if junit_logging == "no":
+            assert not node.find_by_tag(
+                "system-out"
+            ), "system-out should not be generated"
+        if junit_logging == "system-out":
+            systemout = pnode.find_first_by_tag("system-out")
+            assert (
+                "hello-stdout" in systemout.toxml()
+            ), "'hello-stdout' should be in system-out"
 
-    def test_setup_error_captures_stderr(self, testdir, run_and_parse):
+    @pytest.mark.parametrize("junit_logging", ["no", "system-err"])
+    def test_setup_error_captures_stderr(self, testdir, run_and_parse, junit_logging):
         testdir.makepyfile(
             """
             import sys
@@ -756,13 +808,21 @@ class TestPython:
                 pass
         """
         )
-        result, dom = run_and_parse()
+        result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging)
         node = dom.find_first_by_tag("testsuite")
         pnode = node.find_first_by_tag("testcase")
-        systemout = pnode.find_first_by_tag("system-err")
-        assert "hello-stderr" in systemout.toxml()
+        if junit_logging == "no":
+            assert not node.find_by_tag(
+                "system-err"
+            ), "system-err should not be generated"
+        if junit_logging == "system-err":
+            systemerr = pnode.find_first_by_tag("system-err")
+            assert (
+                "hello-stderr" in systemerr.toxml()
+            ), "'hello-stderr' should be in system-err"
 
-    def test_avoid_double_stdout(self, testdir, run_and_parse):
+    @pytest.mark.parametrize("junit_logging", ["no", "system-out"])
+    def test_avoid_double_stdout(self, testdir, run_and_parse, junit_logging):
         testdir.makepyfile(
             """
             import sys
@@ -777,12 +837,17 @@ class TestPython:
                 sys.stdout.write('hello-stdout call')
         """
         )
-        result, dom = run_and_parse()
+        result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging)
         node = dom.find_first_by_tag("testsuite")
         pnode = node.find_first_by_tag("testcase")
-        systemout = pnode.find_first_by_tag("system-out")
-        assert "hello-stdout call" in systemout.toxml()
-        assert "hello-stdout teardown" in systemout.toxml()
+        if junit_logging == "no":
+            assert not node.find_by_tag(
+                "system-out"
+            ), "system-out should not be generated"
+        if junit_logging == "system-out":
+            systemout = pnode.find_first_by_tag("system-out")
+            assert "hello-stdout call" in systemout.toxml()
+            assert "hello-stdout teardown" in systemout.toxml()
 
 
 def test_mangle_test_address():
@@ -850,7 +915,8 @@ class TestNonPython:
         assert "custom item runtest failed" in fnode.toxml()
 
 
-def test_nullbyte(testdir):
+@pytest.mark.parametrize("junit_logging", ["no", "system-out"])
+def test_nullbyte(testdir, junit_logging):
     # A null byte can not occur in XML (see section 2.2 of the spec)
     testdir.makepyfile(
         """
@@ -862,13 +928,17 @@ def test_nullbyte(testdir):
     """
     )
     xmlf = testdir.tmpdir.join("junit.xml")
-    testdir.runpytest("--junitxml=%s" % xmlf)
+    testdir.runpytest("--junitxml=%s" % xmlf, "-o", "junit_logging=%s" % junit_logging)
     text = xmlf.read()
     assert "\x00" not in text
-    assert "#x00" in text
+    if junit_logging == "system-out":
+        assert "#x00" in text
+    if junit_logging == "no":
+        assert "#x00" not in text
 
 
-def test_nullbyte_replace(testdir):
+@pytest.mark.parametrize("junit_logging", ["no", "system-out"])
+def test_nullbyte_replace(testdir, junit_logging):
     # Check if the null byte gets replaced
     testdir.makepyfile(
         """
@@ -880,9 +950,12 @@ def test_nullbyte_replace(testdir):
     """
     )
     xmlf = testdir.tmpdir.join("junit.xml")
-    testdir.runpytest("--junitxml=%s" % xmlf)
+    testdir.runpytest("--junitxml=%s" % xmlf, "-o", "junit_logging=%s" % junit_logging)
     text = xmlf.read()
-    assert "#x0" in text
+    if junit_logging == "system-out":
+        assert "#x0" in text
+    if junit_logging == "no":
+        assert "#x0" not in text
 
 
 def test_invalid_xml_escape():
@@ -1227,6 +1300,7 @@ def test_runs_twice(testdir, run_and_parse):
 
 def test_runs_twice_xdist(testdir, run_and_parse):
     pytest.importorskip("xdist")
+    testdir.monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
     f = testdir.makepyfile(
         """
         def test_pass():

testing/test_main.py

@@ -1,5 +1,8 @@
+from typing import Optional
+
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
+from _pytest.pytester import Testdir
 
 
 @pytest.mark.parametrize(
@@ -50,3 +53,25 @@ def test_wrap_session_notify_exception(ret_exc, testdir):
         assert result.stderr.lines == ["mainloop: caught unexpected SystemExit!"]
     else:
         assert result.stderr.lines == ["Exit: exiting after {}...".format(exc.__name__)]
+
+
+@pytest.mark.parametrize("returncode", (None, 42))
+def test_wrap_session_exit_sessionfinish(
+    returncode: Optional[int], testdir: Testdir
+) -> None:
+    testdir.makeconftest(
+        """
+        import pytest
+        def pytest_sessionfinish():
+            pytest.exit(msg="exit_pytest_sessionfinish", returncode={returncode})
+    """.format(
+            returncode=returncode
+        )
+    )
+    result = testdir.runpytest()
+    if returncode:
+        assert result.ret == returncode
+    else:
+        assert result.ret == ExitCode.NO_TESTS_COLLECTED
+    assert result.stdout.lines[-1] == "collected 0 items"
+    assert result.stderr.lines == ["Exit: exit_pytest_sessionfinish"]

testing/test_mark.py

@@ -3,7 +3,7 @@ import sys
 from unittest import mock
 
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 from _pytest.mark import EMPTY_PARAMETERSET_OPTION
 from _pytest.mark import MarkGenerator as Mark
 from _pytest.nodes import Collector
@@ -962,7 +962,11 @@ def test_mark_expressions_no_smear(testdir):
 
 
 def test_addmarker_order():
-    node = Node("Test", config=mock.Mock(), session=mock.Mock(), nodeid="Test")
+    session = mock.Mock()
+    session.own_markers = []
+    session.parent = None
+    session.nodeid = ""
+    node = Node.from_parent(session, name="Test")
     node.add_marker("foo")
     node.add_marker("bar")
     node.add_marker("baz", append=False)

testing/test_meta.py

@@ -1,3 +1,9 @@
+"""
+Test importing of all internal packages and modules.
+
+This ensures all internal packages can be imported without needing the pytest
+namespace being set, which is critical for the initialization of xdist.
+"""
 import pkgutil
 import subprocess
 import sys

testing/test_modimport.py

@@ -1,40 +0,0 @@
-import subprocess
-import sys
-
-import py
-
-import _pytest
-import pytest
-
-pytestmark = pytest.mark.slow
-
-MODSET = [
-    x
-    for x in py.path.local(_pytest.__file__).dirpath().visit("*.py")
-    if x.purebasename != "__init__"
-]
-
-
-@pytest.mark.parametrize("modfile", MODSET, ids=lambda x: x.purebasename)
-def test_fileimport(modfile):
-    # this test ensures all internal packages can import
-    # without needing the pytest namespace being set
-    # this is critical for the initialization of xdist
-
-    p = subprocess.Popen(
-        [
-            sys.executable,
-            "-c",
-            "import sys, py; py.path.local(sys.argv[1]).pyimport()",
-            modfile.strpath,
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-    (out, err) = p.communicate()
-    assert p.returncode == 0, "importing %s failed (exitcode %d): out=%r, err=%r" % (
-        modfile,
-        p.returncode,
-        out,
-        err,
-    )

testing/test_nodes.py

@@ -22,6 +22,13 @@ def test_ischildnode(baseid, nodeid, expected):
     assert result is expected
 
 
+def test_node_from_parent_disallowed_arguments():
+    with pytest.raises(TypeError, match="session is"):
+        nodes.Node.from_parent(None, session=None)
+    with pytest.raises(TypeError, match="config is"):
+        nodes.Node.from_parent(None, config=None)
+
+
 def test_std_warn_not_pytestwarning(testdir):
     items = testdir.getitems(
         """

testing/test_parseopt.py

@@ -1,7 +1,7 @@
 import argparse
-import distutils.spawn
 import os
 import shlex
+import shutil
 import sys
 
 import py
@@ -12,22 +12,22 @@ from _pytest.config.exceptions import UsageError
 
 
 @pytest.fixture
-def parser():
+def parser() -> parseopt.Parser:
     return parseopt.Parser()
 
 
 class TestParser:
-    def test_no_help_by_default(self):
+    def test_no_help_by_default(self) -> None:
         parser = parseopt.Parser(usage="xyz")
         pytest.raises(UsageError, lambda: parser.parse(["-h"]))
 
-    def test_custom_prog(self, parser):
+    def test_custom_prog(self, parser: parseopt.Parser) -> None:
         """Custom prog can be set for `argparse.ArgumentParser`."""
         assert parser._getparser().prog == os.path.basename(sys.argv[0])
         parser.prog = "custom-prog"
         assert parser._getparser().prog == "custom-prog"
 
-    def test_argument(self):
+    def test_argument(self) -> None:
         with pytest.raises(parseopt.ArgumentError):
             # need a short or long option
             argument = parseopt.Argument()
@@ -45,7 +45,7 @@ class TestParser:
             "Argument(_short_opts: ['-t'], _long_opts: ['--test'], dest: 'abc')"
         )
 
-    def test_argument_type(self):
+    def test_argument_type(self) -> None:
         argument = parseopt.Argument("-t", dest="abc", type=int)
         assert argument.type is int
         argument = parseopt.Argument("-t", dest="abc", type=str)
@@ -60,7 +60,7 @@ class TestParser:
         )
         assert argument.type is str
 
-    def test_argument_processopt(self):
+    def test_argument_processopt(self) -> None:
         argument = parseopt.Argument("-t", type=int)
         argument.default = 42
         argument.dest = "abc"
@@ -68,19 +68,19 @@ class TestParser:
         assert res["default"] == 42
         assert res["dest"] == "abc"
 
-    def test_group_add_and_get(self, parser):
+    def test_group_add_and_get(self, parser: parseopt.Parser) -> None:
         group = parser.getgroup("hello", description="desc")
         assert group.name == "hello"
         assert group.description == "desc"
 
-    def test_getgroup_simple(self, parser):
+    def test_getgroup_simple(self, parser: parseopt.Parser) -> None:
         group = parser.getgroup("hello", description="desc")
         assert group.name == "hello"
         assert group.description == "desc"
         group2 = parser.getgroup("hello")
         assert group2 is group
 
-    def test_group_ordering(self, parser):
+    def test_group_ordering(self, parser: parseopt.Parser) -> None:
         parser.getgroup("1")
         parser.getgroup("2")
         parser.getgroup("3", after="1")
@@ -88,20 +88,20 @@ class TestParser:
         groups_names = [x.name for x in groups]
         assert groups_names == list("132")
 
-    def test_group_addoption(self):
+    def test_group_addoption(self) -> None:
         group = parseopt.OptionGroup("hello")
         group.addoption("--option1", action="store_true")
         assert len(group.options) == 1
         assert isinstance(group.options[0], parseopt.Argument)
 
-    def test_group_addoption_conflict(self):
+    def test_group_addoption_conflict(self) -> None:
         group = parseopt.OptionGroup("hello again")
         group.addoption("--option1", "--option-1", action="store_true")
         with pytest.raises(ValueError) as err:
             group.addoption("--option1", "--option-one", action="store_true")
         assert str({"--option1"}) in str(err.value)
 
-    def test_group_shortopt_lowercase(self, parser):
+    def test_group_shortopt_lowercase(self, parser: parseopt.Parser) -> None:
         group = parser.getgroup("hello")
         with pytest.raises(ValueError):
             group.addoption("-x", action="store_true")
@@ -109,30 +109,30 @@ class TestParser:
         group._addoption("-x", action="store_true")
         assert len(group.options) == 1
 
-    def test_parser_addoption(self, parser):
+    def test_parser_addoption(self, parser: parseopt.Parser) -> None:
         group = parser.getgroup("custom options")
         assert len(group.options) == 0
         group.addoption("--option1", action="store_true")
         assert len(group.options) == 1
 
-    def test_parse(self, parser):
+    def test_parse(self, parser: parseopt.Parser) -> None:
         parser.addoption("--hello", dest="hello", action="store")
         args = parser.parse(["--hello", "world"])
         assert args.hello == "world"
         assert not getattr(args, parseopt.FILE_OR_DIR)
 
-    def test_parse2(self, parser):
+    def test_parse2(self, parser: parseopt.Parser) -> None:
         args = parser.parse([py.path.local()])
         assert getattr(args, parseopt.FILE_OR_DIR)[0] == py.path.local()
 
-    def test_parse_known_args(self, parser):
+    def test_parse_known_args(self, parser: parseopt.Parser) -> None:
         parser.parse_known_args([py.path.local()])
         parser.addoption("--hello", action="store_true")
         ns = parser.parse_known_args(["x", "--y", "--hello", "this"])
         assert ns.hello
         assert ns.file_or_dir == ["x"]
 
-    def test_parse_known_and_unknown_args(self, parser):
+    def test_parse_known_and_unknown_args(self, parser: parseopt.Parser) -> None:
         parser.addoption("--hello", action="store_true")
         ns, unknown = parser.parse_known_and_unknown_args(
             ["x", "--y", "--hello", "this"]
@@ -141,7 +141,7 @@ class TestParser:
         assert ns.file_or_dir == ["x"]
         assert unknown == ["--y", "this"]
 
-    def test_parse_will_set_default(self, parser):
+    def test_parse_will_set_default(self, parser: parseopt.Parser) -> None:
         parser.addoption("--hello", dest="hello", default="x", action="store")
         option = parser.parse([])
         assert option.hello == "x"
@@ -149,25 +149,22 @@ class TestParser:
         parser.parse_setoption([], option)
         assert option.hello == "x"
 
-    def test_parse_setoption(self, parser):
+    def test_parse_setoption(self, parser: parseopt.Parser) -> None:
         parser.addoption("--hello", dest="hello", action="store")
         parser.addoption("--world", dest="world", default=42)
 
-        class A:
-            pass
-
-        option = A()
+        option = argparse.Namespace()
         args = parser.parse_setoption(["--hello", "world"], option)
         assert option.hello == "world"
         assert option.world == 42
         assert not args
 
-    def test_parse_special_destination(self, parser):
+    def test_parse_special_destination(self, parser: parseopt.Parser) -> None:
         parser.addoption("--ultimate-answer", type=int)
         args = parser.parse(["--ultimate-answer", "42"])
         assert args.ultimate_answer == 42
 
-    def test_parse_split_positional_arguments(self, parser):
+    def test_parse_split_positional_arguments(self, parser: parseopt.Parser) -> None:
         parser.addoption("-R", action="store_true")
         parser.addoption("-S", action="store_false")
         args = parser.parse(["-R", "4", "2", "-S"])
@@ -181,7 +178,7 @@ class TestParser:
         assert args.R is True
         assert args.S is False
 
-    def test_parse_defaultgetter(self):
+    def test_parse_defaultgetter(self) -> None:
         def defaultget(option):
             if not hasattr(option, "type"):
                 return
@@ -199,17 +196,17 @@ class TestParser:
         assert option.this == 42
         assert option.no is False
 
-    def test_drop_short_helper(self):
+    def test_drop_short_helper(self) -> None:
         parser = argparse.ArgumentParser(
             formatter_class=parseopt.DropShorterLongHelpFormatter, allow_abbrev=False
         )
         parser.add_argument(
             "-t", "--twoword", "--duo", "--two-word", "--two", help="foo"
-        ).map_long_option = {"two": "two-word"}
+        )
         # throws error on --deux only!
         parser.add_argument(
             "-d", "--deuxmots", "--deux-mots", action="store_true", help="foo"
-        ).map_long_option = {"deux": "deux-mots"}
+        )
         parser.add_argument("-s", action="store_true", help="single short")
         parser.add_argument("--abc", "-a", action="store_true", help="bar")
         parser.add_argument("--klm", "-k", "--kl-m", action="store_true", help="bar")
@@ -221,7 +218,7 @@ class TestParser:
         )
         parser.add_argument(
             "-x", "--exit-on-first", "--exitfirst", action="store_true", help="spam"
-        ).map_long_option = {"exitfirst": "exit-on-first"}
+        )
         parser.add_argument("files_and_dirs", nargs="*")
         args = parser.parse_args(["-k", "--duo", "hallo", "--exitfirst"])
         assert args.twoword == "hallo"
@@ -236,32 +233,32 @@ class TestParser:
         args = parser.parse_args(["file", "dir"])
         assert "|".join(args.files_and_dirs) == "file|dir"
 
-    def test_drop_short_0(self, parser):
+    def test_drop_short_0(self, parser: parseopt.Parser) -> None:
         parser.addoption("--funcarg", "--func-arg", action="store_true")
         parser.addoption("--abc-def", "--abc-def", action="store_true")
         parser.addoption("--klm-hij", action="store_true")
         with pytest.raises(UsageError):
             parser.parse(["--funcarg", "--k"])
 
-    def test_drop_short_2(self, parser):
+    def test_drop_short_2(self, parser: parseopt.Parser) -> None:
         parser.addoption("--func-arg", "--doit", action="store_true")
         args = parser.parse(["--doit"])
         assert args.func_arg is True
 
-    def test_drop_short_3(self, parser):
+    def test_drop_short_3(self, parser: parseopt.Parser) -> None:
         parser.addoption("--func-arg", "--funcarg", "--doit", action="store_true")
         args = parser.parse(["abcd"])
         assert args.func_arg is False
         assert args.file_or_dir == ["abcd"]
 
-    def test_drop_short_help0(self, parser):
+    def test_drop_short_help0(self, parser: parseopt.Parser) -> None:
         parser.addoption("--func-args", "--doit", help="foo", action="store_true")
         parser.parse([])
         help = parser.optparser.format_help()
         assert "--func-args, --doit  foo" in help
 
     # testing would be more helpful with all help generated
-    def test_drop_short_help1(self, parser):
+    def test_drop_short_help1(self, parser: parseopt.Parser) -> None:
         group = parser.getgroup("general")
         group.addoption("--doit", "--func-args", action="store_true", help="foo")
         group._addoption(
@@ -275,7 +272,7 @@ class TestParser:
         help = parser.optparser.format_help()
         assert "-doit, --func-args  foo" in help
 
-    def test_multiple_metavar_help(self, parser):
+    def test_multiple_metavar_help(self, parser: parseopt.Parser) -> None:
         """
         Help text for options with a metavar tuple should display help
         in the form "--preferences=value1 value2 value3" (#2004).
@@ -290,8 +287,8 @@ class TestParser:
         assert "--preferences=value1 value2 value3" in help
 
 
-def test_argcomplete(testdir, monkeypatch):
-    if not distutils.spawn.find_executable("bash"):
+def test_argcomplete(testdir, monkeypatch) -> None:
+    if not shutil.which("bash"):
         pytest.skip("bash not available")
     script = str(testdir.tmpdir.join("test_argcomplete"))
 

testing/test_pluginmanager.py

@@ -3,9 +3,9 @@ import sys
 import types
 
 import pytest
+from _pytest.config import ExitCode
 from _pytest.config import PytestPluginManager
 from _pytest.config.exceptions import UsageError
-from _pytest.main import ExitCode
 from _pytest.main import Session
 
 
@@ -122,7 +122,7 @@ class TestPytestPluginInteractions:
     def test_hook_proxy(self, testdir):
         """Test the gethookproxy function(#2016)"""
         config = testdir.parseconfig()
-        session = Session(config)
+        session = Session.from_config(config)
         testdir.makepyfile(**{"tests/conftest.py": "", "tests/subdir/conftest.py": ""})
 
         conftest1 = testdir.tmpdir.join("tests/conftest.py")

testing/test_pytester.py

@@ -8,8 +8,8 @@ import py.path
 
 import _pytest.pytester as pytester
 import pytest
+from _pytest.config import ExitCode
 from _pytest.config import PytestPluginManager
-from _pytest.main import ExitCode
 from _pytest.outcomes import Failed
 from _pytest.pytester import CwdSnapshot
 from _pytest.pytester import HookRecorder
@@ -458,17 +458,26 @@ def test_testdir_run_timeout_expires(testdir) -> None:
 
 def test_linematcher_with_nonlist() -> None:
     """Test LineMatcher with regard to passing in a set (accidentally)."""
-    lm = LineMatcher([])
+    from _pytest._code.source import Source
 
-    with pytest.raises(AssertionError):
-        lm.fnmatch_lines(set())
-    with pytest.raises(AssertionError):
-        lm.fnmatch_lines({})
+    lm = LineMatcher([])
+    with pytest.raises(TypeError, match="invalid type for lines2: set"):
+        lm.fnmatch_lines(set())  # type: ignore[arg-type]  # noqa: F821
+    with pytest.raises(TypeError, match="invalid type for lines2: dict"):
+        lm.fnmatch_lines({})  # type: ignore[arg-type]  # noqa: F821
+    with pytest.raises(TypeError, match="invalid type for lines2: set"):
+        lm.re_match_lines(set())  # type: ignore[arg-type]  # noqa: F821
+    with pytest.raises(TypeError, match="invalid type for lines2: dict"):
+        lm.re_match_lines({})  # type: ignore[arg-type]  # noqa: F821
+    with pytest.raises(TypeError, match="invalid type for lines2: Source"):
+        lm.fnmatch_lines(Source())  # type: ignore[arg-type]  # noqa: F821
     lm.fnmatch_lines([])
     lm.fnmatch_lines(())
-
-    assert lm._getlines({}) == {}
-    assert lm._getlines(set()) == set()
+    lm.fnmatch_lines("")
+    assert lm._getlines({}) == {}  # type: ignore[arg-type,comparison-overlap]  # noqa: F821
+    assert lm._getlines(set()) == set()  # type: ignore[arg-type,comparison-overlap]  # noqa: F821
+    assert lm._getlines(Source()) == []
+    assert lm._getlines(Source("pass\npass")) == ["pass", "pass"]
 
 
 def test_linematcher_match_failure() -> None:
@@ -499,8 +508,28 @@ def test_linematcher_match_failure() -> None:
     ]
 
 
+def test_linematcher_consecutive():
+    lm = LineMatcher(["1", "", "2"])
+    with pytest.raises(pytest.fail.Exception) as excinfo:
+        lm.fnmatch_lines(["1", "2"], consecutive=True)
+    assert str(excinfo.value).splitlines() == [
+        "exact match: '1'",
+        "no consecutive match: '2'",
+        "   with: ''",
+    ]
+
+    lm.re_match_lines(["1", r"\d?", "2"], consecutive=True)
+    with pytest.raises(pytest.fail.Exception) as excinfo:
+        lm.re_match_lines(["1", r"\d", "2"], consecutive=True)
+    assert str(excinfo.value).splitlines() == [
+        "exact match: '1'",
+        r"no consecutive match: '\\d'",
+        "    with: ''",
+    ]
+
+
 @pytest.mark.parametrize("function", ["no_fnmatch_line", "no_re_match_line"])
-def test_no_matching(function) -> None:
+def test_linematcher_no_matching(function) -> None:
     if function == "no_fnmatch_line":
         good_pattern = "*.py OK*"
         bad_pattern = "*X.py OK*"
@@ -548,7 +577,7 @@ def test_no_matching(function) -> None:
     func(bad_pattern)  # bad pattern does not match any line: passes
 
 
-def test_no_matching_after_match() -> None:
+def test_linematcher_no_matching_after_match() -> None:
     lm = LineMatcher(["1", "2", "3"])
     lm.fnmatch_lines(["1", "3"])
     with pytest.raises(Failed) as e:
@@ -556,17 +585,15 @@ def test_no_matching_after_match() -> None:
     assert str(e.value).splitlines() == ["fnmatch: '*'", "   with: '1'"]
 
 
-def test_pytester_addopts(request, monkeypatch) -> None:
+def test_pytester_addopts_before_testdir(request, monkeypatch) -> None:
+    orig = os.environ.get("PYTEST_ADDOPTS", None)
     monkeypatch.setenv("PYTEST_ADDOPTS", "--orig-unused")
-
     testdir = request.getfixturevalue("testdir")
-
-    try:
-        assert "PYTEST_ADDOPTS" not in os.environ
-    finally:
-        testdir.finalize()
-
-    assert os.environ["PYTEST_ADDOPTS"] == "--orig-unused"
+    assert "PYTEST_ADDOPTS" not in os.environ
+    testdir.finalize()
+    assert os.environ.get("PYTEST_ADDOPTS") == "--orig-unused"
+    monkeypatch.undo()
+    assert os.environ.get("PYTEST_ADDOPTS") == orig
 
 
 def test_run_stdin(testdir) -> None:
@@ -646,14 +673,10 @@ def test_popen_default_stdin_stderr_and_stdin_None(testdir) -> None:
 
 
 def test_spawn_uses_tmphome(testdir) -> None:
-    import os
-
     tmphome = str(testdir.tmpdir)
+    assert os.environ.get("HOME") == tmphome
 
-    # Does use HOME only during run.
-    assert os.environ.get("HOME") != tmphome
-
-    testdir._env_run_update["CUSTOMENV"] = "42"
+    testdir.monkeypatch.setenv("CUSTOMENV", "42")
 
     p1 = testdir.makepyfile(
         """
@@ -710,3 +733,13 @@ def test_testdir_outcomes_with_multiple_errors(testdir):
     result.assert_outcomes(error=2)
 
     assert result.parseoutcomes() == {"error": 2}
+
+
+def test_makefile_joins_absolute_path(testdir: Testdir) -> None:
+    absfile = testdir.tmpdir / "absfile"
+    if sys.platform == "win32":
+        with pytest.raises(OSError):
+            testdir.makepyfile(**{str(absfile): ""})
+    else:
+        p1 = testdir.makepyfile(**{str(absfile): ""})
+        assert str(p1) == (testdir.tmpdir / absfile) + ".py"

testing/test_runner.py

@@ -10,10 +10,10 @@ import py
 
 import _pytest._code
 import pytest
-from _pytest import main
 from _pytest import outcomes
 from _pytest import reports
 from _pytest import runner
+from _pytest.config import ExitCode
 from _pytest.outcomes import Exit
 from _pytest.outcomes import Failed
 from _pytest.outcomes import OutcomeException
@@ -681,7 +681,7 @@ def test_pytest_fail_notrace_non_ascii(testdir) -> None:
 def test_pytest_no_tests_collected_exit_status(testdir) -> None:
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["*collected 0 items*"])
-    assert result.ret == main.ExitCode.NO_TESTS_COLLECTED
+    assert result.ret == ExitCode.NO_TESTS_COLLECTED
 
     testdir.makepyfile(
         test_foo="""
@@ -692,12 +692,12 @@ def test_pytest_no_tests_collected_exit_status(testdir) -> None:
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["*collected 1 item*"])
     result.stdout.fnmatch_lines(["*1 passed*"])
-    assert result.ret == main.ExitCode.OK
+    assert result.ret == ExitCode.OK
 
     result = testdir.runpytest("-k nonmatch")
     result.stdout.fnmatch_lines(["*collected 1 item*"])
     result.stdout.fnmatch_lines(["*1 deselected*"])
-    assert result.ret == main.ExitCode.NO_TESTS_COLLECTED
+    assert result.ret == ExitCode.NO_TESTS_COLLECTED
 
 
 def test_exception_printing_skip() -> None:

testing/test_session.py

@@ -1,5 +1,5 @@
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 
 
 class SessionTests:

testing/test_setuponly.py

@@ -1,5 +1,5 @@
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 
 
 @pytest.fixture(params=["--setup-only", "--setup-plan", "--setup-show"], scope="module")

testing/test_terminal.py

@@ -3,7 +3,6 @@ terminal reporting of the full testing process.
 """
 import collections
 import os
-import re
 import sys
 import textwrap
 from io import StringIO
@@ -16,7 +15,8 @@ import py
 
 import _pytest.config
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
+from _pytest.pytester import Testdir
 from _pytest.reports import BaseReport
 from _pytest.terminal import _folded_skips
 from _pytest.terminal import _get_line_with_reprcrash_message
@@ -26,14 +26,8 @@ from _pytest.terminal import TerminalReporter
 
 DistInfo = collections.namedtuple("DistInfo", ["project_name", "version"])
 
-COLORS = {
-    "red": "\x1b[31m",
-    "green": "\x1b[32m",
-    "yellow": "\x1b[33m",
-    "bold": "\x1b[1m",
-    "reset": "\x1b[0m",
-}
-RE_COLORS = {k: re.escape(v) for k, v in COLORS.items()}
+
+TRANS_FNMATCH = str.maketrans({"[": "[[]", "]": "[]]"})
 
 
 class Option:
@@ -163,6 +157,8 @@ class TestTerminal:
                 "test2.py": "def test_2(): pass",
             }
         )
+        # Explicitly test colored output.
+        testdir.monkeypatch.setenv("PY_COLORS", "1")
 
         child = testdir.spawn_pytest("-v test1.py test2.py")
         child.expect(r"collecting \.\.\.")
@@ -604,6 +600,7 @@ class TestTerminalFunctional:
         assert result.ret == 0
 
     def test_header_trailer_info(self, testdir, request):
+        testdir.monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
         testdir.makepyfile(
             """
             def test_passes():
@@ -677,6 +674,26 @@ class TestTerminalFunctional:
             ]
         )
 
+    def test_showlocals_short(self, testdir):
+        p1 = testdir.makepyfile(
+            """
+            def test_showlocals_short():
+                x = 3
+                y = "xxxx"
+                assert 0
+        """
+        )
+        result = testdir.runpytest(p1, "-l", "--tb=short")
+        result.stdout.fnmatch_lines(
+            [
+                "test_showlocals_short.py:*",
+                "    assert 0",
+                "E   assert 0",
+                "        x          = 3",
+                "        y          = 'xxxx'",
+            ]
+        )
+
     @pytest.fixture
     def verbose_testfile(self, testdir):
         return testdir.makepyfile(
@@ -714,6 +731,7 @@ class TestTerminalFunctional:
         if not pytestconfig.pluginmanager.get_plugin("xdist"):
             pytest.skip("xdist plugin not installed")
 
+        testdir.monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
         result = testdir.runpytest(
             verbose_testfile, "-v", "-n 1", "-Walways::pytest.PytestWarning"
         )
@@ -761,13 +779,42 @@ class TestTerminalFunctional:
         result = testdir.runpytest(*params)
         result.stdout.fnmatch_lines(["collected 3 items", "hello from hook: 3 items"])
 
+    def test_summary_f_alias(self, testdir):
+        """Test that 'f' and 'F' report chars are aliases and don't show up twice in the summary (#6334)"""
+        testdir.makepyfile(
+            """
+            def test():
+                assert False
+            """
+        )
+        result = testdir.runpytest("-rfF")
+        expected = "FAILED test_summary_f_alias.py::test - assert False"
+        result.stdout.fnmatch_lines([expected])
+        assert result.stdout.lines.count(expected) == 1
+
+    def test_summary_s_alias(self, testdir):
+        """Test that 's' and 'S' report chars are aliases and don't show up twice in the summary"""
+        testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.skip
+            def test():
+                pass
+            """
+        )
+        result = testdir.runpytest("-rsS")
+        expected = "SKIPPED [1] test_summary_s_alias.py:3: unconditional skip"
+        result.stdout.fnmatch_lines([expected])
+        assert result.stdout.lines.count(expected) == 1
+
 
 def test_fail_extra_reporting(testdir, monkeypatch):
     monkeypatch.setenv("COLUMNS", "80")
     testdir.makepyfile("def test_this(): assert 0, 'this_failed' * 100")
-    result = testdir.runpytest()
+    result = testdir.runpytest("-rN")
     result.stdout.no_fnmatch_line("*short test summary*")
-    result = testdir.runpytest("-rf")
+    result = testdir.runpytest()
     result.stdout.fnmatch_lines(
         [
             "*test summary*",
@@ -836,7 +883,7 @@ def test_pass_output_reporting(testdir):
     )
 
 
-def test_color_yes(testdir):
+def test_color_yes(testdir, color_mapping):
     p1 = testdir.makepyfile(
         """
         def fail():
@@ -847,16 +894,10 @@ def test_color_yes(testdir):
         """
     )
     result = testdir.runpytest("--color=yes", str(p1))
-    if sys.version_info < (3, 6):
-        # py36 required for ordered markup
-        output = result.stdout.str()
-        assert "test session starts" in output
-        assert "\x1b[1m" in output
-        return
+    color_mapping.requires_ordered_markup(result)
     result.stdout.fnmatch_lines(
-        [
-            line.format(**COLORS).replace("[", "[[]")
-            for line in [
+        color_mapping.format_for_fnmatch(
+            [
                 "{bold}=*= test session starts =*={reset}",
                 "collected 1 item",
                 "",
@@ -865,26 +906,25 @@ def test_color_yes(testdir):
                 "=*= FAILURES =*=",
                 "{red}{bold}_*_ test_this _*_{reset}",
                 "",
-                "{bold}    def test_this():{reset}",
-                "{bold}>       fail(){reset}",
+                "    {kw}def{hl-reset} {function}test_this{hl-reset}():",
+                ">       fail()",
                 "",
                 "{bold}{red}test_color_yes.py{reset}:5: ",
                 "_ _ * _ _*",
                 "",
-                "{bold}    def fail():{reset}",
-                "{bold}>       assert 0{reset}",
+                "    {kw}def{hl-reset} {function}fail{hl-reset}():",
+                ">       {kw}assert{hl-reset} {number}0{hl-reset}",
                 "{bold}{red}E       assert 0{reset}",
                 "",
                 "{bold}{red}test_color_yes.py{reset}:2: AssertionError",
                 "{red}=*= {red}{bold}1 failed{reset}{red} in *s{reset}{red} =*={reset}",
             ]
-        ]
+        )
     )
     result = testdir.runpytest("--color=yes", "--tb=short", str(p1))
     result.stdout.fnmatch_lines(
-        [
-            line.format(**COLORS).replace("[", "[[]")
-            for line in [
+        color_mapping.format_for_fnmatch(
+            [
                 "{bold}=*= test session starts =*={reset}",
                 "collected 1 item",
                 "",
@@ -893,13 +933,13 @@ def test_color_yes(testdir):
                 "=*= FAILURES =*=",
                 "{red}{bold}_*_ test_this _*_{reset}",
                 "{bold}{red}test_color_yes.py{reset}:5: in test_this",
-                "{bold}    fail(){reset}",
+                "    fail()",
                 "{bold}{red}test_color_yes.py{reset}:2: in fail",
-                "{bold}    assert 0{reset}",
+                "    {kw}assert{hl-reset} {number}0{hl-reset}",
                 "{bold}{red}E   assert 0{reset}",
                 "{red}=*= {red}{bold}1 failed{reset}{red} in *s{reset}{red} =*={reset}",
             ]
-        ]
+        )
     )
 
 
@@ -936,37 +976,62 @@ def test_color_yes_collection_on_non_atty(testdir, verbose):
 
 
 def test_getreportopt():
+    from _pytest.terminal import _REPORTCHARS_DEFAULT
+
     class Config:
         class Option:
-            reportchars = ""
-            disable_warnings = True
+            reportchars = _REPORTCHARS_DEFAULT
+            disable_warnings = False
 
         option = Option()
 
     config = Config()
 
+    assert _REPORTCHARS_DEFAULT == "fE"
+
+    # Default.
+    assert getreportopt(config) == "wfE"
+
     config.option.reportchars = "sf"
-    assert getreportopt(config) == "sf"
+    assert getreportopt(config) == "wsf"
+
+    config.option.reportchars = "sfxw"
+    assert getreportopt(config) == "sfxw"
+
+    config.option.reportchars = "a"
+    assert getreportopt(config) == "wsxXEf"
+
+    config.option.reportchars = "N"
+    assert getreportopt(config) == "w"
+
+    config.option.reportchars = "NwfE"
+    assert getreportopt(config) == "wfE"
+
+    config.option.reportchars = "NfENx"
+    assert getreportopt(config) == "wx"
+
+    # Now with --disable-warnings.
+    config.option.disable_warnings = True
+    config.option.reportchars = "a"
+    assert getreportopt(config) == "sxXEf"
+
+    config.option.reportchars = "sfx"
+    assert getreportopt(config) == "sfx"
 
     config.option.reportchars = "sfxw"
     assert getreportopt(config) == "sfx"
 
-    # Now with --disable-warnings.
-    config.option.disable_warnings = False
     config.option.reportchars = "a"
-    assert getreportopt(config) == "sxXwEf"  # NOTE: "w" included!
-
-    config.option.reportchars = "sfx"
-    assert getreportopt(config) == "sfxw"
-
-    config.option.reportchars = "sfxw"
-    assert getreportopt(config) == "sfxw"
-
-    config.option.reportchars = "a"
-    assert getreportopt(config) == "sxXwEf"  # NOTE: "w" included!
+    assert getreportopt(config) == "sxXEf"
 
     config.option.reportchars = "A"
-    assert getreportopt(config) == "PpsxXwEf"
+    assert getreportopt(config) == "PpsxXEf"
+
+    config.option.reportchars = "AN"
+    assert getreportopt(config) == ""
+
+    config.option.reportchars = "NwfE"
+    assert getreportopt(config) == "fE"
 
 
 def test_terminalreporter_reportopt_addopts(testdir):
@@ -1083,7 +1148,7 @@ class TestGenericReporting:
         )
         for tbopt in ["long", "short", "no"]:
             print("testing --tb=%s..." % tbopt)
-            result = testdir.runpytest("--tb=%s" % tbopt)
+            result = testdir.runpytest("-rN", "--tb=%s" % tbopt)
             s = result.stdout.str()
             if tbopt == "long":
                 assert "print(6*7)" in s
@@ -1622,7 +1687,7 @@ class TestProgressOutputStyle:
             ]
         )
 
-    def test_colored_progress(self, testdir, monkeypatch):
+    def test_colored_progress(self, testdir, monkeypatch, color_mapping):
         monkeypatch.setenv("PY_COLORS", "1")
         testdir.makepyfile(
             test_axfail="""
@@ -1651,27 +1716,25 @@ class TestProgressOutputStyle:
         )
         result = testdir.runpytest()
         result.stdout.re_match_lines(
-            [
-                line.format(**RE_COLORS)
-                for line in [
+            color_mapping.format_for_rematch(
+                [
                     r"test_axfail.py {yellow}x{reset}{green} \s+ \[  4%\]{reset}",
                     r"test_bar.py ({green}\.{reset}){{10}}{green} \s+ \[ 52%\]{reset}",
                     r"test_foo.py ({green}\.{reset}){{5}}{yellow} \s+ \[ 76%\]{reset}",
                     r"test_foobar.py ({red}F{reset}){{5}}{red} \s+ \[100%\]{reset}",
                 ]
-            ]
+            )
         )
 
         # Only xfail should have yellow progress indicator.
         result = testdir.runpytest("test_axfail.py")
         result.stdout.re_match_lines(
-            [
-                line.format(**RE_COLORS)
-                for line in [
+            color_mapping.format_for_rematch(
+                [
                     r"test_axfail.py {yellow}x{reset}{yellow} \s+ \[100%\]{reset}",
                     r"^{yellow}=+ ({yellow}{bold}|{bold}{yellow})1 xfailed{reset}{yellow} in ",
                 ]
-            ]
+            )
         )
 
     def test_count(self, many_tests_files, testdir):
@@ -1745,6 +1808,22 @@ class TestProgressOutputStyle:
                 r"\[gw\d\] \[\s*\d+%\] PASSED test_foobar.py::test_foobar\[1\]",
             ]
         )
+        output.stdout.fnmatch_lines_random(
+            [
+                line.translate(TRANS_FNMATCH)
+                for line in [
+                    "test_bar.py::test_bar[0] ",
+                    "test_foo.py::test_foo[0] ",
+                    "test_foobar.py::test_foobar[0] ",
+                    "[gw?] [  5%] PASSED test_*[?] ",
+                    "[gw?] [ 10%] PASSED test_*[?] ",
+                    "[gw?] [ 55%] PASSED test_*[?] ",
+                    "[gw?] [ 60%] PASSED test_*[?] ",
+                    "[gw?] [ 95%] PASSED test_*[?] ",
+                    "[gw?] [100%] PASSED test_*[?] ",
+                ]
+            ]
+        )
 
     def test_capture_no(self, many_tests_files, testdir):
         output = testdir.runpytest("-s")
@@ -1805,12 +1884,16 @@ class TestProgressWithTeardown:
         testdir.makepyfile(
             """
             def test_foo(fail_teardown):
-                assert False
+                assert 0
         """
         )
-        output = testdir.runpytest()
+        output = testdir.runpytest("-rfE")
         output.stdout.re_match_lines(
-            [r"test_teardown_with_test_also_failing.py FE\s+\[100%\]"]
+            [
+                r"test_teardown_with_test_also_failing.py FE\s+\[100%\]",
+                "FAILED test_teardown_with_test_also_failing.py::test_foo - assert 0",
+                "ERROR test_teardown_with_test_also_failing.py::test_foo - assert False",
+            ]
         )
 
     def test_teardown_many(self, testdir, many_files):
@@ -1819,15 +1902,21 @@ class TestProgressWithTeardown:
             [r"test_bar.py (\.E){5}\s+\[ 25%\]", r"test_foo.py (\.E){15}\s+\[100%\]"]
         )
 
-    def test_teardown_many_verbose(self, testdir, many_files):
-        output = testdir.runpytest("-v")
-        output.stdout.re_match_lines(
-            [
-                r"test_bar.py::test_bar\[0\] PASSED\s+\[  5%\]",
-                r"test_bar.py::test_bar\[0\] ERROR\s+\[  5%\]",
-                r"test_bar.py::test_bar\[4\] PASSED\s+\[ 25%\]",
-                r"test_bar.py::test_bar\[4\] ERROR\s+\[ 25%\]",
-            ]
+    def test_teardown_many_verbose(
+        self, testdir: Testdir, many_files, color_mapping
+    ) -> None:
+        result = testdir.runpytest("-v")
+        result.stdout.fnmatch_lines(
+            color_mapping.format_for_fnmatch(
+                [
+                    "test_bar.py::test_bar[0] PASSED  * [  5%]",
+                    "test_bar.py::test_bar[0] ERROR   * [  5%]",
+                    "test_bar.py::test_bar[4] PASSED  * [ 25%]",
+                    "test_foo.py::test_foo[14] PASSED * [100%]",
+                    "test_foo.py::test_foo[14] ERROR  * [100%]",
+                    "=* 20 passed, 20 errors in *",
+                ]
+            )
         )
 
     def test_xdist_normal(self, many_files, testdir, monkeypatch):
@@ -1971,3 +2060,54 @@ def test_collecterror(testdir):
             "*= 1 error in *",
         ]
     )
+
+
+def test_via_exec(testdir: Testdir) -> None:
+    p1 = testdir.makepyfile("exec('def test_via_exec(): pass')")
+    result = testdir.runpytest(str(p1), "-vv")
+    result.stdout.fnmatch_lines(
+        ["test_via_exec.py::test_via_exec <- <string> PASSED*", "*= 1 passed in *"]
+    )
+
+
+class TestCodeHighlight:
+    def test_code_highlight_simple(self, testdir: Testdir, color_mapping) -> None:
+        testdir.makepyfile(
+            """
+            def test_foo():
+                assert 1 == 10
+        """
+        )
+        result = testdir.runpytest("--color=yes")
+        color_mapping.requires_ordered_markup(result)
+        result.stdout.fnmatch_lines(
+            color_mapping.format_for_fnmatch(
+                [
+                    "    {kw}def{hl-reset} {function}test_foo{hl-reset}():",
+                    ">       {kw}assert{hl-reset} {number}1{hl-reset} == {number}10{hl-reset}",
+                    "{bold}{red}E       assert 1 == 10{reset}",
+                ]
+            )
+        )
+
+    def test_code_highlight_continuation(self, testdir: Testdir, color_mapping) -> None:
+        testdir.makepyfile(
+            """
+            def test_foo():
+                print('''
+                '''); assert 0
+        """
+        )
+        result = testdir.runpytest("--color=yes")
+        color_mapping.requires_ordered_markup(result)
+
+        result.stdout.fnmatch_lines(
+            color_mapping.format_for_fnmatch(
+                [
+                    "    {kw}def{hl-reset} {function}test_foo{hl-reset}():",
+                    "        {print}print{hl-reset}({str}'''{hl-reset}{str}{hl-reset}",
+                    ">   {str}    {hl-reset}{str}'''{hl-reset}); {kw}assert{hl-reset} {number}0{hl-reset}",
+                    "{bold}{red}E       assert 0{reset}",
+                ]
+            )
+        )

testing/test_tmpdir.py

@@ -74,19 +74,38 @@ class TestConfigTmpdir:
         assert not mytemp.join("hello").check()
 
 
-def test_basetemp(testdir):
+testdata = [
+    ("mypath", True),
+    ("/mypath1", False),
+    ("./mypath1", True),
+    ("../mypath3", False),
+    ("../../mypath4", False),
+    ("mypath5/..", False),
+    ("mypath6/../mypath6", True),
+    ("mypath7/../mypath7/..", False),
+]
+
+
+@pytest.mark.parametrize("basename, is_ok", testdata)
+def test_mktemp(testdir, basename, is_ok):
     mytemp = testdir.tmpdir.mkdir("mytemp")
     p = testdir.makepyfile(
         """
         import pytest
-        def test_1(tmpdir_factory):
-            tmpdir_factory.mktemp('hello', numbered=False)
-    """
+        def test_abs_path(tmpdir_factory):
+            tmpdir_factory.mktemp('{}', numbered=False)
+        """.format(
+            basename
+        )
     )
+
     result = testdir.runpytest(p, "--basetemp=%s" % mytemp)
-    assert result.ret == 0
-    print(mytemp)
-    assert mytemp.join("hello").check()
+    if is_ok:
+        assert result.ret == 0
+        assert mytemp.join(basename).check()
+    else:
+        assert result.ret == 1
+        result.stdout.fnmatch_lines("*ValueError*")
 
 
 def test_tmpdir_always_is_realpath(testdir):

testing/test_unittest.py

@@ -1,7 +1,7 @@
 import gc
 
 import pytest
-from _pytest.main import ExitCode
+from _pytest.config import ExitCode
 
 
 def test_simple_unittest(testdir):
@@ -537,24 +537,28 @@ class TestTrialUnittest:
         )
         result.stdout.fnmatch_lines(
             [
-                "test_trial_error.py::TC::test_four FAILED",
+                "test_trial_error.py::TC::test_four SKIPPED",
                 "test_trial_error.py::TC::test_four ERROR",
                 "test_trial_error.py::TC::test_one FAILED",
                 "test_trial_error.py::TC::test_three FAILED",
-                "test_trial_error.py::TC::test_two FAILED",
+                "test_trial_error.py::TC::test_two SKIPPED",
+                "test_trial_error.py::TC::test_two ERROR",
                 "*ERRORS*",
                 "*_ ERROR at teardown of TC.test_four _*",
+                "NOTE: Incompatible Exception Representation, displaying natively:",
+                "*DelayedCalls*",
+                "*_ ERROR at teardown of TC.test_two _*",
+                "NOTE: Incompatible Exception Representation, displaying natively:",
                 "*DelayedCalls*",
                 "*= FAILURES =*",
-                "*_ TC.test_four _*",
-                "*NameError*crash*",
+                # "*_ TC.test_four _*",
+                # "*NameError*crash*",
                 "*_ TC.test_one _*",
                 "*NameError*crash*",
                 "*_ TC.test_three _*",
+                "NOTE: Incompatible Exception Representation, displaying natively:",
                 "*DelayedCalls*",
-                "*_ TC.test_two _*",
-                "*NameError*crash*",
-                "*= 4 failed, 1 error in *",
+                "*= 2 failed, 2 skipped, 2 errors in *",
             ]
         )
 
@@ -1096,3 +1100,32 @@ def test_exit_outcome(testdir):
     )
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["*Exit: pytest_exit called*", "*= no tests ran in *"])
+
+
+def test_trace(testdir, monkeypatch):
+    calls = []
+
+    def check_call(*args, **kwargs):
+        calls.append((args, kwargs))
+        assert args == ("runcall",)
+
+        class _pdb:
+            def runcall(*args, **kwargs):
+                calls.append((args, kwargs))
+
+        return _pdb
+
+    monkeypatch.setattr("_pytest.debugging.pytestPDB._init_pdb", check_call)
+
+    p1 = testdir.makepyfile(
+        """
+        import unittest
+
+        class MyTestCase(unittest.TestCase):
+            def test(self):
+                self.assertEqual('foo', 'foo')
+    """
+    )
+    result = testdir.runpytest("--trace", str(p1))
+    assert len(calls) == 2
+    assert result.ret == 0

testing/test_warnings.py

@@ -1,3 +1,4 @@
+import os
 import warnings
 
 import pytest
@@ -599,3 +600,136 @@ def test_pytest_configure_warning(testdir, recwarn):
     assert "INTERNALERROR" not in result.stderr.str()
     warning = recwarn.pop()
     assert str(warning.message) == "from pytest_configure"
+
+
+class TestStackLevel:
+    @pytest.fixture
+    def capwarn(self, testdir):
+        class CapturedWarnings:
+            captured = []
+
+            @classmethod
+            def pytest_warning_captured(cls, warning_message, when, item, location):
+                cls.captured.append((warning_message, location))
+
+        testdir.plugins = [CapturedWarnings()]
+
+        return CapturedWarnings
+
+    def test_issue4445_rewrite(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.assertion.rewrite.py:241
+        """
+        testdir.makepyfile(some_mod="")
+        conftest = testdir.makeconftest(
+            """
+                import some_mod
+                import pytest
+
+                pytest.register_assert_rewrite("some_mod")
+            """
+        )
+        testdir.parseconfig()
+
+        # with stacklevel=5 the warning originates from register_assert_rewrite
+        # function in the created conftest.py
+        assert len(capwarn.captured) == 1
+        warning, location = capwarn.captured.pop()
+        file, lineno, func = location
+
+        assert "Module already imported" in str(warning.message)
+        assert file == str(conftest)
+        assert func == "<module>"  # the above conftest.py
+        assert lineno == 4
+
+    def test_issue4445_preparse(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.config.__init__.py:910
+        """
+        testdir.makeconftest(
+            """
+            import nothing
+            """
+        )
+        testdir.parseconfig("--help")
+
+        # with stacklevel=2 the warning should originate from config._preparse and is
+        # thrown by an errorneous conftest.py
+        assert len(capwarn.captured) == 1
+        warning, location = capwarn.captured.pop()
+        file, _, func = location
+
+        assert "could not load initial conftests" in str(warning.message)
+        assert "config{sep}__init__.py".format(sep=os.sep) in file
+        assert func == "_preparse"
+
+    def test_issue4445_import_plugin(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.config.__init__.py:585
+        """
+        testdir.makepyfile(
+            some_plugin="""
+            import pytest
+            pytest.skip("thing", allow_module_level=True)
+            """
+        )
+        testdir.syspathinsert()
+        testdir.parseconfig("-p", "some_plugin")
+
+        # with stacklevel=2 the warning should originate from
+        # config.PytestPluginManager.import_plugin is thrown by a skipped plugin
+
+        assert len(capwarn.captured) == 1
+        warning, location = capwarn.captured.pop()
+        file, _, func = location
+
+        assert "skipped plugin 'some_plugin': thing" in str(warning.message)
+        assert "config{sep}__init__.py".format(sep=os.sep) in file
+        assert func == "import_plugin"
+
+    def test_issue4445_resultlog(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.resultlog.py:35
+        """
+        testdir.makepyfile(
+            """
+            def test_dummy():
+                pass
+        """
+        )
+        # Use parseconfigure() because the warning in resultlog.py is triggered in
+        # the pytest_configure hook
+        testdir.parseconfigure(
+            "--result-log={dir}".format(dir=testdir.tmpdir.join("result.log"))
+        )
+
+        # with stacklevel=2 the warning originates from resultlog.pytest_configure
+        # and is thrown when --result-log is used
+        warning, location = capwarn.captured.pop()
+        file, _, func = location
+
+        assert "--result-log is deprecated" in str(warning.message)
+        assert "resultlog.py" in file
+        assert func == "pytest_configure"
+
+    def test_issue4445_issue5928_mark_generator(self, testdir):
+        """#4445 and #5928: Make sure the warning from an unknown mark points to
+        the test file where this mark is used.
+        """
+        testfile = testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.unknown
+            def test_it():
+                pass
+            """
+        )
+        result = testdir.runpytest_subprocess()
+        # with stacklevel=2 the warning should originate from the above created test file
+        result.stdout.fnmatch_lines_random(
+            [
+                "*{testfile}:3*".format(testfile=str(testfile)),
+                "*Unknown pytest.mark.unknown*",
+            ]
+        )