From d3e363b97a2a27fecc3c049bcd1fae9fa4ac34f5 Mon Sep 17 00:00:00 2001
From: holger krekel <holger@merlinux.eu>
Date: Tue, 28 Apr 2015 11:54:45 +0200
Subject: [PATCH] - make API between runpytest() and inline_run() more similar
 - shift a number of tests to become inline_run() tests

--HG--
branch : testrefactor
---
 _pytest/config.py          |  24 ++++----
 _pytest/main.py            |   5 +-
 _pytest/pytester.py        |  35 ++++++++++--
 testing/python/collect.py  |  36 ++++++------
 testing/python/fixture.py  | 114 ++++++++++++++++++-------------------
 testing/python/metafunc.py |  60 +++++++++----------
 testing/test_assertion.py  |   2 +-
 testing/test_config.py     |  10 ++--
 testing/test_nose.py       |  36 ++++++------
 9 files changed, 169 insertions(+), 153 deletions(-)

diff --git a/_pytest/config.py b/_pytest/config.py
index f37d417b2..fcad07146 100644
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -29,17 +29,21 @@ def main(args=None, plugins=None):
                   initialization.
     """
     try:
-        config = _prepareconfig(args, plugins)
-    except ConftestImportFailure:
-        e = sys.exc_info()[1]
-        tw = py.io.TerminalWriter(sys.stderr)
-        for line in traceback.format_exception(*e.excinfo):
-            tw.line(line.rstrip(), red=True)
-        tw.line("ERROR: could not load %s\n" % (e.path), red=True)
+        try:
+            config = _prepareconfig(args, plugins)
+        except ConftestImportFailure as e:
+            tw = py.io.TerminalWriter(sys.stderr)
+            for line in traceback.format_exception(*e.excinfo):
+                tw.line(line.rstrip(), red=True)
+            tw.line("ERROR: could not load %s\n" % (e.path), red=True)
+            return 4
+        else:
+            config.pluginmanager.check_pending()
+            return config.hook.pytest_cmdline_main(config=config)
+    except UsageError as e:
+        for msg in e.args:
+            sys.stderr.write("ERROR: %s\n" %(msg,))
         return 4
-    else:
-        config.pluginmanager.check_pending()
-        return config.hook.pytest_cmdline_main(config=config)
 
 class cmdline:  # compatibility namespace
     main = staticmethod(main)
diff --git a/_pytest/main.py b/_pytest/main.py
index b13da3529..737b7f20a 100644
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -83,10 +83,7 @@ def wrap_session(config, doit):
             initstate = 2
             doit(config, session)
         except pytest.UsageError:
-            args = sys.exc_info()[1].args
-            for msg in args:
-                sys.stderr.write("ERROR: %s\n" %(msg,))
-            session.exitstatus = EXIT_USAGEERROR
+            raise
         except KeyboardInterrupt:
             excinfo = py.code.ExceptionInfo()
             config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
index 09517e8ff..b99fb709e 100644
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -204,6 +204,8 @@ def pytest_funcarg__testdir(request):
     tmptestdir = TmpTestdir(request)
     return tmptestdir
 
+
+
 rex_outcome = re.compile("(\d+) (\w+)")
 class RunResult:
     """The result of running a command.
@@ -229,6 +231,8 @@ class RunResult:
         self.duration = duration
 
     def parseoutcomes(self):
+        """ Return a dictionary of outcomestring->num from parsing
+        the terminal output that the test process produced."""
         for line in reversed(self.outlines):
             if 'seconds' in line:
                 outcomes = rex_outcome.findall(line)
@@ -238,13 +242,16 @@ class RunResult:
                         d[cat] = int(num)
                     return d
 
-    def assertoutcome(self, passed=0, skipped=0, failed=0):
+    def assert_outcomes(self, passed=0, skipped=0, failed=0):
+        """ assert that the specified outcomes appear with the respective
+        numbers (0 means it didn't occur) in the text output from a test run."""
         d = self.parseoutcomes()
         assert passed == d.get("passed", 0)
         assert skipped == d.get("skipped", 0)
         assert failed == d.get("failed", 0)
 
 
+
 class TmpTestdir:
     """Temporary test directory with tools to test/run py.test itself.
 
@@ -568,12 +575,32 @@ class TmpTestdir:
         plugins = kwargs.get("plugins") or []
         plugins.append(Collect())
         ret = pytest.main(list(args), plugins=plugins)
-        assert len(rec) == 1
-        reprec = rec[0]
-        reprec.ret = ret
         self.delete_loaded_modules()
+        if len(rec) == 1:
+            reprec = rec.pop()
+        else:
+            class reprec:
+                pass
+        reprec.ret = ret
         return reprec
 
+    def inline_runpytest(self, *args):
+        """ Return result of running pytest in-process, providing a similar
+        interface to what self.runpytest() provides. """
+        now = time.time()
+        capture = py.io.StdCaptureFD()
+        try:
+            reprec = self.inline_run(*args)
+        finally:
+            out, err = capture.reset()
+        assert out or err
+
+        res = RunResult(reprec.ret,
+                        out.split("\n"), err.split("\n"),
+                        time.time()-now)
+        res.reprec = reprec
+        return res
+
     def parseconfig(self, *args):
         """Return a new py.test Config instance from given commandline args.
 
diff --git a/testing/python/collect.py b/testing/python/collect.py
index 6629d0833..998ee8bc9 100644
--- a/testing/python/collect.py
+++ b/testing/python/collect.py
@@ -15,7 +15,7 @@ class TestModule:
         p.pyimport()
         del py.std.sys.modules['test_whatever']
         b.ensure("test_whatever.py")
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*import*mismatch*",
             "*imported*test_whatever*",
@@ -59,7 +59,7 @@ class TestClass:
                 def __init__(self):
                     pass
         """)
-        result = testdir.runpytest("-rw")
+        result = testdir.inline_runpytest("-rw")
         result.stdout.fnmatch_lines_random("""
             WC1*test_class_with_init_warning.py*__init__*
         """)
@@ -69,7 +69,7 @@ class TestClass:
             class test(object):
                 pass
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*collected 0*",
         ])
@@ -86,7 +86,7 @@ class TestClass:
                 def teardown_class(cls):
                     pass
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*1 passed*",
         ])
@@ -534,7 +534,7 @@ class TestConftestCustomization:
         """)
         testdir.makepyfile("def test_some(): pass")
         testdir.makepyfile(test_xyz="def test_func(): pass")
-        result = testdir.runpytest("--collect-only")
+        result = testdir.inline_runpytest("--collect-only")
         result.stdout.fnmatch_lines([
             "*<Module*test_pytest*",
             "*<MyModule*xyz*",
@@ -590,7 +590,7 @@ class TestConftestCustomization:
                     return MyFunction(name, collector)
         """)
         testdir.makepyfile("def some(): pass")
-        result = testdir.runpytest("--collect-only")
+        result = testdir.inline_runpytest("--collect-only")
         result.stdout.fnmatch_lines([
             "*MyFunction*some*",
         ])
@@ -626,7 +626,7 @@ def test_setup_only_available_in_subdir(testdir):
     """))
     sub1.join("test_in_sub1.py").write("def test_1(): pass")
     sub2.join("test_in_sub2.py").write("def test_2(): pass")
-    result = testdir.runpytest("-v", "-s")
+    result = testdir.inline_runpytest("-v", "-s")
     result.stdout.fnmatch_lines([
         "*2 passed*"
     ])
@@ -650,7 +650,7 @@ class TestTracebackCutting:
                 raise ValueError("xyz")
         """)
         p = testdir.makepyfile("def test(hello): pass")
-        result = testdir.runpytest(p)
+        result = testdir.inline_runpytest(p)
         assert result.ret != 0
         out = result.stdout.str()
         assert out.find("xyz") != -1
@@ -658,7 +658,7 @@ class TestTracebackCutting:
         numentries = out.count("_ _ _") # separator for traceback entries
         assert numentries == 0
 
-        result = testdir.runpytest("--fulltrace", p)
+        result = testdir.inline_runpytest("--fulltrace", p)
         out = result.stdout.str()
         assert out.find("conftest.py:2: ValueError") != -1
         numentries = out.count("_ _ _ _") # separator for traceback entries
@@ -671,7 +671,7 @@ class TestTracebackCutting:
             x = 17
             asd
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret != 0
         out = result.stdout.str()
         assert "x = 1" not in out
@@ -680,7 +680,7 @@ class TestTracebackCutting:
             " *asd*",
             "E*NameError*",
         ])
-        result = testdir.runpytest("--fulltrace")
+        result = testdir.inline_runpytest("--fulltrace")
         out = result.stdout.str()
         assert "x = 1" in out
         assert "x = 2" in out
@@ -771,7 +771,7 @@ def test_customized_python_discovery(testdir):
     """)
     p2 = p.new(basename=p.basename.replace("test", "check"))
     p.move(p2)
-    result = testdir.runpytest("--collect-only", "-s")
+    result = testdir.inline_runpytest("--collect-only", "-s")
     result.stdout.fnmatch_lines([
         "*check_customized*",
         "*check_simple*",
@@ -779,7 +779,7 @@ def test_customized_python_discovery(testdir):
         "*check_meth*",
     ])
 
-    result = testdir.runpytest()
+    result = testdir.inline_runpytest()
     assert result.ret == 0
     result.stdout.fnmatch_lines([
         "*2 passed*",
@@ -795,12 +795,12 @@ def test_customized_python_discovery_functions(testdir):
         def _test_underscore():
             pass
     """)
-    result = testdir.runpytest("--collect-only", "-s")
+    result = testdir.inline_runpytest("--collect-only", "-s")
     result.stdout.fnmatch_lines([
         "*_test_underscore*",
     ])
 
-    result = testdir.runpytest()
+    result = testdir.inline_runpytest()
     assert result.ret == 0
     result.stdout.fnmatch_lines([
         "*1 passed*",
@@ -820,7 +820,7 @@ def test_collector_attributes(testdir):
          def test_hello():
             pass
     """)
-    result = testdir.runpytest()
+    result = testdir.inline_runpytest()
     result.stdout.fnmatch_lines([
         "*1 passed*",
     ])
@@ -844,7 +844,7 @@ def test_customize_through_attributes(testdir):
             def test_hello(self):
                 pass
     """)
-    result = testdir.runpytest("--collect-only")
+    result = testdir.inline_runpytest("--collect-only")
     result.stdout.fnmatch_lines([
         "*MyClass*",
         "*MyInstance*",
@@ -864,6 +864,6 @@ def test_unorderable_types(testdir):
             return Test
         TestFoo = make_test()
     """)
-    result = testdir.runpytest()
+    result = testdir.inline_runpytest()
     assert "TypeError" not in result.stdout.str()
     assert result.ret == 0
diff --git a/testing/python/fixture.py b/testing/python/fixture.py
index 274f80965..0eee3b8a8 100644
--- a/testing/python/fixture.py
+++ b/testing/python/fixture.py
@@ -33,7 +33,7 @@ class TestFillFixtures:
             def test_func(some):
                 pass
         """)
-        result = testdir.runpytest() # "--collect-only")
+        result = testdir.inline_runpytest() # "--collect-only")
         assert result.ret != 0
         result.stdout.fnmatch_lines([
             "*def test_func(some)*",
@@ -78,7 +78,7 @@ class TestFillFixtures:
                 def test_method(self, something):
                     assert something is self
         """)
-        result = testdir.runpytest(p)
+        result = testdir.inline_runpytest(p)
         result.stdout.fnmatch_lines([
             "*1 passed*"
         ])
@@ -99,7 +99,7 @@ class TestFillFixtures:
 
         sub1.join("test_in_sub1.py").write("def test_1(arg1): pass")
         sub2.join("test_in_sub2.py").write("def test_2(arg2): pass")
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         result.stdout.fnmatch_lines([
             "*2 passed*"
         ])
@@ -121,9 +121,9 @@ class TestFillFixtures:
                  def test_spam(self, spam):
                      assert spam == 'spamspam'
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        result = testdir.runpytest(testfile)
+        result = testdir.inline_runpytest(testfile)
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_extend_fixture_conftest_module(self, testdir):
@@ -144,9 +144,9 @@ class TestFillFixtures:
             def test_spam(spam):
                 assert spam == 'spamspam'
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        result = testdir.runpytest(testfile)
+        result = testdir.inline_runpytest(testfile)
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_extend_fixture_conftest_conftest(self, testdir):
@@ -170,9 +170,9 @@ class TestFillFixtures:
             def test_spam(spam):
                 assert spam == "spamspam"
         """))
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        result = testdir.runpytest(testfile)
+        result = testdir.inline_runpytest(testfile)
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_extend_fixture_conftest_plugin(self, testdir):
@@ -197,7 +197,7 @@ class TestFillFixtures:
             def test_foo(foo):
                 assert foo == 14
         """)
-        result = testdir.runpytest('-s')
+        result = testdir.inline_runpytest('-s')
         assert result.ret == 0
 
     def test_extend_fixture_plugin_plugin(self, testdir):
@@ -223,7 +223,7 @@ class TestFillFixtures:
             def test_foo(foo):
                 assert foo == 14
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret == 0
 
     def test_override_parametrized_fixture_conftest_module(self, testdir):
@@ -245,9 +245,9 @@ class TestFillFixtures:
             def test_spam(spam):
                 assert spam == 'spam'
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        result = testdir.runpytest(testfile)
+        result = testdir.inline_runpytest(testfile)
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_override_parametrized_fixture_conftest_conftest(self, testdir):
@@ -272,9 +272,9 @@ class TestFillFixtures:
             def test_spam(spam):
                 assert spam == "spam"
         """))
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        result = testdir.runpytest(testfile)
+        result = testdir.inline_runpytest(testfile)
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_override_non_parametrized_fixture_conftest_module(self, testdir):
@@ -299,9 +299,9 @@ class TestFillFixtures:
                 assert spam == params['spam']
                 params['spam'] += 1
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines(["*3 passed*"])
-        result = testdir.runpytest(testfile)
+        result = testdir.inline_runpytest(testfile)
         result.stdout.fnmatch_lines(["*3 passed*"])
 
     def test_override_non_parametrized_fixture_conftest_conftest(self, testdir):
@@ -329,9 +329,9 @@ class TestFillFixtures:
                 assert spam == params['spam']
                 params['spam'] += 1
         """))
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines(["*3 passed*"])
-        result = testdir.runpytest(testfile)
+        result = testdir.inline_runpytest(testfile)
         result.stdout.fnmatch_lines(["*3 passed*"])
 
     def test_autouse_fixture_plugin(self, testdir):
@@ -351,7 +351,7 @@ class TestFillFixtures:
             def test_foo(request):
                 assert request.function.foo == 7
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret == 0
 
     def test_funcarg_lookup_error(self, testdir):
@@ -359,7 +359,7 @@ class TestFillFixtures:
             def test_lookup_error(unknown):
                 pass
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*ERROR*test_lookup_error*",
             "*def test_lookup_error(unknown):*",
@@ -388,7 +388,7 @@ class TestFillFixtures:
                     traceback.print_exc()
                 assert sys.exc_info() == (None, None, None)
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret == 0
 
 
@@ -531,7 +531,7 @@ class TestRequestBasic:
             def test_second():
                 assert len(l) == 1
         """)
-        result = testdir.runpytest(p)
+        result = testdir.inline_runpytest(p)
         result.stdout.fnmatch_lines([
             "*1 error*"  # XXX the whole module collection fails
             ])
@@ -616,7 +616,7 @@ class TestRequestBasic:
         """))
         p = b.join("test_module.py")
         p.write("def test_func(arg1): pass")
-        result = testdir.runpytest(p, "--fixtures")
+        result = testdir.inline_runpytest(p, "--fixtures")
         assert result.ret == 0
         result.stdout.fnmatch_lines("""
             *fixtures defined*conftest*
@@ -785,7 +785,7 @@ class TestRequestCachedSetup:
             def test_two_different_setups(arg1, arg2):
                 assert arg1 != arg2
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         result.stdout.fnmatch_lines([
             "*1 passed*"
         ])
@@ -800,7 +800,7 @@ class TestRequestCachedSetup:
             def test_two_funcarg(arg1):
                 assert arg1 == 11
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         result.stdout.fnmatch_lines([
             "*1 passed*"
         ])
@@ -827,7 +827,7 @@ class TestRequestCachedSetup:
             def test_check_test0_has_teardown_correct():
                 assert test_0.l == [2]
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         result.stdout.fnmatch_lines([
             "*3 passed*"
         ])
@@ -843,7 +843,7 @@ class TestRequestCachedSetup:
             def test_func(app):
                 pass
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret != 0
         result.stdout.fnmatch_lines([
             "*3/x*",
@@ -898,7 +898,7 @@ class TestFixtureUsages:
             def test_add(arg2):
                 assert arg2 == 2
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*ScopeMismatch*involved factories*",
             "* def arg2*",
@@ -920,7 +920,7 @@ class TestFixtureUsages:
             def test_add(arg1, arg2):
                 assert arg2 == 2
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*ScopeMismatch*involved factories*",
             "* def arg2*",
@@ -944,7 +944,7 @@ class TestFixtureUsages:
                 assert arg2 == arg1 + 1
                 assert len(l) == arg1
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*2 passed*"
         ])
@@ -964,7 +964,7 @@ class TestFixtureUsages:
             def test_missing(call_fail):
                 pass
             """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines("""
             *pytest.fixture()*
             *def call_fail(fail)*
@@ -1046,7 +1046,7 @@ class TestFixtureUsages:
         reprec.assertoutcome(passed=2)
 
     def test_usefixtures_seen_in_showmarkers(self, testdir):
-        result = testdir.runpytest("--markers")
+        result = testdir.inline_runpytest("--markers")
         result.stdout.fnmatch_lines("""
             *usefixtures(fixturename1*mark tests*fixtures*
         """)
@@ -1313,7 +1313,7 @@ class TestAutouseDiscovery:
         conftest.move(a.join(conftest.basename))
         a.join("test_something.py").write("def test_func(): pass")
         b.join("test_otherthing.py").write("def test_func(): pass")
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines("""
             *1 passed*1 error*
         """)
@@ -1767,7 +1767,7 @@ class TestFixtureMarker:
                 def test_1(arg):
                     pass
             """ % method)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret != 0
         result.stdout.fnmatch_lines([
             "*ScopeMismatch*You tried*function*session*request*",
@@ -1825,7 +1825,7 @@ class TestFixtureMarker:
             def test_mismatch(arg):
                 pass
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*ScopeMismatch*",
             "*1 error*",
@@ -1876,7 +1876,7 @@ class TestFixtureMarker:
             def test_func4(marg):
                 pass
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         result.stdout.fnmatch_lines("""
             test_mod1.py::test_func[s1] PASSED
             test_mod2.py::test_func2[s1] PASSED
@@ -1928,7 +1928,7 @@ class TestFixtureMarker:
                 def test_3(self):
                     pass
         """)
-        result = testdir.runpytest("-vs")
+        result = testdir.inline_runpytest("-vs")
         result.stdout.fnmatch_lines("""
             test_class_ordering.py::TestClass2::test_1[1-a] PASSED
             test_class_ordering.py::TestClass2::test_1[2-a] PASSED
@@ -2019,7 +2019,7 @@ class TestFixtureMarker:
             def test_finish():
                 assert not l
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         result.stdout.fnmatch_lines("""
             *3 passed*
         """)
@@ -2049,7 +2049,7 @@ class TestFixtureMarker:
             def test_browser(browser):
                 assert browser['visited'] is True
         """))
-        reprec = testdir.runpytest("-s")
+        reprec = testdir.inline_runpytest("-s")
         for test in ['test_browser']:
             reprec.stdout.fnmatch_lines('*Finalized*')
 
@@ -2260,7 +2260,7 @@ class TestFixtureMarker:
             def test_foo(fix):
                 assert 1
         """)
-        res = testdir.runpytest('-v')
+        res = testdir.inline_runpytest('-v')
         res.stdout.fnmatch_lines([
             '*test_foo*alpha*',
             '*test_foo*beta*'])
@@ -2277,7 +2277,7 @@ class TestFixtureMarker:
             def test_foo(fix):
                 assert 1
         """)
-        res = testdir.runpytest('-v')
+        res = testdir.inline_runpytest('-v')
         res.stdout.fnmatch_lines([
             '*test_foo*alpha*',
             '*test_foo*beta*'])
@@ -2337,7 +2337,7 @@ class TestErrors:
             def test_something(gen):
                 pass
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret != 0
         result.stdout.fnmatch_lines([
             "*def gen(qwe123):*",
@@ -2363,7 +2363,7 @@ class TestErrors:
             def test_3():
                 assert l[0] != l[1]
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines("""
             *ERROR*teardown*test_1*
             *KeyError*
@@ -2383,7 +2383,7 @@ class TestErrors:
             def test_something():
                 pass
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret != 0
         result.stdout.fnmatch_lines([
             "*def gen(qwe123):*",
@@ -2397,7 +2397,7 @@ class TestShowFixtures:
         assert config.option.showfixtures
 
     def test_show_fixtures(self, testdir):
-        result = testdir.runpytest("--fixtures")
+        result = testdir.inline_runpytest("--fixtures")
         result.stdout.fnmatch_lines([
                 "*tmpdir*",
                 "*temporary directory*",
@@ -2405,7 +2405,7 @@ class TestShowFixtures:
         )
 
     def test_show_fixtures_verbose(self, testdir):
-        result = testdir.runpytest("--fixtures", "-v")
+        result = testdir.inline_runpytest("--fixtures", "-v")
         result.stdout.fnmatch_lines([
                 "*tmpdir*--*tmpdir.py*",
                 "*temporary directory*",
@@ -2422,7 +2422,7 @@ class TestShowFixtures:
             def arg1():
                 """  hello world """
         ''')
-        result = testdir.runpytest("--fixtures", p)
+        result = testdir.inline_runpytest("--fixtures", p)
         result.stdout.fnmatch_lines("""
             *tmpdir
             *fixtures defined from*
@@ -2444,7 +2444,7 @@ class TestShowFixtures:
                 def test_hello():
                     pass
             """)
-        result = testdir.runpytest("--fixtures")
+        result = testdir.inline_runpytest("--fixtures")
         result.stdout.fnmatch_lines("""
             *tmpdir*
             *fixtures defined from*conftest*
@@ -2470,7 +2470,7 @@ class TestShowFixtures:
 
                 """
         ''')
-        result = testdir.runpytest("--fixtures", p)
+        result = testdir.inline_runpytest("--fixtures", p)
         result.stdout.fnmatch_lines("""
             * fixtures defined from test_show_fixtures_trimmed_doc *
             arg2
@@ -2498,7 +2498,7 @@ class TestContextManagerFixtureFuncs:
                 print ("test2 %s" % arg1)
                 assert 0
         """)
-        result = testdir.runpytest("-s")
+        result = testdir.inline_runpytest("-s")
         result.stdout.fnmatch_lines("""
             *setup*
             *test1 1*
@@ -2521,7 +2521,7 @@ class TestContextManagerFixtureFuncs:
             def test_2(arg1):
                 print ("test2 %s" % arg1)
         """)
-        result = testdir.runpytest("-s")
+        result = testdir.inline_runpytest("-s")
         result.stdout.fnmatch_lines("""
             *setup*
             *test1 1*
@@ -2539,7 +2539,7 @@ class TestContextManagerFixtureFuncs:
             def test_1(arg1):
                 pass
         """)
-        result = testdir.runpytest("-s")
+        result = testdir.inline_runpytest("-s")
         result.stdout.fnmatch_lines("""
             *pytest.fail*setup*
             *1 error*
@@ -2555,7 +2555,7 @@ class TestContextManagerFixtureFuncs:
             def test_1(arg1):
                 pass
         """)
-        result = testdir.runpytest("-s")
+        result = testdir.inline_runpytest("-s")
         result.stdout.fnmatch_lines("""
             *pytest.fail*teardown*
             *1 passed*1 error*
@@ -2571,7 +2571,7 @@ class TestContextManagerFixtureFuncs:
             def test_1(arg1):
                 pass
         """)
-        result = testdir.runpytest("-s")
+        result = testdir.inline_runpytest("-s")
         result.stdout.fnmatch_lines("""
             *fixture function*
             *test_yields*:2*
@@ -2587,7 +2587,7 @@ class TestContextManagerFixtureFuncs:
             def test_1(arg1):
                 pass
         """)
-        result = testdir.runpytest("-s")
+        result = testdir.inline_runpytest("-s")
         result.stdout.fnmatch_lines("""
             *yield_fixture*requires*yield*
             *yield_fixture*
@@ -2603,7 +2603,7 @@ class TestContextManagerFixtureFuncs:
             def test_1(arg1):
                 pass
         """)
-        result = testdir.runpytest("-s")
+        result = testdir.inline_runpytest("-s")
         result.stdout.fnmatch_lines("""
             *fixture*cannot use*yield*
             *def arg1*
diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py
index 700e0a790..1675603fe 100644
--- a/testing/python/metafunc.py
+++ b/testing/python/metafunc.py
@@ -246,7 +246,7 @@ class TestMetafunc:
                 assert x in (10,20)
                 assert y == 2
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         result.stdout.fnmatch_lines([
             "*test_simple*1-2*",
             "*test_simple*2-2*",
@@ -290,11 +290,9 @@ class TestMetafunc:
                 def test_meth(self, x, y):
                     assert 0, x
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret == 1
-        result.stdout.fnmatch_lines([
-            "*6 fail*",
-        ])
+        result.assert_outcomes(failed=6)
 
     def test_parametrize_CSV(self, testdir):
         testdir.makepyfile("""
@@ -332,7 +330,7 @@ class TestMetafunc:
                def test_3(self, arg, arg2):
                   pass
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         assert result.ret == 0
         result.stdout.fnmatch_lines("""
             *test_1*1*
@@ -374,8 +372,8 @@ class TestMetafuncFunctional:
                     assert metafunc.function == unbound
                     assert metafunc.cls == TestClass
         """)
-        result = testdir.runpytest(p, "-v")
-        result.assertoutcome(passed=2)
+        result = testdir.inline_runpytest(p, "-v")
+        result.assert_outcomes(passed=2)
 
     def test_addcall_with_two_funcargs_generators(self, testdir):
         testdir.makeconftest("""
@@ -391,7 +389,7 @@ class TestMetafuncFunctional:
                 def test_myfunc(self, arg1, arg2):
                     assert arg1 == arg2
         """)
-        result = testdir.runpytest("-v", p)
+        result = testdir.inline_runpytest("-v", p)
         result.stdout.fnmatch_lines([
             "*test_myfunc*0*PASS*",
             "*test_myfunc*1*FAIL*",
@@ -412,7 +410,7 @@ class TestMetafuncFunctional:
             def test_func2(arg1):
                 assert arg1 in (10, 20)
         """)
-        result = testdir.runpytest("-v", p)
+        result = testdir.inline_runpytest("-v", p)
         result.stdout.fnmatch_lines([
             "*test_func1*0*PASS*",
             "*test_func1*1*FAIL*",
@@ -429,10 +427,8 @@ class TestMetafuncFunctional:
                 def test_hello(xyz):
                     pass
         """)
-        result = testdir.runpytest(p)
-        result.stdout.fnmatch_lines([
-            "*1 pass*",
-        ])
+        result = testdir.inline_runpytest(p)
+        result.assert_outcomes(passed=1)
 
 
     def test_generate_plugin_and_module(self, testdir):
@@ -454,7 +450,7 @@ class TestMetafuncFunctional:
                 def test_myfunc(self, arg1, arg2):
                     assert arg1 == arg2
         """)
-        result = testdir.runpytest("-v", p)
+        result = testdir.inline_runpytest("-v", p)
         result.stdout.fnmatch_lines([
             "*test_myfunc*hello*PASS*",
             "*test_myfunc*world*FAIL*",
@@ -470,7 +466,7 @@ class TestMetafuncFunctional:
                 def test_myfunc(self, hello):
                     assert hello == "world"
         """)
-        result = testdir.runpytest("-v", p)
+        result = testdir.inline_runpytest("-v", p)
         result.stdout.fnmatch_lines([
             "*test_myfunc*hello*PASS*",
             "*1 passed*"
@@ -487,7 +483,7 @@ class TestMetafuncFunctional:
                     assert not hasattr(self, 'x')
                     self.x = 1
         """)
-        result = testdir.runpytest("-v", p)
+        result = testdir.inline_runpytest("-v", p)
         result.stdout.fnmatch_lines([
             "*test_func*0*PASS*",
             "*test_func*1*PASS*",
@@ -505,10 +501,8 @@ class TestMetafuncFunctional:
                 def setup_method(self, func):
                     self.val = 1
             """)
-        result = testdir.runpytest(p)
-        result.stdout.fnmatch_lines([
-            "*1 pass*",
-        ])
+        result = testdir.inline_runpytest(p)
+        result.assert_outcomes(passed=1)
 
     def test_parametrize_functional2(self, testdir):
         testdir.makepyfile("""
@@ -518,7 +512,7 @@ class TestMetafuncFunctional:
             def test_hello(arg1, arg2):
                 assert 0, (arg1, arg2)
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         result.stdout.fnmatch_lines([
             "*(1, 4)*",
             "*(1, 5)*",
@@ -543,7 +537,7 @@ class TestMetafuncFunctional:
             def test_func1(arg1, arg2):
                 assert arg1 == 11
         """)
-        result = testdir.runpytest("-v", p)
+        result = testdir.inline_runpytest("-v", p)
         result.stdout.fnmatch_lines([
             "*test_func1*1*PASS*",
             "*1 passed*"
@@ -564,7 +558,7 @@ class TestMetafuncFunctional:
             def test_func(arg2):
                 assert arg2 == 10
         """)
-        result = testdir.runpytest("-v", p)
+        result = testdir.inline_runpytest("-v", p)
         result.stdout.fnmatch_lines([
             "*test_func*1*PASS*",
             "*1 passed*"
@@ -580,7 +574,7 @@ class TestMetafuncFunctional:
             def test_function(a, b):
                 assert a == b
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         assert result.ret == 1
         result.stdout.fnmatch_lines_random([
             "*test_function*basic*PASSED",
@@ -597,7 +591,7 @@ class TestMetafuncFunctional:
             def test_function(a, b):
                 assert 1
         """)
-        result = testdir.runpytest("-v")
+        result = testdir.inline_runpytest("-v")
         result.stdout.fnmatch_lines("""
             *test_function*1-b0*
             *test_function*1.3-b1*
@@ -653,8 +647,8 @@ class TestMetafuncFunctional:
             def test_function():
                 pass
         """)
-        reprec = testdir.inline_run()
-        reprec.assertoutcome(passed=1)
+        reprec = testdir.inline_runpytest()
+        reprec.assert_outcomes(passed=1)
 
     def test_generate_tests_only_done_in_subdir(self, testdir):
         sub1 = testdir.mkpydir("sub1")
@@ -669,10 +663,8 @@ class TestMetafuncFunctional:
         """))
         sub1.join("test_in_sub1.py").write("def test_1(): pass")
         sub2.join("test_in_sub2.py").write("def test_2(): pass")
-        result = testdir.runpytest("-v", "-s", sub1, sub2, sub1)
-        result.stdout.fnmatch_lines([
-            "*3 passed*"
-        ])
+        result = testdir.inline_runpytest("-v", "-s", sub1, sub2, sub1)
+        result.assert_outcomes(passed=3)
 
     def test_generate_same_function_names_issue403(self, testdir):
         testdir.makepyfile("""
@@ -687,8 +679,8 @@ class TestMetafuncFunctional:
             test_x = make_tests()
             test_y = make_tests()
         """)
-        reprec = testdir.inline_run()
-        reprec.assertoutcome(passed=4)
+        reprec = testdir.inline_runpytest()
+        reprec.assert_outcomes(passed=4)
 
     @pytest.mark.issue463
     def test_parameterize_misspelling(self, testdir):
diff --git a/testing/test_assertion.py b/testing/test_assertion.py
index 1cf00dfed..5ba29edc0 100644
--- a/testing/test_assertion.py
+++ b/testing/test_assertion.py
@@ -451,7 +451,7 @@ def test_assertion_options(testdir):
             x = 3
             assert x == 4
     """)
-    result = testdir.runpytest()
+    result = testdir.inline_runpytest()
     assert "3 == 4" in result.stdout.str()
     off_options = (("--no-assert",),
                    ("--nomagic",),
diff --git a/testing/test_config.py b/testing/test_config.py
index 160276d93..ae9b777c6 100644
--- a/testing/test_config.py
+++ b/testing/test_config.py
@@ -39,7 +39,7 @@ class TestParseIni:
             [pytest]
             minversion=9.0
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.ret != 0
         result.stderr.fnmatch_lines([
             "*tox.ini:2*requires*9.0*actual*"
@@ -75,7 +75,7 @@ class TestParseIni:
             [pytest]
             addopts = --qwe
         """)
-        result = testdir.runpytest("--confcutdir=.")
+        result = testdir.inline_run("--confcutdir=.")
         assert result.ret == 0
 
 class TestConfigCmdlineParsing:
@@ -320,7 +320,7 @@ def test_cmdline_processargs_simple(testdir):
         def pytest_cmdline_preparse(args):
             args.append("-h")
     """)
-    result = testdir.runpytest()
+    result = testdir.inline_runpytest()
     result.stdout.fnmatch_lines([
         "*pytest*",
         "*-h*",
@@ -389,11 +389,11 @@ class TestWarning:
             def test_hello(fix):
                 pass
         """)
-        result = testdir.runpytest()
+        result = testdir.inline_runpytest()
         assert result.parseoutcomes()["warnings"] > 0
         assert "hello" not in result.stdout.str()
 
-        result = testdir.runpytest("-rw")
+        result = testdir.inline_runpytest("-rw")
         result.stdout.fnmatch_lines("""
             ===*warning summary*===
             *WT1*test_warn_on_test_item*:5*hello*
diff --git a/testing/test_nose.py b/testing/test_nose.py
index af878da78..b4b3d8836 100644
--- a/testing/test_nose.py
+++ b/testing/test_nose.py
@@ -18,10 +18,8 @@ def test_nose_setup(testdir):
         test_hello.setup = lambda: l.append(1)
         test_hello.teardown = lambda: l.append(2)
     """)
-    result = testdir.runpytest(p, '-p', 'nose')
-    result.stdout.fnmatch_lines([
-        "*2 passed*"
-    ])
+    result = testdir.inline_runpytest(p, '-p', 'nose')
+    result.assert_outcomes(passed=2)
 
 
 def test_setup_func_with_setup_decorator():
@@ -65,10 +63,8 @@ def test_nose_setup_func(testdir):
             assert l == [1,2]
 
     """)
-    result = testdir.runpytest(p, '-p', 'nose')
-    result.stdout.fnmatch_lines([
-        "*2 passed*"
-    ])
+    result = testdir.inline_runpytest(p, '-p', 'nose')
+    result.assert_outcomes(passed=2)
 
 
 def test_nose_setup_func_failure(testdir):
@@ -89,7 +85,7 @@ def test_nose_setup_func_failure(testdir):
             assert l == [1,2]
 
     """)
-    result = testdir.runpytest(p, '-p', 'nose')
+    result = testdir.inline_runpytest(p, '-p', 'nose')
     result.stdout.fnmatch_lines([
         "*TypeError: <lambda>()*"
     ])
@@ -140,7 +136,7 @@ def test_nose_setup_partial(testdir):
         test_hello.setup = my_setup_partial
         test_hello.teardown = my_teardown_partial
     """)
-    result = testdir.runpytest(p, '-p', 'nose')
+    result = testdir.inline_runpytest(p, '-p', 'nose')
     result.stdout.fnmatch_lines([
         "*2 passed*"
     ])
@@ -207,7 +203,7 @@ def test_nose_test_generator_fixtures(testdir):
                 #expect.append('setup')
                 eq_(self.called, expect)
     """)
-    result = testdir.runpytest(p, '-p', 'nose')
+    result = testdir.inline_runpytest(p, '-p', 'nose')
     result.stdout.fnmatch_lines([
         "*10 passed*"
     ])
@@ -238,7 +234,7 @@ def test_module_level_setup(testdir):
             assert items[2] == 2
             assert 1 not in items
     """)
-    result = testdir.runpytest('-p', 'nose')
+    result = testdir.inline_runpytest('-p', 'nose')
     result.stdout.fnmatch_lines([
         "*2 passed*",
     ])
@@ -260,7 +256,7 @@ def test_nose_style_setup_teardown(testdir):
         def test_world():
             assert l == [1]
         """)
-    result = testdir.runpytest('-p', 'nose')
+    result = testdir.inline_runpytest('-p', 'nose')
     result.stdout.fnmatch_lines([
         "*2 passed*",
     ])
@@ -276,7 +272,7 @@ def test_nose_setup_ordering(testdir):
             def test_first(self):
                 pass
         """)
-    result = testdir.runpytest()
+    result = testdir.inline_runpytest()
     result.stdout.fnmatch_lines([
         "*1 passed*",
     ])
@@ -301,8 +297,8 @@ def test_apiwrapper_problem_issue260(testdir):
             def test_fun(self):
                 pass
         """)
-    result = testdir.runpytest()
-    result.stdout.fnmatch_lines("*1 passed*")
+    result = testdir.inline_runpytest()
+    result.assert_outcomes(passed=1)
 
 @pytest.mark.skipif("sys.version_info < (2,6)")
 def test_setup_teardown_linking_issue265(testdir):
@@ -327,8 +323,8 @@ def test_setup_teardown_linking_issue265(testdir):
                 """Undoes the setup."""
                 raise Exception("should not call teardown for skipped tests")
         ''')
-    reprec = testdir.inline_run()
-    reprec.assertoutcome(passed=1, skipped=1)
+    reprec = testdir.inline_runpytest()
+    reprec.assert_outcomes(passed=1, skipped=1)
 
 
 def test_SkipTest_during_collection(testdir):
@@ -338,8 +334,8 @@ def test_SkipTest_during_collection(testdir):
         def test_failing():
             assert False
         """)
-    result = testdir.runpytest(p)
-    result.assertoutcome(skipped=1)
+    result = testdir.inline_runpytest(p)
+    result.assert_outcomes(skipped=1)
 
 
 def test_SkipTest_in_test(testdir):