Fix error with --import-mode=importlib and modules containing dataclasses or pickle (#7870)

Co-authored-by: Bruno Oliveira <nicoddemus@gmail.com>

Fixes #7856, fixes #7859

AUTHORS

@@ -98,6 +98,7 @@ Dominic Mortlock
 Duncan Betts
 Edison Gustavo Muenz
 Edoardo Batini
+Edson Tadeu M. Manoel
 Eduardo Schettino
 Eli Boyarski
 Elizaveta Shashkova

changelog/7856.feature.rst

@@ -0,0 +1,2 @@
+:ref:`--import-mode=importlib <import-modes>` now works with features that
+depend on modules being on :py:data:`sys.modules`, such as :mod:`pickle` and :mod:`dataclasses`.

doc/en/explanation/goodpractices.rst

@@ -151,7 +151,7 @@ This layout prevents a lot of common pitfalls and has many benefits, which are b
 
 .. note::
     The new ``--import-mode=importlib`` (see :ref:`import-modes`) doesn't have
-    any of the drawbacks above because ``sys.path`` and ``sys.modules`` are not changed when importing
+    any of the drawbacks above because ``sys.path`` is not changed when importing
     test modules, so users that run
     into this issue are strongly encouraged to try it and report if the new option works well for them.
 

doc/en/explanation/pythonpath.rst

@@ -16,14 +16,14 @@ import process can be controlled through the ``--import-mode`` command-line flag
 these values:
 
 * ``prepend`` (default): the directory path containing each module will be inserted into the *beginning*
-  of ``sys.path`` if not already there, and then imported with the `__import__ <https://docs.python.org/3/library/functions.html#__import__>`__ builtin.
+  of :py:data:`sys.path` if not already there, and then imported with the `__import__ <https://docs.python.org/3/library/functions.html#__import__>`__ builtin.
 
   This requires test module names to be unique when the test directory tree is not arranged in
-  packages, because the modules will put in ``sys.modules`` after importing.
+  packages, because the modules will put in :py:data:`sys.modules` after importing.
 
   This is the classic mechanism, dating back from the time Python 2 was still supported.
 
-* ``append``: the directory containing each module is appended to the end of ``sys.path`` if not already
+* ``append``: the directory containing each module is appended to the end of :py:data:`sys.path` if not already
   there, and imported with ``__import__``.
 
   This better allows to run test modules against installed versions of a package even if the
@@ -41,17 +41,14 @@ these values:
   we advocate for using :ref:`src <src-layout>` layouts.
 
   Same as ``prepend``, requires test module names to be unique when the test directory tree is
-  not arranged in packages, because the modules will put in ``sys.modules`` after importing.
+  not arranged in packages, because the modules will put in :py:data:`sys.modules` after importing.
 
-* ``importlib``: new in pytest-6.0, this mode uses `importlib <https://docs.python.org/3/library/importlib.html>`__ to import test modules. This gives full control over the import process, and doesn't require
+* ``importlib``: new in pytest-6.0, this mode uses `importlib <https://docs.python.org/3/library/importlib.html>`__ to import test modules. This gives full control over the import process, and doesn't require changing :py:data:`sys.path`.
-  changing ``sys.path`` or ``sys.modules`` at all.
 
-  For this reason this doesn't require test module names to be unique at all, but also makes test
+  For this reason this doesn't require test module names to be unique, but also makes test
-  modules non-importable by each other. This was made possible in previous modes, for tests not residing
+  modules non-importable by each other.
-  in Python packages, because of the side-effects of changing ``sys.path`` and ``sys.modules``
-  mentioned above. Users which require this should turn their tests into proper packages instead.
 
-  We intend to make ``importlib`` the default in future releases.
+  We intend to make ``importlib`` the default in future releases, depending on feedback.
 
 ``prepend`` and ``append`` import modes scenarios
 -------------------------------------------------

src/_pytest/config/__init__.py

@@ -481,7 +481,9 @@ class PytestPluginManager(PluginManager):
     #
     # Internal API for local conftest plugin handling.
     #
-    def _set_initial_conftests(self, namespace: argparse.Namespace) -> None:
+    def _set_initial_conftests(
+        self, namespace: argparse.Namespace, rootpath: Path
+    ) -> None:
         """Load initial conftest files given a preparsed "namespace".
 
         As conftest files may add their own command line options which have
@@ -507,26 +509,24 @@ class PytestPluginManager(PluginManager):
                 path = path[:i]
             anchor = absolutepath(current / path)
             if anchor.exists():  # we found some file object
-                self._try_load_conftest(anchor, namespace.importmode)
+                self._try_load_conftest(anchor, namespace.importmode, rootpath)
                 foundanchor = True
         if not foundanchor:
-            self._try_load_conftest(current, namespace.importmode)
+            self._try_load_conftest(current, namespace.importmode, rootpath)
 
     def _try_load_conftest(
-        self, anchor: Path, importmode: Union[str, ImportMode]
+        self, anchor: Path, importmode: Union[str, ImportMode], rootpath: Path
     ) -> None:
-        self._getconftestmodules(anchor, importmode)
+        self._getconftestmodules(anchor, importmode, rootpath)
         # let's also consider test* subdirs
         if anchor.is_dir():
             for x in anchor.glob("test*"):
                 if x.is_dir():
-                    self._getconftestmodules(x, importmode)
+                    self._getconftestmodules(x, importmode, rootpath)
 
     @lru_cache(maxsize=128)
     def _getconftestmodules(
-        self,
+        self, path: Path, importmode: Union[str, ImportMode], rootpath: Path
-        path: Path,
-        importmode: Union[str, ImportMode],
     ) -> List[types.ModuleType]:
         if self._noconftest:
             return []
@@ -545,7 +545,7 @@ class PytestPluginManager(PluginManager):
                 continue
             conftestpath = parent / "conftest.py"
             if conftestpath.is_file():
-                mod = self._importconftest(conftestpath, importmode)
+                mod = self._importconftest(conftestpath, importmode, rootpath)
                 clist.append(mod)
         self._dirpath2confmods[directory] = clist
         return clist
@@ -555,8 +555,9 @@ class PytestPluginManager(PluginManager):
         name: str,
         path: Path,
         importmode: Union[str, ImportMode],
+        rootpath: Path,
     ) -> Tuple[types.ModuleType, Any]:
-        modules = self._getconftestmodules(path, importmode)
+        modules = self._getconftestmodules(path, importmode, rootpath=rootpath)
         for mod in reversed(modules):
             try:
                 return mod, getattr(mod, name)
@@ -565,9 +566,7 @@ class PytestPluginManager(PluginManager):
         raise KeyError(name)
 
     def _importconftest(
-        self,
+        self, conftestpath: Path, importmode: Union[str, ImportMode], rootpath: Path
-        conftestpath: Path,
-        importmode: Union[str, ImportMode],
     ) -> types.ModuleType:
         # Use a resolved Path object as key to avoid loading the same conftest
         # twice with build systems that create build directories containing
@@ -584,7 +583,7 @@ class PytestPluginManager(PluginManager):
             _ensure_removed_sysmodule(conftestpath.stem)
 
         try:
-            mod = import_path(conftestpath, mode=importmode)
+            mod = import_path(conftestpath, mode=importmode, root=rootpath)
         except Exception as e:
             assert e.__traceback__ is not None
             exc_info = (type(e), e, e.__traceback__)
@@ -1086,7 +1085,9 @@ class Config:
 
     @hookimpl(trylast=True)
     def pytest_load_initial_conftests(self, early_config: "Config") -> None:
-        self.pluginmanager._set_initial_conftests(early_config.known_args_namespace)
+        self.pluginmanager._set_initial_conftests(
+            early_config.known_args_namespace, rootpath=early_config.rootpath
+        )
 
     def _initini(self, args: Sequence[str]) -> None:
         ns, unknown_args = self._parser.parse_known_and_unknown_args(
@@ -1437,10 +1438,12 @@ class Config:
             assert type in [None, "string"]
             return value
 
-    def _getconftest_pathlist(self, name: str, path: Path) -> Optional[List[Path]]:
+    def _getconftest_pathlist(
+        self, name: str, path: Path, rootpath: Path
+    ) -> Optional[List[Path]]:
         try:
             mod, relroots = self.pluginmanager._rget_with_confmod(
-                name, path, self.getoption("importmode")
+                name, path, self.getoption("importmode"), rootpath
             )
         except KeyError:
             return None

src/_pytest/doctest.py

@@ -534,11 +534,13 @@ class DoctestModule(pytest.Module):
 
         if self.path.name == "conftest.py":
             module = self.config.pluginmanager._importconftest(
-                self.path, self.config.getoption("importmode")
+                self.path,
+                self.config.getoption("importmode"),
+                rootpath=self.config.rootpath,
             )
         else:
             try:
-                module = import_path(self.path)
+                module = import_path(self.path, root=self.config.rootpath)
             except ImportError:
                 if self.config.getvalue("doctest_ignore_import_errors"):
                     pytest.skip("unable to import module %r" % self.path)

src/_pytest/main.py

@@ -378,7 +378,9 @@ def _in_venv(path: Path) -> bool:
 
 
 def pytest_ignore_collect(fspath: Path, config: Config) -> Optional[bool]:
-    ignore_paths = config._getconftest_pathlist("collect_ignore", path=fspath.parent)
+    ignore_paths = config._getconftest_pathlist(
+        "collect_ignore", path=fspath.parent, rootpath=config.rootpath
+    )
     ignore_paths = ignore_paths or []
     excludeopt = config.getoption("ignore")
     if excludeopt:
@@ -388,7 +390,7 @@ def pytest_ignore_collect(fspath: Path, config: Config) -> Optional[bool]:
         return True
 
     ignore_globs = config._getconftest_pathlist(
-        "collect_ignore_glob", path=fspath.parent
+        "collect_ignore_glob", path=fspath.parent, rootpath=config.rootpath
     )
     ignore_globs = ignore_globs or []
     excludeglobopt = config.getoption("ignore_glob")
@@ -546,7 +548,9 @@ class Session(nodes.FSCollector):
         # hooks with all conftest.py files.
         pm = self.config.pluginmanager
         my_conftestmodules = pm._getconftestmodules(
-            Path(fspath), self.config.getoption("importmode")
+            Path(fspath),
+            self.config.getoption("importmode"),
+            rootpath=self.config.rootpath,
         )
         remove_mods = pm._conftest_plugins.difference(my_conftestmodules)
         if remove_mods:

src/_pytest/pathlib.py

@@ -23,6 +23,7 @@ from pathlib import PurePath
 from posixpath import sep as posix_sep
 from types import ModuleType
 from typing import Callable
+from typing import Dict
 from typing import Iterable
 from typing import Iterator
 from typing import Optional
@@ -454,6 +455,7 @@ def import_path(
     p: Union[str, "os.PathLike[str]"],
     *,
     mode: Union[str, ImportMode] = ImportMode.prepend,
+    root: Path,
 ) -> ModuleType:
     """Import and return a module from the given path, which can be a file (a module) or
     a directory (a package).
@@ -471,6 +473,11 @@ def import_path(
       to import the module, which avoids having to use `__import__` and muck with `sys.path`
       at all. It effectively allows having same-named test modules in different places.
 
+    :param root:
+        Used as an anchor when mode == ImportMode.importlib to obtain
+        a unique name for the module being imported so it can safely be stored
+        into ``sys.modules``.
+
     :raises ImportPathMismatchError:
         If after importing the given `path` and the module `__file__`
         are different. Only raised in `prepend` and `append` modes.
@@ -483,7 +490,7 @@ def import_path(
         raise ImportError(path)
 
     if mode is ImportMode.importlib:
-        module_name = path.stem
+        module_name = module_name_from_path(path, root)
 
         for meta_importer in sys.meta_path:
             spec = meta_importer.find_spec(module_name, [str(path.parent)])
@@ -497,7 +504,9 @@ def import_path(
                 "Can't find module {} at location {}".format(module_name, str(path))
             )
         mod = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = mod
         spec.loader.exec_module(mod)  # type: ignore[union-attr]
+        insert_missing_modules(sys.modules, module_name)
         return mod
 
     pkg_path = resolve_package_path(path)
@@ -562,6 +571,47 @@ else:
         return os.path.samefile(f1, f2)
 
 
+def module_name_from_path(path: Path, root: Path) -> str:
+    """
+    Return a dotted module name based on the given path, anchored on root.
+
+    For example: path="projects/src/tests/test_foo.py" and root="/projects", the
+    resulting module name will be "src.tests.test_foo".
+    """
+    path = path.with_suffix("")
+    try:
+        relative_path = path.relative_to(root)
+    except ValueError:
+        # If we can't get a relative path to root, use the full path, except
+        # for the first part ("d:\\" or "/" depending on the platform, for example).
+        path_parts = path.parts[1:]
+    else:
+        # Use the parts for the relative path to the root path.
+        path_parts = relative_path.parts
+
+    return ".".join(path_parts)
+
+
+def insert_missing_modules(modules: Dict[str, ModuleType], module_name: str) -> None:
+    """
+    Used by ``import_path`` to create intermediate modules when using mode=importlib.
+
+    When we want to import a module as "src.tests.test_foo" for example, we need
+    to create empty modules "src" and "src.tests" after inserting "src.tests.test_foo",
+    otherwise "src.tests.test_foo" is not importable by ``__import__``.
+    """
+    module_parts = module_name.split(".")
+    while module_name:
+        if module_name not in modules:
+            module = ModuleType(
+                module_name,
+                doc="Empty module created by pytest's importmode=importlib.",
+            )
+            modules[module_name] = module
+        module_parts.pop(-1)
+        module_name = ".".join(module_parts)
+
+
 def resolve_package_path(path: Path) -> Optional[Path]:
     """Return the Python package path by looking for the last
     directory upwards which still contains an __init__.py.

src/_pytest/python.py

@@ -577,7 +577,7 @@ class Module(nodes.File, PyCollector):
         # We assume we are only called once per module.
         importmode = self.config.getoption("--import-mode")
         try:
-            mod = import_path(self.path, mode=importmode)
+            mod = import_path(self.path, mode=importmode, root=self.config.rootpath)
         except SyntaxError as e:
             raise self.CollectError(
                 ExceptionInfo.from_current().getrepr(style="short")

testing/code/test_excinfo.py

@@ -162,7 +162,7 @@ class TestTraceback_f_g_h:
     def test_traceback_cut_excludepath(self, pytester: Pytester) -> None:
         p = pytester.makepyfile("def f(): raise ValueError")
         with pytest.raises(ValueError) as excinfo:
-            import_path(p).f()  # type: ignore[attr-defined]
+            import_path(p, root=pytester.path).f()  # type: ignore[attr-defined]
         basedir = Path(pytest.__file__).parent
         newtraceback = excinfo.traceback.cut(excludepath=basedir)
         for x in newtraceback:
@@ -443,7 +443,7 @@ class TestFormattedExcinfo:
             tmp_path.joinpath("__init__.py").touch()
             modpath.write_text(source)
             importlib.invalidate_caches()
-            return import_path(modpath)
+            return import_path(modpath, root=tmp_path)
 
         return importasmod
 

testing/code/test_source.py

@@ -298,7 +298,7 @@ def test_source_of_class_at_eof_without_newline(_sys_snapshot, tmp_path: Path) -
     )
     path = tmp_path.joinpath("a.py")
     path.write_text(str(source))
-    mod: Any = import_path(path)
+    mod: Any = import_path(path, root=tmp_path)
     s2 = Source(mod.A)
     assert str(source).strip() == str(s2).strip()
 

testing/python/fixtures.py

@@ -2069,9 +2069,9 @@ class TestAutouseManagement:
         reprec = pytester.inline_run("-v", "-s", "--confcutdir", pytester.path)
         reprec.assertoutcome(passed=8)
         config = reprec.getcalls("pytest_unconfigure")[0].config
-        values = config.pluginmanager._getconftestmodules(p, importmode="prepend")[
+        values = config.pluginmanager._getconftestmodules(
-            0
+            p, importmode="prepend", rootpath=pytester.path
-        ].values
+        )[0].values
         assert values == ["fin_a1", "fin_a2", "fin_b1", "fin_b2"] * 2
 
     def test_scope_ordering(self, pytester: Pytester) -> None:

testing/test_config.py

@@ -597,8 +597,14 @@ class TestConfigAPI:
         p = tmp_path.joinpath("conftest.py")
         p.write_text(f"pathlist = ['.', {str(somepath)!r}]")
         config = pytester.parseconfigure(p)
-        assert config._getconftest_pathlist("notexist", path=tmp_path) is None
+        assert (
-        pl = config._getconftest_pathlist("pathlist", path=tmp_path) or []
+            config._getconftest_pathlist("notexist", path=tmp_path, rootpath=tmp_path)
+            is None
+        )
+        pl = (
+            config._getconftest_pathlist("pathlist", path=tmp_path, rootpath=tmp_path)
+            or []
+        )
         print(pl)
         assert len(pl) == 2
         assert pl[0] == tmp_path

testing/test_conftest.py

@@ -35,7 +35,7 @@ def conftest_setinitial(
             self.importmode = "prepend"
 
     namespace = cast(argparse.Namespace, Namespace())
-    conftest._set_initial_conftests(namespace)
+    conftest._set_initial_conftests(namespace, rootpath=Path(args[0]))
 
 
 @pytest.mark.usefixtures("_sys_snapshot")
@@ -57,39 +57,62 @@ class TestConftestValueAccessGlobal:
     def test_basic_init(self, basedir: Path) -> None:
         conftest = PytestPluginManager()
         p = basedir / "adir"
-        assert conftest._rget_with_confmod("a", p, importmode="prepend")[1] == 1
+        assert (
+            conftest._rget_with_confmod("a", p, importmode="prepend", rootpath=basedir)[
+                1
+            ]
+            == 1
+        )
 
     def test_immediate_initialiation_and_incremental_are_the_same(
         self, basedir: Path
     ) -> None:
         conftest = PytestPluginManager()
         assert not len(conftest._dirpath2confmods)
-        conftest._getconftestmodules(basedir, importmode="prepend")
+        conftest._getconftestmodules(
+            basedir, importmode="prepend", rootpath=Path(basedir)
+        )
         snap1 = len(conftest._dirpath2confmods)
         assert snap1 == 1
-        conftest._getconftestmodules(basedir / "adir", importmode="prepend")
+        conftest._getconftestmodules(
+            basedir / "adir", importmode="prepend", rootpath=basedir
+        )
         assert len(conftest._dirpath2confmods) == snap1 + 1
-        conftest._getconftestmodules(basedir / "b", importmode="prepend")
+        conftest._getconftestmodules(
+            basedir / "b", importmode="prepend", rootpath=basedir
+        )
         assert len(conftest._dirpath2confmods) == snap1 + 2
 
     def test_value_access_not_existing(self, basedir: Path) -> None:
         conftest = ConftestWithSetinitial(basedir)
         with pytest.raises(KeyError):
-            conftest._rget_with_confmod("a", basedir, importmode="prepend")
+            conftest._rget_with_confmod(
+                "a", basedir, importmode="prepend", rootpath=Path(basedir)
+            )
 
     def test_value_access_by_path(self, basedir: Path) -> None:
         conftest = ConftestWithSetinitial(basedir)
         adir = basedir / "adir"
-        assert conftest._rget_with_confmod("a", adir, importmode="prepend")[1] == 1
         assert (
-            conftest._rget_with_confmod("a", adir / "b", importmode="prepend")[1] == 1.5
+            conftest._rget_with_confmod(
+                "a", adir, importmode="prepend", rootpath=basedir
+            )[1]
+            == 1
+        )
+        assert (
+            conftest._rget_with_confmod(
+                "a", adir / "b", importmode="prepend", rootpath=basedir
+            )[1]
+            == 1.5
         )
 
     def test_value_access_with_confmod(self, basedir: Path) -> None:
         startdir = basedir / "adir" / "b"
         startdir.joinpath("xx").mkdir()
         conftest = ConftestWithSetinitial(startdir)
-        mod, value = conftest._rget_with_confmod("a", startdir, importmode="prepend")
+        mod, value = conftest._rget_with_confmod(
+            "a", startdir, importmode="prepend", rootpath=Path(basedir)
+        )
         assert value == 1.5
         path = Path(mod.__file__)
         assert path.parent == basedir / "adir" / "b"
@@ -110,7 +133,9 @@ def test_doubledash_considered(pytester: Pytester) -> None:
     conf.joinpath("conftest.py").touch()
     conftest = PytestPluginManager()
     conftest_setinitial(conftest, [conf.name, conf.name])
-    values = conftest._getconftestmodules(conf, importmode="prepend")
+    values = conftest._getconftestmodules(
+        conf, importmode="prepend", rootpath=pytester.path
+    )
     assert len(values) == 1
 
 
@@ -134,7 +159,7 @@ def test_conftest_global_import(pytester: Pytester) -> None:
         import pytest
         from _pytest.config import PytestPluginManager
         conf = PytestPluginManager()
-        mod = conf._importconftest(Path("conftest.py"), importmode="prepend")
+        mod = conf._importconftest(Path("conftest.py"), importmode="prepend", rootpath=Path.cwd())
         assert mod.x == 3
         import conftest
         assert conftest is mod, (conftest, mod)
@@ -142,7 +167,7 @@ def test_conftest_global_import(pytester: Pytester) -> None:
         sub.mkdir()
         subconf = sub / "conftest.py"
         subconf.write_text("y=4")
-        mod2 = conf._importconftest(subconf, importmode="prepend")
+        mod2 = conf._importconftest(subconf, importmode="prepend", rootpath=Path.cwd())
         assert mod != mod2
         assert mod2.y == 4
         import conftest
@@ -158,17 +183,25 @@ def test_conftestcutdir(pytester: Pytester) -> None:
     p = pytester.mkdir("x")
     conftest = PytestPluginManager()
     conftest_setinitial(conftest, [pytester.path], confcutdir=p)
-    values = conftest._getconftestmodules(p, importmode="prepend")
+    values = conftest._getconftestmodules(
+        p, importmode="prepend", rootpath=pytester.path
+    )
     assert len(values) == 0
-    values = conftest._getconftestmodules(conf.parent, importmode="prepend")
+    values = conftest._getconftestmodules(
+        conf.parent, importmode="prepend", rootpath=pytester.path
+    )
     assert len(values) == 0
     assert Path(conf) not in conftest._conftestpath2mod
     # but we can still import a conftest directly
-    conftest._importconftest(conf, importmode="prepend")
+    conftest._importconftest(conf, importmode="prepend", rootpath=pytester.path)
-    values = conftest._getconftestmodules(conf.parent, importmode="prepend")
+    values = conftest._getconftestmodules(
+        conf.parent, importmode="prepend", rootpath=pytester.path
+    )
     assert values[0].__file__.startswith(str(conf))
     # and all sub paths get updated properly
-    values = conftest._getconftestmodules(p, importmode="prepend")
+    values = conftest._getconftestmodules(
+        p, importmode="prepend", rootpath=pytester.path
+    )
     assert len(values) == 1
     assert values[0].__file__.startswith(str(conf))
 
@@ -177,7 +210,9 @@ def test_conftestcutdir_inplace_considered(pytester: Pytester) -> None:
     conf = pytester.makeconftest("")
     conftest = PytestPluginManager()
     conftest_setinitial(conftest, [conf.parent], confcutdir=conf.parent)
-    values = conftest._getconftestmodules(conf.parent, importmode="prepend")
+    values = conftest._getconftestmodules(
+        conf.parent, importmode="prepend", rootpath=pytester.path
+    )
     assert len(values) == 1
     assert values[0].__file__.startswith(str(conf))
 
@@ -347,13 +382,16 @@ def test_conftest_import_order(pytester: Pytester, monkeypatch: MonkeyPatch) ->
     ct2 = sub / "conftest.py"
     ct2.write_text("")
 
-    def impct(p, importmode):
+    def impct(p, importmode, root):
         return p
 
     conftest = PytestPluginManager()
     conftest._confcutdir = pytester.path
     monkeypatch.setattr(conftest, "_importconftest", impct)
-    mods = cast(List[Path], conftest._getconftestmodules(sub, importmode="prepend"))
+    mods = cast(
+        List[Path],
+        conftest._getconftestmodules(sub, importmode="prepend", rootpath=pytester.path),
+    )
     expected = [ct1, ct2]
     assert mods == expected
 

testing/test_pathlib.py

@@ -5,6 +5,7 @@ import unittest.mock
 from pathlib import Path
 from textwrap import dedent
 from types import ModuleType
+from typing import Any
 from typing import Generator
 
 import pytest
@@ -17,7 +18,9 @@ from _pytest.pathlib import get_extended_length_path_str
 from _pytest.pathlib import get_lock_path
 from _pytest.pathlib import import_path
 from _pytest.pathlib import ImportPathMismatchError
+from _pytest.pathlib import insert_missing_modules
 from _pytest.pathlib import maybe_delete_a_numbered_dir
+from _pytest.pathlib import module_name_from_path
 from _pytest.pathlib import resolve_package_path
 from _pytest.pathlib import symlink_or_skip
 from _pytest.pathlib import visit
@@ -136,7 +139,7 @@ class TestImportPath:
         )
 
     def test_smoke_test(self, path1: Path) -> None:
-        obj = import_path(path1 / "execfile.py")
+        obj = import_path(path1 / "execfile.py", root=path1)
         assert obj.x == 42  # type: ignore[attr-defined]
         assert obj.__name__ == "execfile"
 
@@ -146,25 +149,25 @@ class TestImportPath:
         tmp_path.joinpath("a").mkdir()
         p = tmp_path.joinpath("a", "test_x123.py")
         p.touch()
-        import_path(p)
+        import_path(p, root=tmp_path)
         tmp_path.joinpath("a").rename(tmp_path.joinpath("b"))
         with pytest.raises(ImportPathMismatchError):
-            import_path(tmp_path.joinpath("b", "test_x123.py"))
+            import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path)
 
         # Errors can be ignored.
         monkeypatch.setenv("PY_IGNORE_IMPORTMISMATCH", "1")
-        import_path(tmp_path.joinpath("b", "test_x123.py"))
+        import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path)
 
         # PY_IGNORE_IMPORTMISMATCH=0 does not ignore error.
         monkeypatch.setenv("PY_IGNORE_IMPORTMISMATCH", "0")
         with pytest.raises(ImportPathMismatchError):
-            import_path(tmp_path.joinpath("b", "test_x123.py"))
+            import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path)
 
     def test_messy_name(self, tmp_path: Path) -> None:
         # http://bitbucket.org/hpk42/py-trunk/issue/129
         path = tmp_path / "foo__init__.py"
         path.touch()
-        module = import_path(path)
+        module = import_path(path, root=tmp_path)
         assert module.__name__ == "foo__init__"
 
     def test_dir(self, tmp_path: Path) -> None:
@@ -172,31 +175,31 @@ class TestImportPath:
         p.mkdir()
         p_init = p / "__init__.py"
         p_init.touch()
-        m = import_path(p)
+        m = import_path(p, root=tmp_path)
         assert m.__name__ == "hello_123"
-        m = import_path(p_init)
+        m = import_path(p_init, root=tmp_path)
         assert m.__name__ == "hello_123"
 
     def test_a(self, path1: Path) -> None:
         otherdir = path1 / "otherdir"
-        mod = import_path(otherdir / "a.py")
+        mod = import_path(otherdir / "a.py", root=path1)
         assert mod.result == "got it"  # type: ignore[attr-defined]
         assert mod.__name__ == "otherdir.a"
 
     def test_b(self, path1: Path) -> None:
         otherdir = path1 / "otherdir"
-        mod = import_path(otherdir / "b.py")
+        mod = import_path(otherdir / "b.py", root=path1)
         assert mod.stuff == "got it"  # type: ignore[attr-defined]
         assert mod.__name__ == "otherdir.b"
 
     def test_c(self, path1: Path) -> None:
         otherdir = path1 / "otherdir"
-        mod = import_path(otherdir / "c.py")
+        mod = import_path(otherdir / "c.py", root=path1)
         assert mod.value == "got it"  # type: ignore[attr-defined]
 
     def test_d(self, path1: Path) -> None:
         otherdir = path1 / "otherdir"
-        mod = import_path(otherdir / "d.py")
+        mod = import_path(otherdir / "d.py", root=path1)
         assert mod.value2 == "got it"  # type: ignore[attr-defined]
 
     def test_import_after(self, tmp_path: Path) -> None:
@@ -204,7 +207,7 @@ class TestImportPath:
         tmp_path.joinpath("xxxpackage", "__init__.py").touch()
         mod1path = tmp_path.joinpath("xxxpackage", "module1.py")
         mod1path.touch()
-        mod1 = import_path(mod1path)
+        mod1 = import_path(mod1path, root=tmp_path)
         assert mod1.__name__ == "xxxpackage.module1"
         from xxxpackage import module1
 
@@ -222,7 +225,7 @@ class TestImportPath:
             pseudopath.touch()
             mod.__file__ = str(pseudopath)
             monkeypatch.setitem(sys.modules, name, mod)
-            newmod = import_path(p)
+            newmod = import_path(p, root=tmp_path)
             assert mod == newmod
         monkeypatch.undo()
         mod = ModuleType(name)
@@ -231,7 +234,7 @@ class TestImportPath:
         mod.__file__ = str(pseudopath)
         monkeypatch.setitem(sys.modules, name, mod)
         with pytest.raises(ImportPathMismatchError) as excinfo:
-            import_path(p)
+            import_path(p, root=tmp_path)
         modname, modfile, orig = excinfo.value.args
         assert modname == name
         assert modfile == str(pseudopath)
@@ -248,8 +251,8 @@ class TestImportPath:
         tmp_path.joinpath("sub", "proja").mkdir(parents=True)
         p2 = tmp_path.joinpath("sub", "proja", "__init__.py")
         p2.touch()
-        m1 = import_path(p1)
+        m1 = import_path(p1, root=tmp_path)
-        m2 = import_path(p2)
+        m2 = import_path(p2, root=tmp_path)
         assert m1 == m2
 
     def test_ensuresyspath_append(self, tmp_path: Path) -> None:
@@ -258,44 +261,45 @@ class TestImportPath:
         file1 = root1 / "x123.py"
         file1.touch()
         assert str(root1) not in sys.path
-        import_path(file1, mode="append")
+        import_path(file1, mode="append", root=tmp_path)
         assert str(root1) == sys.path[-1]
         assert str(root1) not in sys.path[:-1]
 
     def test_invalid_path(self, tmp_path: Path) -> None:
         with pytest.raises(ImportError):
-            import_path(tmp_path / "invalid.py")
+            import_path(tmp_path / "invalid.py", root=tmp_path)
 
     @pytest.fixture
     def simple_module(self, tmp_path: Path) -> Path:
-        fn = tmp_path / "mymod.py"
+        fn = tmp_path / "_src/tests/mymod.py"
-        fn.write_text(
+        fn.parent.mkdir(parents=True)
-            dedent(
+        fn.write_text("def foo(x): return 40 + x")
-                """
-            def foo(x): return 40 + x
-            """
-            )
-        )
         return fn
 
-    def test_importmode_importlib(self, simple_module: Path) -> None:
+    def test_importmode_importlib(self, simple_module: Path, tmp_path: Path) -> None:
         """`importlib` mode does not change sys.path."""
-        module = import_path(simple_module, mode="importlib")
+        module = import_path(simple_module, mode="importlib", root=tmp_path)
         assert module.foo(2) == 42  # type: ignore[attr-defined]
         assert str(simple_module.parent) not in sys.path
+        assert module.__name__ in sys.modules
+        assert module.__name__ == "_src.tests.mymod"
+        assert "_src" in sys.modules
+        assert "_src.tests" in sys.modules
 
-    def test_importmode_twice_is_different_module(self, simple_module: Path) -> None:
+    def test_importmode_twice_is_different_module(
+        self, simple_module: Path, tmp_path: Path
+    ) -> None:
         """`importlib` mode always returns a new module."""
-        module1 = import_path(simple_module, mode="importlib")
+        module1 = import_path(simple_module, mode="importlib", root=tmp_path)
-        module2 = import_path(simple_module, mode="importlib")
+        module2 = import_path(simple_module, mode="importlib", root=tmp_path)
         assert module1 is not module2
 
     def test_no_meta_path_found(
-        self, simple_module: Path, monkeypatch: MonkeyPatch
+        self, simple_module: Path, monkeypatch: MonkeyPatch, tmp_path: Path
     ) -> None:
         """Even without any meta_path should still import module."""
         monkeypatch.setattr(sys, "meta_path", [])
-        module = import_path(simple_module, mode="importlib")
+        module = import_path(simple_module, mode="importlib", root=tmp_path)
         assert module.foo(2) == 42  # type: ignore[attr-defined]
 
         # mode='importlib' fails if no spec is found to load the module
@@ -305,7 +309,7 @@ class TestImportPath:
             importlib.util, "spec_from_file_location", lambda *args: None
         )
         with pytest.raises(ImportError):
-            import_path(simple_module, mode="importlib")
+            import_path(simple_module, mode="importlib", root=tmp_path)
 
 
 def test_resolve_package_path(tmp_path: Path) -> None:
@@ -441,5 +445,130 @@ def test_samefile_false_negatives(tmp_path: Path, monkeypatch: MonkeyPatch) -> N
         # the paths too. Using a context to narrow the patch as much as possible given
         # this is an important system function.
         mp.setattr(os.path, "samefile", lambda x, y: False)
-        module = import_path(module_path)
+        module = import_path(module_path, root=tmp_path)
     assert getattr(module, "foo")() == 42
+
+
+class TestImportLibMode:
+    @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+")
+    def test_importmode_importlib_with_dataclass(self, tmp_path: Path) -> None:
+        """Ensure that importlib mode works with a module containing dataclasses (#7856)."""
+        fn = tmp_path.joinpath("_src/tests/test_dataclass.py")
+        fn.parent.mkdir(parents=True)
+        fn.write_text(
+            dedent(
+                """
+                from dataclasses import dataclass
+
+                @dataclass
+                class Data:
+                    value: str
+                """
+            )
+        )
+
+        module = import_path(fn, mode="importlib", root=tmp_path)
+        Data: Any = getattr(module, "Data")
+        data = Data(value="foo")
+        assert data.value == "foo"
+        assert data.__module__ == "_src.tests.test_dataclass"
+
+    def test_importmode_importlib_with_pickle(self, tmp_path: Path) -> None:
+        """Ensure that importlib mode works with pickle (#7859)."""
+        fn = tmp_path.joinpath("_src/tests/test_pickle.py")
+        fn.parent.mkdir(parents=True)
+        fn.write_text(
+            dedent(
+                """
+                import pickle
+
+                def _action():
+                    return 42
+
+                def round_trip():
+                    s = pickle.dumps(_action)
+                    return pickle.loads(s)
+                """
+            )
+        )
+
+        module = import_path(fn, mode="importlib", root=tmp_path)
+        round_trip = getattr(module, "round_trip")
+        action = round_trip()
+        assert action() == 42
+
+    def test_importmode_importlib_with_pickle_separate_modules(
+        self, tmp_path: Path
+    ) -> None:
+        """
+        Ensure that importlib mode works can load pickles that look similar but are
+        defined in separate modules.
+        """
+        fn1 = tmp_path.joinpath("_src/m1/tests/test.py")
+        fn1.parent.mkdir(parents=True)
+        fn1.write_text(
+            dedent(
+                """
+                import attr
+                import pickle
+
+                @attr.s(auto_attribs=True)
+                class Data:
+                    x: int = 42
+                """
+            )
+        )
+
+        fn2 = tmp_path.joinpath("_src/m2/tests/test.py")
+        fn2.parent.mkdir(parents=True)
+        fn2.write_text(
+            dedent(
+                """
+                import attr
+                import pickle
+
+                @attr.s(auto_attribs=True)
+                class Data:
+                    x: str = ""
+                """
+            )
+        )
+
+        import pickle
+
+        def round_trip(obj):
+            s = pickle.dumps(obj)
+            return pickle.loads(s)
+
+        module = import_path(fn1, mode="importlib", root=tmp_path)
+        Data1 = getattr(module, "Data")
+
+        module = import_path(fn2, mode="importlib", root=tmp_path)
+        Data2 = getattr(module, "Data")
+
+        assert round_trip(Data1(20)) == Data1(20)
+        assert round_trip(Data2("hello")) == Data2("hello")
+        assert Data1.__module__ == "_src.m1.tests.test"
+        assert Data2.__module__ == "_src.m2.tests.test"
+
+    def test_module_name_from_path(self, tmp_path: Path) -> None:
+        result = module_name_from_path(tmp_path / "src/tests/test_foo.py", tmp_path)
+        assert result == "src.tests.test_foo"
+
+        # Path is not relative to root dir: use the full path to obtain the module name.
+        result = module_name_from_path(Path("/home/foo/test_foo.py"), Path("/bar"))
+        assert result == "home.foo.test_foo"
+
+    def test_insert_missing_modules(self) -> None:
+        modules = {"src.tests.foo": ModuleType("src.tests.foo")}
+        insert_missing_modules(modules, "src.tests.foo")
+        assert sorted(modules) == ["src", "src.tests", "src.tests.foo"]
+
+        mod = ModuleType("mod", doc="My Module")
+        modules = {"src": mod}
+        insert_missing_modules(modules, "src")
+        assert modules == {"src": mod}
+
+        modules = {}
+        insert_missing_modules(modules, "")
+        assert modules == {}

testing/test_pluginmanager.py

@@ -44,7 +44,9 @@ class TestPytestPluginInteractions:
         pm.hook.pytest_addhooks.call_historic(
             kwargs=dict(pluginmanager=config.pluginmanager)
         )
-        config.pluginmanager._importconftest(conf, importmode="prepend")
+        config.pluginmanager._importconftest(
+            conf, importmode="prepend", rootpath=pytester.path
+        )
         # print(config.pluginmanager.get_plugins())
         res = config.hook.pytest_myhook(xyz=10)
         assert res == [11]
@@ -71,7 +73,9 @@ class TestPytestPluginInteractions:
                     default=True)
         """
         )
-        config.pluginmanager._importconftest(p, importmode="prepend")
+        config.pluginmanager._importconftest(
+            p, importmode="prepend", rootpath=pytester.path
+        )
         assert config.option.test123
 
     def test_configure(self, pytester: Pytester) -> None:
@@ -136,10 +140,14 @@ class TestPytestPluginInteractions:
         conftest1 = pytester.path.joinpath("tests/conftest.py")
         conftest2 = pytester.path.joinpath("tests/subdir/conftest.py")
 
-        config.pluginmanager._importconftest(conftest1, importmode="prepend")
+        config.pluginmanager._importconftest(
+            conftest1, importmode="prepend", rootpath=pytester.path
+        )
         ihook_a = session.gethookproxy(pytester.path / "tests")
         assert ihook_a is not None
-        config.pluginmanager._importconftest(conftest2, importmode="prepend")
+        config.pluginmanager._importconftest(
+            conftest2, importmode="prepend", rootpath=pytester.path
+        )
         ihook_b = session.gethookproxy(pytester.path / "tests")
         assert ihook_a is not ihook_b
 
@@ -350,7 +358,9 @@ class TestPytestPluginManager:
         pytester: Pytester,
         pytestpm: PytestPluginManager,
     ) -> None:
-        mod = import_path(pytester.makepyfile("pytest_plugins='xyz'"))
+        mod = import_path(
+            pytester.makepyfile("pytest_plugins='xyz'"), root=pytester.path
+        )
         with pytest.raises(ImportError):
             pytestpm.consider_conftest(mod)