diff --git a/changelog/11475.feature.rst b/changelog/11475.feature.rst new file mode 100644 index 000000000..6b73c8158 --- /dev/null +++ b/changelog/11475.feature.rst @@ -0,0 +1,3 @@ +pytest now correctly identifies modules that are part of `namespace packages `__, for example when importing user-level modules for doctesting. + +Previously pytest was not aware of namespace packages, so running a doctest from a subpackage that is part of a namespace package would import just the subpackage (for example ``app.models``) instead of its full path (for example ``com.company.app.models``). diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index e2fa4db12..8097adb22 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -534,7 +534,9 @@ def import_path( return mod try: - pkg_root, module_name = resolve_pkg_root_and_module_name(path) + pkg_root, module_name = resolve_pkg_root_and_module_name( + path, consider_ns_packages=True + ) except CouldNotResolvePathError: pkg_root, module_name = path.parent, path.stem @@ -714,7 +716,9 @@ def resolve_package_path(path: Path) -> Optional[Path]: return result -def resolve_pkg_root_and_module_name(path: Path) -> Tuple[Path, str]: +def resolve_pkg_root_and_module_name( + path: Path, *, consider_ns_packages: bool = False +) -> Tuple[Path, str]: """ Return the path to the directory of the root package that contains the given Python file, and its module name: @@ -728,11 +732,31 @@ def resolve_pkg_root_and_module_name(path: Path) -> Tuple[Path, str]: Passing the full path to `models.py` will yield Path("src") and "app.core.models". + If consider_ns_packages is True, then we additionally check upwards in the hierarchy + until we find a directory that is reachable from sys.path, which marks it as a namespace package: + + https://packaging.python.org/en/latest/guides/packaging-namespace-packages + Raises CouldNotResolvePathError if the given path does not belong to a package (missing any __init__.py files). """ pkg_path = resolve_package_path(path) if pkg_path is not None: pkg_root = pkg_path.parent + # https://packaging.python.org/en/latest/guides/packaging-namespace-packages/ + if consider_ns_packages: + # Go upwards in the hierarchy, if we find a parent path included + # in sys.path, it means the package found by resolve_package_path() + # actually belongs to a namespace package. + for parent in pkg_root.parents: + # If any of the parent paths has a __init__.py, it means it is not + # a namespace package (see the docs linked above). + if (parent / "__init__.py").is_file(): + break + if str(parent) in sys.path: + # Point the pkg_root to the root of the namespace package. + pkg_root = parent + break + names = list(path.with_suffix("").relative_to(pkg_root).parts) if names[-1] == "__init__": names.pop() diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 087090071..d515cb5bd 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -9,11 +9,13 @@ from types import ModuleType from typing import Any from typing import Generator from typing import Iterator +from typing import Tuple import unittest.mock from _pytest.monkeypatch import MonkeyPatch from _pytest.pathlib import bestrelpath from _pytest.pathlib import commonpath +from _pytest.pathlib import CouldNotResolvePathError from _pytest.pathlib import ensure_deletable from _pytest.pathlib import fnmatch_ex from _pytest.pathlib import get_extended_length_path_str @@ -25,6 +27,7 @@ from _pytest.pathlib import insert_missing_modules from _pytest.pathlib import maybe_delete_a_numbered_dir from _pytest.pathlib import module_name_from_path from _pytest.pathlib import resolve_package_path +from _pytest.pathlib import resolve_pkg_root_and_module_name from _pytest.pathlib import safe_exists from _pytest.pathlib import symlink_or_skip from _pytest.pathlib import visit @@ -33,6 +36,20 @@ from _pytest.tmpdir import TempPathFactory import pytest +@pytest.fixture(autouse=True) +def autouse_pytester(pytester: Pytester) -> None: + """ + Fixture to make pytester() being autouse for all tests in this module. + + pytester makes sure to restore sys.path to its previous state, and many tests in this module + import modules and change sys.path because of that, so common module names such as "test" or "test.conftest" + end up leaking to tests in other modules. + + Note: we might consider extracting the sys.path restoration aspect into its own fixture, and apply it + to the entire test suite always. + """ + + class TestFNMatcherPort: """Test our port of py.common.FNMatcher (fnmatch_ex).""" @@ -596,6 +613,33 @@ class TestImportLibMode: ) assert result == "_env_310.tests.test_foo" + def test_resolve_pkg_root_and_module_name( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + # Create a directory structure first without __init__.py files. + (tmp_path / "src/app/core").mkdir(parents=True) + models_py = tmp_path / "src/app/core/models.py" + models_py.touch() + with pytest.raises(CouldNotResolvePathError): + _ = resolve_pkg_root_and_module_name(models_py) + + # Create the __init__.py files, it should now resolve to a proper module name. + (tmp_path / "src/app/__init__.py").touch() + (tmp_path / "src/app/core/__init__.py").touch() + assert resolve_pkg_root_and_module_name(models_py) == ( + tmp_path / "src", + "app.core.models", + ) + + # If we add tmp_path to sys.path, src becomes a namespace package. + monkeypatch.syspath_prepend(tmp_path) + assert resolve_pkg_root_and_module_name( + models_py, consider_ns_packages=True + ) == ( + tmp_path, + "src.app.core.models", + ) + def test_insert_missing_modules( self, monkeypatch: MonkeyPatch, tmp_path: Path ) -> None: @@ -741,3 +785,102 @@ def test_safe_exists(tmp_path: Path) -> None: side_effect=ValueError("name too long"), ): assert safe_exists(p) is False + + +class TestNamespacePackages: + """Test import_path support when importing from properly namespace packages.""" + + def setup_directories( + self, tmp_path: Path, monkeypatch: MonkeyPatch, pytester: Pytester + ) -> Tuple[Path, Path]: + # Set up a namespace package "com.company", containing + # two subpackages, "app" and "calc". + (tmp_path / "src/dist1/com/company/app/core").mkdir(parents=True) + (tmp_path / "src/dist1/com/company/app/__init__.py").touch() + (tmp_path / "src/dist1/com/company/app/core/__init__.py").touch() + models_py = tmp_path / "src/dist1/com/company/app/core/models.py" + models_py.touch() + + (tmp_path / "src/dist2/com/company/calc/algo").mkdir(parents=True) + (tmp_path / "src/dist2/com/company/calc/__init__.py").touch() + (tmp_path / "src/dist2/com/company/calc/algo/__init__.py").touch() + algorithms_py = tmp_path / "src/dist2/com/company/calc/algo/algorithms.py" + algorithms_py.touch() + + # Validate the namespace package by importing it in a Python subprocess. + r = pytester.runpython_c( + dedent( + f""" + import sys + sys.path.append(r{str(tmp_path / "src/dist1")!r}) + sys.path.append(r{str(tmp_path / "src/dist2")!r}) + import com.company.app.core.models + import com.company.calc.algo.algorithms + """ + ) + ) + assert r.ret == 0 + + monkeypatch.syspath_prepend(tmp_path / "src/dist1") + monkeypatch.syspath_prepend(tmp_path / "src/dist2") + return models_py, algorithms_py + + @pytest.mark.parametrize("import_mode", ["prepend", "append"]) + def test_resolve_pkg_root_and_module_name_ns_multiple_levels( + self, + tmp_path: Path, + monkeypatch: MonkeyPatch, + pytester: Pytester, + import_mode: str, + ) -> None: + models_py, algorithms_py = self.setup_directories( + tmp_path, monkeypatch, pytester + ) + + pkg_root, module_name = resolve_pkg_root_and_module_name( + models_py, consider_ns_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist1", + "com.company.app.core.models", + ) + + mod = import_path(models_py, mode=import_mode, root=tmp_path) + assert mod.__name__ == "com.company.app.core.models" + assert mod.__file__ == str(models_py) + + pkg_root, module_name = resolve_pkg_root_and_module_name( + algorithms_py, consider_ns_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist2", + "com.company.calc.algo.algorithms", + ) + + mod = import_path(algorithms_py, mode=import_mode, root=tmp_path) + assert mod.__name__ == "com.company.calc.algo.algorithms" + assert mod.__file__ == str(algorithms_py) + + @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) + def test_incorrect_namespace_package( + self, + tmp_path: Path, + monkeypatch: MonkeyPatch, + pytester: Pytester, + import_mode: str, + ) -> None: + models_py, algorithms_py = self.setup_directories( + tmp_path, monkeypatch, pytester + ) + # Namespace packages must not have an __init__.py at any of its + # directories; if it does, we then fall back to importing just the + # part of the package containing the __init__.py files. + (tmp_path / "src/dist1/com/__init__.py").touch() + + pkg_root, module_name = resolve_pkg_root_and_module_name( + models_py, consider_ns_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist1/com/company", + "app.core.models", + )