From f52327953101b325b41d0c97a0ff908131db5676 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 6 Apr 2024 10:40:26 -0300 Subject: [PATCH] Support full namespace packages without __init__ --- src/_pytest/pathlib.py | 64 ++++++++++++++++++++--------------- testing/test_pathlib.py | 75 ++++++++++++++++++++++++----------------- 2 files changed, 82 insertions(+), 57 deletions(-) diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 054c21445..05f125d7d 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -8,6 +8,7 @@ from errno import ENOENT from errno import ENOTDIR import fnmatch from functools import partial +from importlib.machinery import ModuleSpec import importlib.util import itertools import os @@ -36,6 +37,8 @@ from typing import Union import uuid import warnings +from typing_extensions import TypeGuard + from _pytest.compat import assert_never from _pytest.outcomes import skip from _pytest.warning_types import PytestWarning @@ -628,11 +631,12 @@ def _import_module_using_spec( # such as our own assertion-rewrite hook. for meta_importer in sys.meta_path: spec = meta_importer.find_spec(module_name, [str(module_location)]) - if spec is not None: + if spec_matches_module_path(spec, module_path): break else: spec = importlib.util.spec_from_file_location(module_name, str(module_path)) - if spec is not None: + + if spec_matches_module_path(spec, module_path): mod = importlib.util.module_from_spec(spec) sys.modules[module_name] = mod spec.loader.exec_module(mod) # type: ignore[union-attr] @@ -643,6 +647,15 @@ def _import_module_using_spec( return None +def spec_matches_module_path( + module_spec: Optional[ModuleSpec], module_path: Path +) -> TypeGuard[ModuleSpec]: + if module_spec is not None and module_spec.origin is not None: + if Path(module_spec.origin) == module_path: + return True + return False + + # Implement a special _is_same function on Windows which returns True if the two filenames # compare equal, to circumvent os.path.samefile returning False for mounts in UNC (#7678). if sys.platform.startswith("win"): @@ -768,21 +781,19 @@ def resolve_pkg_root_and_module_name( Raises CouldNotResolvePathError if the given path does not belong to a package (missing any __init__.py files). """ + pkg_root: Optional[Path] = None pkg_path = resolve_package_path(path) if pkg_path is not None: pkg_root = pkg_path.parent - if consider_namespace_packages: - for candidate in (pkg_root, *pkg_root.parents): - # If any of the parent paths has an __init__.py, it means it is not a namespace package: - # https://packaging.python.org/en/latest/guides/packaging-namespace-packages - if (candidate / "__init__.py").is_file(): - break - - if _is_namespace_package(candidate): - # Point the pkg_root to the root of the namespace package. - pkg_root = candidate.parent - break + if consider_namespace_packages: + start = path.parent if pkg_path is None else pkg_path.parent + for candidate in (start, *start.parents): + if _is_importable(candidate, path): + # Point the pkg_root to the root of the namespace package. + pkg_root = candidate.parent + break + if pkg_root is not None: names = list(path.with_suffix("").relative_to(pkg_root).parts) if names[-1] == "__init__": names.pop() @@ -792,25 +803,24 @@ def resolve_pkg_root_and_module_name( raise CouldNotResolvePathError(f"Could not resolve for {path}") -def _is_namespace_package(module_path: Path) -> bool: - module_name = module_path.name - - # Empty module names (such as Path.cwd()) might break meta_path hooks (like our own assertion rewriter). - if not module_name: +def _is_importable(root: Path, path: Path) -> bool: + try: + path_without_suffix = path.with_suffix("") + except ValueError: + # Empty paths (such as Path.cwd()) might break meta_path hooks (like our own assertion rewriter). return False + names = list(path_without_suffix.relative_to(root.parent).parts) + if names[-1] == "__init__": + names.pop() + module_name = ".".join(names) + try: spec = importlib.util.find_spec(module_name) - except ImportError: + except (ImportError, ValueError, ImportWarning): return False - if spec is not None and spec.submodule_search_locations: - # Found a spec, however make sure the module_path is in one of the search locations -- - # this ensures common module name like "src" (which might be in sys.path under different locations) - # is only considered for the module_path we intend to. - # Make sure to compare Path(s) instead of strings, this normalizes them on Windows. - if module_path in [Path(x) for x in spec.submodule_search_locations]: - return True - return False + else: + return spec_matches_module_path(spec, path) class CouldNotResolvePathError(Exception): diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index d5358a6cd..6c5b2fb57 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -18,7 +18,7 @@ from typing import Tuple import unittest.mock from _pytest.monkeypatch import MonkeyPatch -from _pytest.pathlib import _is_namespace_package +from _pytest.pathlib import _is_importable from _pytest.pathlib import bestrelpath from _pytest.pathlib import commonpath from _pytest.pathlib import CouldNotResolvePathError @@ -723,12 +723,13 @@ class TestImportLibMode: assert result == "_env_310.tests.test_foo" def test_resolve_pkg_root_and_module_name( - self, tmp_path: Path, monkeypatch: MonkeyPatch + self, tmp_path: Path, monkeypatch: MonkeyPatch, pytester: Pytester ) -> None: # Create a directory structure first without __init__.py files. (tmp_path / "src/app/core").mkdir(parents=True) models_py = tmp_path / "src/app/core/models.py" models_py.touch() + with pytest.raises(CouldNotResolvePathError): _ = resolve_pkg_root_and_module_name(models_py) @@ -744,6 +745,8 @@ class TestImportLibMode: # If we add tmp_path to sys.path, src becomes a namespace package. monkeypatch.syspath_prepend(tmp_path) + validate_namespace_package(pytester, [tmp_path], ["src.app.core.models"]) + assert resolve_pkg_root_and_module_name( models_py, consider_namespace_packages=True ) == ( @@ -1143,7 +1146,7 @@ class TestNamespacePackages: algorithms_py.touch() # Validate the namespace package by importing it in a Python subprocess. - r = self.run_ns_imports( + r = validate_namespace_package( pytester, [tmp_path / "src/dist1", tmp_path / "src/dist2"], ["com.company.app.core.models", "com.company.calc.algo.algorithms"], @@ -1154,19 +1157,6 @@ class TestNamespacePackages: monkeypatch.syspath_prepend(tmp_path / "src/dist2") return models_py, algorithms_py - def run_ns_imports( - self, pytester: Pytester, paths: Sequence[Path], imports: Sequence[str] - ) -> RunResult: - """Validate a Python namespace package by importing modules from it in a Python subprocess""" - lines = [ - "import sys", - # Configure sys.path. - *[f"sys.path.append(r{str(x)!r})" for x in paths], - # Imports. - *[f"import {x}" for x in imports], - ] - return pytester.runpython_c("\n".join(lines)) - @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) def test_resolve_pkg_root_and_module_name_ns_multiple_levels( self, @@ -1242,7 +1232,7 @@ class TestNamespacePackages: (tmp_path / "src/dist1/com/__init__.py").touch() # Ensure Python no longer considers dist1/com a namespace package. - r = self.run_ns_imports( + r = validate_namespace_package( pytester, [tmp_path / "src/dist1", tmp_path / "src/dist2"], ["com.company.app.core.models", "com.company.calc.algo.algorithms"], @@ -1250,12 +1240,23 @@ class TestNamespacePackages: assert r.ret == 1 r.stderr.fnmatch_lines("*No module named 'com.company.calc*") + # dist1 is not a namespace package, but its module is importable; not being a namespace + # package prevents "com.company.calc" from being importable. pkg_root, module_name = resolve_pkg_root_and_module_name( models_py, consider_namespace_packages=True ) assert (pkg_root, module_name) == ( - tmp_path / "src/dist1/com/company", - "app.core.models", + tmp_path / "src/dist1", + "com.company.app.core.models", + ) + + # dist2/com/company will contain a normal Python package. + pkg_root, module_name = resolve_pkg_root_and_module_name( + algorithms_py, consider_namespace_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist2/com/company", + "calc.algo.algorithms", ) def test_detect_meta_path( @@ -1316,34 +1317,34 @@ class TestNamespacePackages: "com.company.app.core.models", ) - def test_is_namespace_package_bad_arguments(self, pytester: Pytester) -> None: + def test_is_importable_bad_arguments(self, pytester: Pytester) -> None: pytester.syspathinsert() path = pytester.path / "bar.x" path.mkdir() - assert _is_namespace_package(path) is False + assert _is_importable(path.parent, path) is False path = pytester.path / ".bar.x" path.mkdir() - assert _is_namespace_package(path) is False + assert _is_importable(path.parent, path) is False - assert _is_namespace_package(Path()) is False + assert _is_importable(Path(), Path()) is False - def test_intermediate_init_file( - self, pytester: Pytester, tmp_path: Path, monkeypatch: MonkeyPatch + @pytest.mark.parametrize("insert", [True, False]) + def test_full_ns_packages_without_init_files( + self, pytester: Pytester, tmp_path: Path, monkeypatch: MonkeyPatch, insert: bool ) -> None: (tmp_path / "src/dist1/ns/b/app/bar/test").mkdir(parents=True) - # (tmp_path / "src/dist1/ns/b/app/bar/test/__init__.py").touch() (tmp_path / "src/dist1/ns/b/app/bar/m.py").touch() - # The presence of this __init__.py is not a problem, ns.b.app is still part of the namespace package. - (tmp_path / "src/dist1/ns/b/app/__init__.py").touch() + if insert: + # The presence of this __init__.py is not a problem, ns.b.app is still part of the namespace package. + (tmp_path / "src/dist1/ns/b/app/__init__.py").touch() (tmp_path / "src/dist2/ns/a/core/foo/test").mkdir(parents=True) - # (tmp_path / "src/dist2/ns/a/core/foo/test/__init__.py").touch() (tmp_path / "src/dist2/ns/a/core/foo/m.py").touch() # Validate the namespace package by importing it in a Python subprocess. - r = self.run_ns_imports( + r = validate_namespace_package( pytester, [tmp_path / "src/dist1", tmp_path / "src/dist2"], ["ns.b.app.bar.m", "ns.a.core.foo.m"], @@ -1358,3 +1359,17 @@ class TestNamespacePackages: assert resolve_pkg_root_and_module_name( tmp_path / "src/dist2/ns/a/core/foo/m.py", consider_namespace_packages=True ) == (tmp_path / "src/dist2", "ns.a.core.foo.m") + + +def validate_namespace_package( + pytester: Pytester, paths: Sequence[Path], imports: Sequence[str] +) -> RunResult: + """Validate a Python namespace package by importing modules from it in a Python subprocess""" + lines = [ + "import sys", + # Configure sys.path. + *[f"sys.path.append(r{str(x)!r})" for x in paths], + # Imports. + *[f"import {x}" for x in imports], + ] + return pytester.runpython_c("\n".join(lines))