Support full namespace packages without __init__

src/_pytest/pathlib.py

@@ -8,6 +8,7 @@ from errno import ENOENT
 from errno import ENOTDIR
 import fnmatch
 from functools import partial
+from importlib.machinery import ModuleSpec
 import importlib.util
 import itertools
 import os
@@ -36,6 +37,8 @@ from typing import Union
 import uuid
 import warnings
 
+from typing_extensions import TypeGuard
+
 from _pytest.compat import assert_never
 from _pytest.outcomes import skip
 from _pytest.warning_types import PytestWarning
@@ -628,11 +631,12 @@ def _import_module_using_spec(
     # such as our own assertion-rewrite hook.
     for meta_importer in sys.meta_path:
         spec = meta_importer.find_spec(module_name, [str(module_location)])
-        if spec is not None:
+        if spec_matches_module_path(spec, module_path):
             break
     else:
         spec = importlib.util.spec_from_file_location(module_name, str(module_path))
-    if spec is not None:
+
+    if spec_matches_module_path(spec, module_path):
         mod = importlib.util.module_from_spec(spec)
         sys.modules[module_name] = mod
         spec.loader.exec_module(mod)  # type: ignore[union-attr]
@@ -643,6 +647,15 @@ def _import_module_using_spec(
     return None
 
 
+def spec_matches_module_path(
+    module_spec: Optional[ModuleSpec], module_path: Path
+) -> TypeGuard[ModuleSpec]:
+    if module_spec is not None and module_spec.origin is not None:
+        if Path(module_spec.origin) == module_path:
+            return True
+    return False
+
+
 # Implement a special _is_same function on Windows which returns True if the two filenames
 # compare equal, to circumvent os.path.samefile returning False for mounts in UNC (#7678).
 if sys.platform.startswith("win"):
@@ -768,21 +781,19 @@ def resolve_pkg_root_and_module_name(
 
     Raises CouldNotResolvePathError if the given path does not belong to a package (missing any __init__.py files).
     """
+    pkg_root: Optional[Path] = None
     pkg_path = resolve_package_path(path)
     if pkg_path is not None:
         pkg_root = pkg_path.parent
-        if consider_namespace_packages:
+    if consider_namespace_packages:
-            for candidate in (pkg_root, *pkg_root.parents):
+        start = path.parent if pkg_path is None else pkg_path.parent
-                # If any of the parent paths has an __init__.py, it means it is not a namespace package:
+        for candidate in (start, *start.parents):
-                # https://packaging.python.org/en/latest/guides/packaging-namespace-packages
+            if _is_importable(candidate, path):
-                if (candidate / "__init__.py").is_file():
+                # Point the pkg_root to the root of the namespace package.
-                    break
+                pkg_root = candidate.parent
-
+                break
-                if _is_namespace_package(candidate):
-                    # Point the pkg_root to the root of the namespace package.
-                    pkg_root = candidate.parent
-                    break
 
+    if pkg_root is not None:
         names = list(path.with_suffix("").relative_to(pkg_root).parts)
         if names[-1] == "__init__":
             names.pop()
@@ -792,25 +803,24 @@ def resolve_pkg_root_and_module_name(
     raise CouldNotResolvePathError(f"Could not resolve for {path}")
 
 
-def _is_namespace_package(module_path: Path) -> bool:
+def _is_importable(root: Path, path: Path) -> bool:
-    module_name = module_path.name
+    try:
-
+        path_without_suffix = path.with_suffix("")
-    # Empty module names (such as Path.cwd()) might break meta_path hooks (like our own assertion rewriter).
+    except ValueError:
-    if not module_name:
+        # Empty paths (such as Path.cwd()) might break meta_path hooks (like our own assertion rewriter).
         return False
 
+    names = list(path_without_suffix.relative_to(root.parent).parts)
+    if names[-1] == "__init__":
+        names.pop()
+    module_name = ".".join(names)
+
     try:
         spec = importlib.util.find_spec(module_name)
-    except ImportError:
+    except (ImportError, ValueError, ImportWarning):
         return False
-    if spec is not None and spec.submodule_search_locations:
+    else:
-        # Found a spec, however make sure the module_path is in one of the search locations --
+        return spec_matches_module_path(spec, path)
-        # this ensures common module name like "src" (which might be in sys.path under different locations)
-        # is only considered for the module_path we intend to.
-        # Make sure to compare Path(s) instead of strings, this normalizes them on Windows.
-        if module_path in [Path(x) for x in spec.submodule_search_locations]:
-            return True
-    return False
 
 
 class CouldNotResolvePathError(Exception):

testing/test_pathlib.py

@@ -18,7 +18,7 @@ from typing import Tuple
 import unittest.mock
 
 from _pytest.monkeypatch import MonkeyPatch
-from _pytest.pathlib import _is_namespace_package
+from _pytest.pathlib import _is_importable
 from _pytest.pathlib import bestrelpath
 from _pytest.pathlib import commonpath
 from _pytest.pathlib import CouldNotResolvePathError
@@ -723,12 +723,13 @@ class TestImportLibMode:
         assert result == "_env_310.tests.test_foo"
 
     def test_resolve_pkg_root_and_module_name(
-        self, tmp_path: Path, monkeypatch: MonkeyPatch
+        self, tmp_path: Path, monkeypatch: MonkeyPatch, pytester: Pytester
     ) -> None:
         # Create a directory structure first without __init__.py files.
         (tmp_path / "src/app/core").mkdir(parents=True)
         models_py = tmp_path / "src/app/core/models.py"
         models_py.touch()
+
         with pytest.raises(CouldNotResolvePathError):
             _ = resolve_pkg_root_and_module_name(models_py)
 
@@ -744,6 +745,8 @@ class TestImportLibMode:
 
         # If we add tmp_path to sys.path, src becomes a namespace package.
         monkeypatch.syspath_prepend(tmp_path)
+        validate_namespace_package(pytester, [tmp_path], ["src.app.core.models"])
+
         assert resolve_pkg_root_and_module_name(
             models_py, consider_namespace_packages=True
         ) == (
@@ -1143,7 +1146,7 @@ class TestNamespacePackages:
         algorithms_py.touch()
 
         # Validate the namespace package by importing it in a Python subprocess.
-        r = self.run_ns_imports(
+        r = validate_namespace_package(
             pytester,
             [tmp_path / "src/dist1", tmp_path / "src/dist2"],
             ["com.company.app.core.models", "com.company.calc.algo.algorithms"],
@@ -1154,19 +1157,6 @@ class TestNamespacePackages:
             monkeypatch.syspath_prepend(tmp_path / "src/dist2")
         return models_py, algorithms_py
 
-    def run_ns_imports(
-        self, pytester: Pytester, paths: Sequence[Path], imports: Sequence[str]
-    ) -> RunResult:
-        """Validate a Python namespace package by importing modules from  it in a Python subprocess"""
-        lines = [
-            "import sys",
-            # Configure sys.path.
-            *[f"sys.path.append(r{str(x)!r})" for x in paths],
-            # Imports.
-            *[f"import {x}" for x in imports],
-        ]
-        return pytester.runpython_c("\n".join(lines))
-
     @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"])
     def test_resolve_pkg_root_and_module_name_ns_multiple_levels(
         self,
@@ -1242,7 +1232,7 @@ class TestNamespacePackages:
         (tmp_path / "src/dist1/com/__init__.py").touch()
 
         # Ensure Python no longer considers dist1/com a namespace package.
-        r = self.run_ns_imports(
+        r = validate_namespace_package(
             pytester,
             [tmp_path / "src/dist1", tmp_path / "src/dist2"],
             ["com.company.app.core.models", "com.company.calc.algo.algorithms"],
@@ -1250,12 +1240,23 @@ class TestNamespacePackages:
         assert r.ret == 1
         r.stderr.fnmatch_lines("*No module named 'com.company.calc*")
 
+        # dist1 is not a namespace package, but its module is importable; not being a namespace
+        # package prevents "com.company.calc" from being importable.
         pkg_root, module_name = resolve_pkg_root_and_module_name(
             models_py, consider_namespace_packages=True
         )
         assert (pkg_root, module_name) == (
-            tmp_path / "src/dist1/com/company",
+            tmp_path / "src/dist1",
-            "app.core.models",
+            "com.company.app.core.models",
+        )
+
+        # dist2/com/company will contain a normal Python package.
+        pkg_root, module_name = resolve_pkg_root_and_module_name(
+            algorithms_py, consider_namespace_packages=True
+        )
+        assert (pkg_root, module_name) == (
+            tmp_path / "src/dist2/com/company",
+            "calc.algo.algorithms",
         )
 
     def test_detect_meta_path(
@@ -1316,34 +1317,34 @@ class TestNamespacePackages:
             "com.company.app.core.models",
         )
 
-    def test_is_namespace_package_bad_arguments(self, pytester: Pytester) -> None:
+    def test_is_importable_bad_arguments(self, pytester: Pytester) -> None:
         pytester.syspathinsert()
         path = pytester.path / "bar.x"
         path.mkdir()
-        assert _is_namespace_package(path) is False
+        assert _is_importable(path.parent, path) is False
 
         path = pytester.path / ".bar.x"
         path.mkdir()
-        assert _is_namespace_package(path) is False
+        assert _is_importable(path.parent, path) is False
 
-        assert _is_namespace_package(Path()) is False
+        assert _is_importable(Path(), Path()) is False
 
-    def test_intermediate_init_file(
+    @pytest.mark.parametrize("insert", [True, False])
-        self, pytester: Pytester, tmp_path: Path, monkeypatch: MonkeyPatch
+    def test_full_ns_packages_without_init_files(
+        self, pytester: Pytester, tmp_path: Path, monkeypatch: MonkeyPatch, insert: bool
     ) -> None:
         (tmp_path / "src/dist1/ns/b/app/bar/test").mkdir(parents=True)
-        # (tmp_path / "src/dist1/ns/b/app/bar/test/__init__.py").touch()
         (tmp_path / "src/dist1/ns/b/app/bar/m.py").touch()
 
-        # The presence of this __init__.py is not a problem, ns.b.app is still part of the namespace package.
+        if insert:
-        (tmp_path / "src/dist1/ns/b/app/__init__.py").touch()
+            # The presence of this __init__.py is not a problem, ns.b.app is still part of the namespace package.
+            (tmp_path / "src/dist1/ns/b/app/__init__.py").touch()
 
         (tmp_path / "src/dist2/ns/a/core/foo/test").mkdir(parents=True)
-        # (tmp_path / "src/dist2/ns/a/core/foo/test/__init__.py").touch()
         (tmp_path / "src/dist2/ns/a/core/foo/m.py").touch()
 
         # Validate the namespace package by importing it in a Python subprocess.
-        r = self.run_ns_imports(
+        r = validate_namespace_package(
             pytester,
             [tmp_path / "src/dist1", tmp_path / "src/dist2"],
             ["ns.b.app.bar.m", "ns.a.core.foo.m"],
@@ -1358,3 +1359,17 @@ class TestNamespacePackages:
         assert resolve_pkg_root_and_module_name(
             tmp_path / "src/dist2/ns/a/core/foo/m.py", consider_namespace_packages=True
         ) == (tmp_path / "src/dist2", "ns.a.core.foo.m")
+
+
+def validate_namespace_package(
+    pytester: Pytester, paths: Sequence[Path], imports: Sequence[str]
+) -> RunResult:
+    """Validate a Python namespace package by importing modules from  it in a Python subprocess"""
+    lines = [
+        "import sys",
+        # Configure sys.path.
+        *[f"sys.path.append(r{str(x)!r})" for x in paths],
+        # Imports.
+        *[f"import {x}" for x in imports],
+    ]
+    return pytester.runpython_c("\n".join(lines))