Merge pull request #7541 from bluetech/py-visit

pathlib: stop using py.path.local.visit(), use os.scandir

src/_pytest/main.py

@@ -32,6 +32,7 @@ from _pytest.config.argparsing import Parser
 from _pytest.fixtures import FixtureManager
 from _pytest.outcomes import exit
 from _pytest.pathlib import Path
+from _pytest.pathlib import visit
 from _pytest.reports import CollectReport
 from _pytest.reports import TestReport
 from _pytest.runner import collect_one_node
@@ -617,10 +618,13 @@ class Session(nodes.FSCollector):
             assert not names, "invalid arg {!r}".format((argpath, names))
 
             seen_dirs = set()  # type: Set[py.path.local]
-            for path in argpath.visit(
-                fil=self._visit_filter, rec=self._recurse, bf=True, sort=True
-            ):
+            for direntry in visit(str(argpath), self._recurse):
+                if not direntry.is_file():
+                    continue
+
+                path = py.path.local(direntry.path)
                 dirpath = path.dirpath()
+
                 if dirpath not in seen_dirs:
                     # Collect packages first.
                     seen_dirs.add(dirpath)
@@ -668,11 +672,6 @@ class Session(nodes.FSCollector):
                 return
             yield from m
 
-    @staticmethod
-    def _visit_filter(f: py.path.local) -> bool:
-        # TODO: Remove type: ignore once `py` is typed.
-        return f.check(file=1)  # type: ignore
-
     def _tryconvertpyarg(self, x: str) -> str:
         """Convert a dotted module name to path."""
         try:

src/_pytest/nodes.py

@@ -562,17 +562,18 @@ class FSCollector(Collector):
     def gethookproxy(self, fspath: py.path.local):
         raise NotImplementedError()
 
-    def _recurse(self, dirpath: py.path.local) -> bool:
-        if dirpath.basename == "__pycache__":
+    def _recurse(self, direntry: "os.DirEntry[str]") -> bool:
+        if direntry.name == "__pycache__":
             return False
-        ihook = self._gethookproxy(dirpath.dirpath())
-        if ihook.pytest_ignore_collect(path=dirpath, config=self.config):
+        path = py.path.local(direntry.path)
+        ihook = self._gethookproxy(path.dirpath())
+        if ihook.pytest_ignore_collect(path=path, config=self.config):
             return False
         for pat in self._norecursepatterns:
-            if dirpath.check(fnmatch=pat):
+            if path.check(fnmatch=pat):
                 return False
-        ihook = self._gethookproxy(dirpath)
-        ihook.pytest_collect_directory(path=dirpath, parent=self)
+        ihook = self._gethookproxy(path)
+        ihook.pytest_collect_directory(path=path, parent=self)
         return True
 
     def isinitpath(self, path: py.path.local) -> bool:

src/_pytest/pathlib.py

@@ -16,6 +16,7 @@ from os.path import isabs
 from os.path import sep
 from posixpath import sep as posix_sep
 from types import ModuleType
+from typing import Callable
 from typing import Iterable
 from typing import Iterator
 from typing import Optional
@@ -556,3 +557,17 @@ def resolve_package_path(path: Path) -> Optional[Path]:
                 break
             result = parent
     return result
+
+
+def visit(
+    path: str, recurse: Callable[["os.DirEntry[str]"], bool]
+) -> Iterator["os.DirEntry[str]"]:
+    """Walk a directory recursively, in breadth-first order.
+
+    Entries at each directory level are sorted.
+    """
+    entries = sorted(os.scandir(path), key=lambda entry: entry.name)
+    yield from entries
+    for entry in entries:
+        if entry.is_dir(follow_symlinks=False) and recurse(entry):
+            yield from visit(entry.path, recurse)

src/_pytest/python.py

@@ -65,6 +65,7 @@ from _pytest.outcomes import skip
 from _pytest.pathlib import import_path
 from _pytest.pathlib import ImportPathMismatchError
 from _pytest.pathlib import parts
+from _pytest.pathlib import visit
 from _pytest.reports import TerminalRepr
 from _pytest.warning_types import PytestCollectionWarning
 from _pytest.warning_types import PytestUnhandledCoroutineWarning
@@ -641,23 +642,24 @@ class Package(Module):
         ):
             yield Module.from_parent(self, fspath=init_module)
         pkg_prefixes = set()  # type: Set[py.path.local]
-        for path in this_path.visit(rec=self._recurse, bf=True, sort=True):
+        for direntry in visit(str(this_path), recurse=self._recurse):
+            path = py.path.local(direntry.path)
+
             # We will visit our own __init__.py file, in which case we skip it.
-            is_file = path.isfile()
-            if is_file:
-                if path.basename == "__init__.py" and path.dirpath() == this_path:
+            if direntry.is_file():
+                if direntry.name == "__init__.py" and path.dirpath() == this_path:
                     continue
 
-            parts_ = parts(path.strpath)
+            parts_ = parts(direntry.path)
             if any(
                 str(pkg_prefix) in parts_ and pkg_prefix.join("__init__.py") != path
                 for pkg_prefix in pkg_prefixes
             ):
                 continue
 
-            if is_file:
+            if direntry.is_file():
                 yield from self._collectfile(path)
-            elif not path.isdir():
+            elif not direntry.is_dir():
                 # Broken symlink or invalid/missing file.
                 continue
             elif path.join("__init__.py").check(file=1):