diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py
index b3173d7d7..a3ab78503 100644
--- a/src/_pytest/config/__init__.py
+++ b/src/_pytest/config/__init__.py
@@ -528,9 +528,14 @@ class PytestPluginManager(PluginManager):
             if i != -1:
                 path = path[:i]
             anchor = absolutepath(current / path)
-            if anchor.exists():  # we found some file object
-                self._try_load_conftest(anchor, namespace.importmode, rootpath)
-                foundanchor = True
+            try:
+                if anchor.exists():  # we found some file object
+                    self._try_load_conftest(anchor, namespace.importmode, rootpath)
+                    foundanchor = True
+            except OSError:
+                import pytest
+
+                raise pytest.UsageError(f"file or directory not found: {anchor}")
         if not foundanchor:
             self._try_load_conftest(current, namespace.importmode, rootpath)