diff --git a/changelog/3973.bugfix.rst b/changelog/3973.bugfix.rst
new file mode 100644
index 000000000..29c70b840
--- /dev/null
+++ b/changelog/3973.bugfix.rst
@@ -0,0 +1 @@
+Fix crash of the assertion rewriter if a test changed the current working directory without restoring it afterwards.
diff --git a/src/_pytest/assertion/rewrite.py b/src/_pytest/assertion/rewrite.py
index 738d63396..2b3a0ab38 100644
--- a/src/_pytest/assertion/rewrite.py
+++ b/src/_pytest/assertion/rewrite.py
@@ -199,7 +199,11 @@ class AssertionRewritingHook(object):
 
         # For matching the name it must be as if it was a filename.
         parts[-1] = parts[-1] + ".py"
-        fn_pypath = py.path.local(os.path.sep.join(parts))
+        try:
+            fn_pypath = py.path.local(os.path.sep.join(parts))
+        except EnvironmentError:
+            return False
+
         for pat in self.fnpats:
             # if the pattern contains subdirectories ("tests/**.py" for example) we can't bail out based
             # on the name alone because we need to match against the full path
diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py
index aaf3e4785..394d30a05 100644
--- a/testing/test_assertrewrite.py
+++ b/testing/test_assertrewrite.py
@@ -1232,3 +1232,27 @@ class TestEarlyRewriteBailout(object):
         hook.fnpats[:] = ["tests/**.py"]
         assert hook.find_module("file") is not None
         assert self.find_module_calls == ["file"]
+
+    @pytest.mark.skipif(
+        sys.platform.startswith("win32"), reason="cannot remove cwd on Windows"
+    )
+    def test_cwd_changed(self, testdir):
+        testdir.makepyfile(
+            **{
+                "test_bar.py": """
+                import os
+                import shutil
+                import tempfile
+
+                d = tempfile.mkdtemp()
+                os.chdir(d)
+                shutil.rmtree(d)
+            """,
+                "test_foo.py": """
+                def test():
+                    pass
+            """,
+            }
+        )
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines("* 1 passed in *")