diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py
index c05d7925d..d8d11917a 100644
--- a/testing/test_stepwise.py
+++ b/testing/test_stepwise.py
@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+import sys
+
 import pytest
 
 
@@ -208,7 +210,8 @@ def test_xfail_handling(testdir):
 
     # because we are writing to the same file, mtime might not be affected enough to
     # invalidate the cache, making this next run flaky
-    testdir.tmpdir.join("__pycache__").remove()
+    if not sys.dont_write_bytecode:
+        testdir.tmpdir.join("__pycache__").remove()
     testdir.makepyfile(contents.format(assert_value="0", strict="True"))
     result = testdir.runpytest("--sw", "-v")
     result.stdout.fnmatch_lines(