From 62810f61b22341bc2b0a09da30252dfb0f57e027 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 26 Jul 2017 21:06:08 -0300 Subject: [PATCH] Make cache plugin always remember failed tests --- _pytest/cacheprovider.py | 26 ++++++++----------- changelog/2621.feature | 2 ++ testing/test_cache.py | 55 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 changelog/2621.feature diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py index 5f2c6b062..14fd86f6b 100755 --- a/_pytest/cacheprovider.py +++ b/_pytest/cacheprovider.py @@ -105,27 +105,22 @@ class LFPlugin: self.config = config active_keys = 'lf', 'failedfirst' self.active = any(config.getvalue(key) for key in active_keys) - if self.active: - self.lastfailed = config.cache.get("cache/lastfailed", {}) - else: - self.lastfailed = {} + self.lastfailed = config.cache.get("cache/lastfailed", {}) def pytest_report_header(self): if self.active: if not self.lastfailed: mode = "run all (no recorded failures)" else: - mode = "rerun last %d failures%s" % ( - len(self.lastfailed), + mode = "rerun previous failures%s" % ( " first" if self.config.getvalue("failedfirst") else "") return "run-last-failure: %s" % mode def pytest_runtest_logreport(self, report): - if report.failed and "xfail" not in report.keywords: + if report.passed and report.when == 'call': + self.lastfailed.pop(report.nodeid, None) + elif report.failed: self.lastfailed[report.nodeid] = True - elif not report.failed: - if report.when == "call": - self.lastfailed.pop(report.nodeid, None) def pytest_collectreport(self, report): passed = report.outcome in ('passed', 'skipped') @@ -147,11 +142,11 @@ class LFPlugin: previously_failed.append(item) else: previously_passed.append(item) - if not previously_failed and previously_passed: + if not previously_failed: # running a subset of all tests with recorded failures outside # of the set of tests currently executing - pass - elif self.config.getvalue("lf"): + return + if self.config.getvalue("lf"): items[:] = previously_failed config.hook.pytest_deselected(items=previously_passed) else: @@ -161,8 +156,9 @@ class LFPlugin: config = self.config if config.getvalue("cacheshow") or hasattr(config, "slaveinput"): return - prev_failed = config.cache.get("cache/lastfailed", None) is not None - if (session.testscollected and prev_failed) or self.lastfailed: + + saved_lastfailed = config.cache.get("cache/lastfailed", {}) + if saved_lastfailed != self.lastfailed: config.cache.set("cache/lastfailed", self.lastfailed) diff --git a/changelog/2621.feature b/changelog/2621.feature new file mode 100644 index 000000000..19ca96355 --- /dev/null +++ b/changelog/2621.feature @@ -0,0 +1,2 @@ +``--last-failed`` now remembers forever when a test has failed and only forgets it if it passes again. This makes it +easy to fix a test suite by selectively running files and fixing tests incrementally. diff --git a/testing/test_cache.py b/testing/test_cache.py index 36059ec29..04ce6671c 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -437,3 +437,58 @@ class TestLastFailed(object): testdir.makepyfile(test_errored='def test_error():\n assert False') testdir.runpytest('-q', '--lf') assert os.path.exists('.cache') + + def get_cached_last_failed(self, testdir): + config = testdir.parseconfigure() + return sorted(config.cache.get("cache/lastfailed", {})) + + def test_cache_cumulative(self, testdir): + """ + Test workflow where user fixes errors gradually file by file using --lf. + """ + # 1. initial run + test_bar = testdir.makepyfile(test_bar=""" + def test_bar_1(): + pass + def test_bar_2(): + assert 0 + """) + test_foo = testdir.makepyfile(test_foo=""" + def test_foo_3(): + pass + def test_foo_4(): + assert 0 + """) + testdir.runpytest() + assert self.get_cached_last_failed(testdir) == ['test_bar.py::test_bar_2', 'test_foo.py::test_foo_4'] + + # 2. fix test_bar_2, run only test_bar.py + testdir.makepyfile(test_bar=""" + def test_bar_1(): + pass + def test_bar_2(): + pass + """) + result = testdir.runpytest(test_bar) + result.stdout.fnmatch_lines('*2 passed*') + # ensure cache does not forget that test_foo_4 failed once before + assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4'] + + result = testdir.runpytest('--last-failed') + result.stdout.fnmatch_lines('*1 failed, 3 deselected*') + assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4'] + + # 3. fix test_foo_4, run only test_foo.py + test_foo = testdir.makepyfile(test_foo=""" + def test_foo_3(): + pass + def test_foo_4(): + pass + """) + result = testdir.runpytest(test_foo, '--last-failed') + result.stdout.fnmatch_lines('*1 passed, 1 deselected*') + assert self.get_cached_last_failed(testdir) == [] + + result = testdir.runpytest('--last-failed') + result.stdout.fnmatch_lines('*4 passed*') + assert self.get_cached_last_failed(testdir) == []