Prepare release version 8.2.2

[8.2.x] fixtures: fix catastrophic performance problem in `reorder_items`

AUTHORS

@@ -192,6 +192,7 @@ Jake VanderPlas
 Jakob van Santen
 Jakub Mitoraj
 James Bourbeau
+James Frost
 Jan Balster
 Janne Vanhala
 Jason R. Coombs
@@ -289,6 +290,7 @@ Mike Lundy
 Milan Lesnek
 Miro Hrončok
 mrbean-bremen
+Nathan Goldbaum
 Nathaniel Compton
 Nathaniel Waisbrot
 Ned Batchelder
@@ -358,6 +360,7 @@ Sadra Barikbin
 Saiprasad Kale
 Samuel Colvin
 Samuel Dion-Girardeau
+Samuel Jirovec
 Samuel Searles-Bryant
 Samuel Therrien (Avasam)
 Samuele Pedroni

CONTRIBUTING.rst

@@ -5,10 +5,6 @@ Contribution getting started
 Contributions are highly welcomed and appreciated.  Every little bit of help counts,
 so do not hesitate!
 
-.. contents::
-   :depth: 2
-   :backlinks: none
-
 
 .. _submitfeedback:
 

doc/en/_templates/sidebar/brand.html

@@ -0,0 +1,7 @@
+<style>
+    .logo {text-align: center;}
+</style>
+
+<a class="logo" href="{{ pathto('contents') }}">
+    <img src="{{ pathto('_static/pytest1.png', 1) }}" width="70%" height="70%" text="Pytest Logo"/>
+</a>

doc/en/_templates/slim_searchbox.html

@@ -1,14 +0,0 @@
-{#
-    basic/searchbox.html with heading removed.
-#}
-{%- if pagename != "search" and builder != "singlehtml" %}
-<div id="searchbox" style="display: none" role="search">
-  <div class="searchformwrapper">
-    <form class="search" action="{{ pathto('search') }}" method="get">
-      <input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
-      <input type="submit" value="{{ _('Go') }}" />
-    </form>
-  </div>
-</div>
-<script>document.getElementById('searchbox').style.display = "block"</script>
-{%- endif %}

doc/en/_templates/style.html

@@ -0,0 +1,7 @@
+<style>
+    ul {list-style: none;}
+    li {margin: 0.4em 0;}
+    @media (min-width: 46em) {
+        #features {width: 50%;}
+    }
+</style>

doc/en/announce/index.rst

@@ -6,6 +6,7 @@ Release announcements
    :maxdepth: 2
 
 
+   release-8.2.2
    release-8.2.1
    release-8.2.0
    release-8.1.2

doc/en/announce/release-8.2.2.rst

@@ -0,0 +1,19 @@
+pytest-8.2.2
+=======================================
+
+pytest 8.2.2 has just been released to PyPI.
+
+This is a bug-fix release, being a drop-in replacement. To upgrade::
+
+  pip install --upgrade pytest
+
+The full changelog is available at https://docs.pytest.org/en/stable/changelog.html.
+
+Thanks to all of the contributors to this release:
+
+* Bruno Oliveira
+* Ran Benita
+
+
+Happy testing,
+The pytest Development Team

doc/en/builtin.rst

@@ -22,7 +22,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a
     cachedir: .pytest_cache
     rootdir: /home/sweet/project
     collected 0 items
-    cache -- .../_pytest/cacheprovider.py:549
+    cache -- .../_pytest/cacheprovider.py:560
         Return a cache object that can persist state between testing sessions.
 
         cache.get(key, default)
@@ -115,7 +115,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a
 
         For more details: :ref:`doctest_namespace`.
 
-    pytestconfig [session scope] -- .../_pytest/fixtures.py:1335
+    pytestconfig [session scope] -- .../_pytest/fixtures.py:1338
         Session-scoped fixture that returns the session's :class:`pytest.Config`
         object.
 

doc/en/changelog.rst

@@ -28,6 +28,35 @@ with advance notice in the **Deprecations** section of releases.
 
 .. towncrier release notes start
 
+pytest 8.2.2 (2024-06-04)
+=========================
+
+Bug Fixes
+---------
+
+- `#12355 <https://github.com/pytest-dev/pytest/issues/12355>`_: Fix possible catastrophic performance slowdown on a certain parametrization pattern involving many higher-scoped parameters.
+
+
+- `#12367 <https://github.com/pytest-dev/pytest/issues/12367>`_: Fix a regression in pytest 8.2.0 where unittest class instances (a fresh one is created for each test) were not released promptly on test teardown but only on session teardown.
+
+
+- `#12381 <https://github.com/pytest-dev/pytest/issues/12381>`_: Fix possible "Directory not empty" crashes arising from concurent cache dir (``.pytest_cache``) creation. Regressed in pytest 8.2.0.
+
+
+
+Improved Documentation
+----------------------
+
+- `#12290 <https://github.com/pytest-dev/pytest/issues/12290>`_: Updated Sphinx theme to use Furo instead of Flask, enabling Dark mode theme.
+
+
+- `#12356 <https://github.com/pytest-dev/pytest/issues/12356>`_: Added a subsection to the documentation for debugging flaky tests to mention
+  lack of thread safety in pytest as a possible source of flakyness.
+
+
+- `#12363 <https://github.com/pytest-dev/pytest/issues/12363>`_: The documentation webpages now links to a canonical version to reduce outdated documentation in search engine results.
+
+
 pytest 8.2.1 (2024-05-19)
 =========================
 
@@ -57,7 +86,7 @@ Bug Fixes
 Trivial/Internal Changes
 ------------------------
 
-- `#12333 <https://github.com/pytest-dev/pytest/issues/12333>`_: pytest releases are now attested using the recent `Artifact Attestation <https://github.blog/2024-05-02-introducing-artifact-attestations-now-in-public-beta/>` support from GitHub, allowing users to verify the provenance of pytest's sdist and wheel artifacts.
+- `#12333 <https://github.com/pytest-dev/pytest/issues/12333>`_: pytest releases are now attested using the recent `Artifact Attestation <https://github.blog/2024-05-02-introducing-artifact-attestations-now-in-public-beta/>`_ support from GitHub, allowing users to verify the provenance of pytest's sdist and wheel artifacts.
 
 
 pytest 8.2.0 (2024-04-27)

doc/en/conf.py

@@ -15,9 +15,7 @@
 #
 # The full version, including alpha/beta/rc tags.
 # The short X.Y version.
-import os
 import shutil
-import sys
 from textwrap import dedent
 from typing import TYPE_CHECKING
 
@@ -65,7 +63,6 @@ latex_elements = {
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
-    "pallets_sphinx_themes",
     "pygments_pytest",
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
@@ -140,10 +137,6 @@ add_module_names = False
 # output. They are ignored by default.
 # show_authors = False
 
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
-
-
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
 
@@ -216,12 +209,9 @@ nitpick_ignore = [
 
 # -- Options for HTML output ---------------------------------------------------
 
-sys.path.append(os.path.abspath("_themes"))
-html_theme_path = ["_themes"]
-
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = "flask"
+html_theme = "furo"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -266,18 +256,24 @@ html_favicon = "img/favicon.png"
 
 html_sidebars = {
     "index": [
-        "slim_searchbox.html",
+        "sidebar/brand.html",
+        "sidebar/search.html",
+        "sidebar/scroll-start.html",
         "sidebarintro.html",
         "globaltoc.html",
         "links.html",
-        "sourcelink.html",
+        "sidebar/scroll-end.html",
+        "style.html",
     ],
     "**": [
-        "slim_searchbox.html",
+        "sidebar/brand.html",
+        "sidebar/search.html",
+        "sidebar/scroll-start.html",
         "globaltoc.html",
         "relations.html",
         "links.html",
-        "sourcelink.html",
+        "sidebar/scroll-end.html",
+        "style.html",
     ],
 }
 
@@ -316,6 +312,9 @@ html_show_sourcelink = False
 # Output file base name for HTML help builder.
 htmlhelp_basename = "pytestdoc"
 
+# The base URL which points to the root of the HTML documentation. It is used
+# to indicate the location of document using the canonical link relation (#12363).
+html_baseurl = "https://docs.pytest.org/en/stable/"
 
 # -- Options for LaTeX output --------------------------------------------------
 
@@ -337,10 +336,6 @@ latex_documents = [
     )
 ]
 
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-latex_logo = "img/pytest1.png"
-
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
 # latex_use_parts = False

doc/en/deprecations.rst

@@ -7,10 +7,6 @@ This page lists all pytest features that are currently deprecated or have been r
 The objective is to give users a clear rationale why a certain feature has been removed, and what alternatives
 should be used instead.
 
-.. contents::
-    :depth: 3
-    :local:
-
 
 Deprecated Features
 -------------------

doc/en/example/parametrize.rst

@@ -162,7 +162,7 @@ objects, they are still using the default pytest representation:
     rootdir: /home/sweet/project
     collected 8 items
 
-    <Dir parametrize.rst-199>
+    <Dir parametrize.rst-200>
       <Module test_time.py>
         <Function test_timedistance_v0[a0-b0-expected0]>
         <Function test_timedistance_v0[a1-b1-expected1]>
@@ -239,7 +239,7 @@ If you just collect tests you'll also nicely see 'advanced' and 'basic' as varia
     rootdir: /home/sweet/project
     collected 4 items
 
-    <Dir parametrize.rst-199>
+    <Dir parametrize.rst-200>
       <Module test_scenarios.py>
         <Class TestSampleWithScenarios>
           <Function test_demo1[basic]>
@@ -318,7 +318,7 @@ Let's first see how it looks like at collection time:
     rootdir: /home/sweet/project
     collected 2 items
 
-    <Dir parametrize.rst-199>
+    <Dir parametrize.rst-200>
       <Module test_backends.py>
         <Function test_db_initialized[d1]>
         <Function test_db_initialized[d2]>

doc/en/example/pythoncollection.rst

@@ -152,7 +152,7 @@ The test collection would look like this:
     configfile: pytest.ini
     collected 2 items
 
-    <Dir pythoncollection.rst-200>
+    <Dir pythoncollection.rst-201>
       <Module check_myapp.py>
         <Class CheckMyApp>
           <Function simple_check>
@@ -215,7 +215,7 @@ You can always peek at the collection tree without running tests like this:
     configfile: pytest.ini
     collected 3 items
 
-    <Dir pythoncollection.rst-200>
+    <Dir pythoncollection.rst-201>
       <Dir CWD>
         <Module pythoncollection.py>
           <Function test_function>

doc/en/explanation/flaky.rst

@@ -18,7 +18,7 @@ System state
 
 Broadly speaking, a flaky test indicates that the test relies on some system state that is not being appropriately controlled - the test environment is not sufficiently isolated. Higher level tests are more likely to be flaky as they rely on more state.
 
-Flaky tests sometimes appear when a test suite is run in parallel (such as use of pytest-xdist). This can indicate a test is reliant on test ordering.
+Flaky tests sometimes appear when a test suite is run in parallel (such as use of `pytest-xdist`_). This can indicate a test is reliant on test ordering.
 
 -  Perhaps a different test is failing to clean up after itself and leaving behind data which causes the flaky test to fail.
 - The flaky test is reliant on data from a previous test that doesn't clean up after itself, and in parallel runs that previous test is not always present
@@ -30,9 +30,22 @@ Overly strict assertion
 
 Overly strict assertions can cause problems with floating point comparison as well as timing issues. :func:`pytest.approx` is useful here.
 
+Thread safety
+~~~~~~~~~~~~~
 
-Pytest features
-^^^^^^^^^^^^^^^
+pytest is single-threaded, executing its tests always in the same thread, sequentially, never spawning any threads itself.
+
+Even in case of plugins which run tests in parallel, for example `pytest-xdist`_, usually work by spawning multiple *processes* and running tests in batches, without using multiple threads.
+
+It is of course possible (and common) for tests and fixtures to spawn threads themselves as part of their testing workflow (for example, a fixture that starts a server thread in the background, or a test which executes production code that spawns threads), but some care must be taken:
+
+* Make sure to eventually wait on any spawned threads -- for example at the end of a test, or during the teardown of a fixture.
+* Avoid using primitives provided by pytest (:func:`pytest.warns`, :func:`pytest.raises`, etc) from multiple threads, as they are not thread-safe.
+
+If your test suite uses threads and your are seeing flaky test results, do not discount the possibility that the test is implicitly using global state in pytest itself.
+
+Related features
+^^^^^^^^^^^^^^^^
 
 Xfail strict
 ~~~~~~~~~~~~
@@ -123,3 +136,6 @@ Resources
 
   * `Flaky Tests at Google and How We Mitigate Them <https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html>`_ by John Micco, 2016
   * `Where do Google's flaky tests come from? <https://testing.googleblog.com/2017/04/where-do-our-flaky-tests-come-from.html>`_  by Jeff Listfield, 2017
+
+
+.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist

doc/en/getting-started.rst

@@ -22,7 +22,7 @@ Install ``pytest``
 .. code-block:: bash
 
     $ pytest --version
-    pytest 8.2.1
+    pytest 8.2.2
 
 .. _`simpletest`:
 

doc/en/how-to/fixtures.rst

@@ -1418,7 +1418,7 @@ Running the above tests results in the following test IDs being used:
    rootdir: /home/sweet/project
    collected 12 items
 
-   <Dir fixtures.rst-218>
+   <Dir fixtures.rst-219>
      <Module test_anothersmtp.py>
        <Function test_showhelo[smtp.gmail.com]>
        <Function test_showhelo[mail.python.org]>

doc/en/index.rst

@@ -1,17 +1,16 @@
 :orphan:
 
-.. sidebar:: Next Open Trainings and Events
-
-   - `Professional Testing with Python <https://python-academy.com/courses/python_course_testing.html>`_, via `Python Academy <https://www.python-academy.com/>`_ (3 day in-depth training):
-      * **June 11th to 13th 2024**, Remote
-      * **March 4th to 6th 2025**, Leipzig, Germany / Remote
-   - `pytest development sprint <https://github.com/pytest-dev/sprint>`_, **June 17th -- 22nd 2024**
-   - pytest tips and tricks for a better testsuite, `Europython 2024 <https://ep2024.europython.eu/>`_, **July 8th -- 14th 2024** (3h), Prague
-
-   Also see :doc:`previous talks and blogposts <talks>`.
-
 .. _features:
 
+.. sidebar:: **Next Open Trainings and Events**
+
+    - `pytest development sprint <https://github.com/pytest-dev/sprint>`_, **June 17th -- 22nd 2024**, Klaus (AT) / Remote
+    - `pytest tips and tricks for a better testsuite <https://ep2024.europython.eu/session/pytest-tips-and-tricks-for-a-better-testsuite>`_, at `Europython 2024 <https://ep2024.europython.eu/>`_, **July 8th -- 14th 2024** (3h), Prague (CZ)
+    - `pytest: Professionelles Testen (nicht nur) für Python <https://pretalx.com/workshoptage-2024/talk/9VUHYB/>`_, at `CH Open Workshoptage <https://workshoptage.ch/>`_, **September 2nd 2024**, HSLU Rotkreuz (CH)
+    - `Professional Testing with Python <https://python-academy.com/courses/python_course_testing.html>`_, via `Python Academy <https://www.python-academy.com/>`_ (3 day in-depth training), **March 4th -- 6th 2025**, Leipzig (DE) / Remote
+
+    Also see :doc:`previous talks and blogposts <talks>`
+
 pytest: helps you write better programs
 =======================================
 
@@ -25,7 +24,6 @@ scale to support complex functional testing for applications and libraries.
 
 **PyPI package name**: :pypi:`pytest`
 
-
 A quick example
 ---------------
 

doc/en/reference/reference.rst

@@ -7,9 +7,6 @@ API Reference
 
 This page contains the full reference to pytest's API.
 
-.. contents::
-    :depth: 3
-    :local:
 
 Constants
 ---------

doc/en/requirements.txt

@@ -1,4 +1,3 @@
-pallets-sphinx-themes
 pluggy>=1.5.0
 pygments-pytest>=2.3.0
 sphinx-removed-in>=0.2.0
@@ -8,4 +7,5 @@ sphinxcontrib-svg2pdfconverter
 # Pin packaging because it no longer handles 'latest' version, which
 # is the version that is assigned to the docs.
 # See https://github.com/pytest-dev/pytest/pull/10578#issuecomment-1348249045.
-packaging <22
+packaging
+furo

src/_pytest/cacheprovider.py

@@ -4,6 +4,7 @@
 # This plugin was not named "cache" to avoid conflicts with the external
 # pytest-cache version.
 import dataclasses
+import errno
 import json
 import os
 from pathlib import Path
@@ -227,14 +228,24 @@ class Cache:
             with open(path.joinpath("CACHEDIR.TAG"), "xb") as f:
                 f.write(CACHEDIR_TAG_CONTENT)
 
-            path.rename(self._cachedir)
-            # Create a directory in place of the one we just moved so that `TemporaryDirectory`'s
-            # cleanup doesn't complain.
-            #
-            # TODO: pass ignore_cleanup_errors=True when we no longer support python < 3.10. See
-            # https://github.com/python/cpython/issues/74168. Note that passing delete=False would
-            # do the wrong thing in case of errors and isn't supported until python 3.12.
-            path.mkdir()
+            try:
+                path.rename(self._cachedir)
+            except OSError as e:
+                # If 2 concurrent pytests both race to the rename, the loser
+                # gets "Directory not empty" from the rename. In this case,
+                # everything is handled so just continue (while letting the
+                # temporary directory be cleaned up).
+                if e.errno != errno.ENOTEMPTY:
+                    raise
+            else:
+                # Create a directory in place of the one we just moved so that
+                # `TemporaryDirectory`'s cleanup doesn't complain.
+                #
+                # TODO: pass ignore_cleanup_errors=True when we no longer support python < 3.10.
+                # See https://github.com/python/cpython/issues/74168. Note that passing
+                # delete=False would do the wrong thing in case of errors and isn't supported
+                # until python 3.12.
+                path.mkdir()
 
 
 class LFPluginCollWrapper:

src/_pytest/fixtures.py

@@ -23,6 +23,7 @@ from typing import List
 from typing import MutableMapping
 from typing import NoReturn
 from typing import Optional
+from typing import OrderedDict
 from typing import overload
 from typing import Sequence
 from typing import Set
@@ -75,8 +76,6 @@ if sys.version_info < (3, 11):
 
 
 if TYPE_CHECKING:
-    from typing import Deque
-
     from _pytest.main import Session
     from _pytest.python import CallSpec2
     from _pytest.python import Function
@@ -207,16 +206,18 @@ def get_parametrized_fixture_keys(
 
 def reorder_items(items: Sequence[nodes.Item]) -> List[nodes.Item]:
     argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[FixtureArgKey, None]]] = {}
-    items_by_argkey: Dict[Scope, Dict[FixtureArgKey, Deque[nodes.Item]]] = {}
+    items_by_argkey: Dict[
+        Scope, Dict[FixtureArgKey, OrderedDict[nodes.Item, None]]
+    ] = {}
     for scope in HIGH_SCOPES:
         scoped_argkeys_cache = argkeys_cache[scope] = {}
-        scoped_items_by_argkey = items_by_argkey[scope] = defaultdict(deque)
+        scoped_items_by_argkey = items_by_argkey[scope] = defaultdict(OrderedDict)
         for item in items:
             keys = dict.fromkeys(get_parametrized_fixture_keys(item, scope), None)
             if keys:
                 scoped_argkeys_cache[item] = keys
                 for key in keys:
-                    scoped_items_by_argkey[key].append(item)
+                    scoped_items_by_argkey[key][item] = None
     items_dict = dict.fromkeys(items, None)
     return list(
         reorder_items_atscope(items_dict, argkeys_cache, items_by_argkey, Scope.Session)
@@ -226,17 +227,19 @@ def reorder_items(items: Sequence[nodes.Item]) -> List[nodes.Item]:
 def fix_cache_order(
     item: nodes.Item,
     argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[FixtureArgKey, None]]],
-    items_by_argkey: Dict[Scope, Dict[FixtureArgKey, "Deque[nodes.Item]"]],
+    items_by_argkey: Dict[Scope, Dict[FixtureArgKey, OrderedDict[nodes.Item, None]]],
 ) -> None:
     for scope in HIGH_SCOPES:
+        scoped_items_by_argkey = items_by_argkey[scope]
         for key in argkeys_cache[scope].get(item, []):
-            items_by_argkey[scope][key].appendleft(item)
+            scoped_items_by_argkey[key][item] = None
+            scoped_items_by_argkey[key].move_to_end(item, last=False)
 
 
 def reorder_items_atscope(
     items: Dict[nodes.Item, None],
     argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[FixtureArgKey, None]]],
-    items_by_argkey: Dict[Scope, Dict[FixtureArgKey, "Deque[nodes.Item]"]],
+    items_by_argkey: Dict[Scope, Dict[FixtureArgKey, OrderedDict[nodes.Item, None]]],
     scope: Scope,
 ) -> Dict[nodes.Item, None]:
     if scope is Scope.Function or len(items) < 3:

src/_pytest/hookspec.py

@@ -310,7 +310,12 @@ def pytest_collection_finish(session: "Session") -> None:
 def pytest_ignore_collect(
     collection_path: Path, path: "LEGACY_PATH", config: "Config"
 ) -> Optional[bool]:
-    """Return True to prevent considering this path for collection.
+    """Return ``True`` to ignore this path for collection.
+
+    Return ``None`` to let other plugins ignore the path for collection.
+
+    Returning ``False`` will forcefully *not* ignore this path for collection,
+    without giving a chance for other plugins to ignore this path.
 
     This hook is consulted for all files and directories prior to calling
     more specific hooks.

src/_pytest/unittest.py

@@ -212,11 +212,12 @@ class TestCaseFunction(Function):
         super().setup()
 
     def teardown(self) -> None:
-        super().teardown()
         if self._explicit_tearDown is not None:
             self._explicit_tearDown()
             self._explicit_tearDown = None
         self._obj = None
+        self._instance = None
+        super().teardown()
 
     def startTest(self, testcase: "unittest.TestCase") -> None:
         pass

testing/python/fixtures.py

@@ -2219,6 +2219,25 @@ class TestAutouseManagement:
         reprec = pytester.inline_run("-s")
         reprec.assertoutcome(passed=2)
 
+    def test_reordering_catastrophic_performance(self, pytester: Pytester) -> None:
+        """Check that a certain high-scope parametrization pattern doesn't cause
+        a catasrophic slowdown.
+
+        Regression test for #12355.
+        """
+        pytester.makepyfile("""
+            import pytest
+
+            params = tuple("abcdefghijklmnopqrstuvwxyz")
+            @pytest.mark.parametrize(params, [range(len(params))] * 3, scope="module")
+            def test_parametrize(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z):
+                pass
+        """)
+
+        result = pytester.runpytest()
+
+        result.assert_outcomes(passed=3)
+
 
 class TestFixtureMarker:
     def test_parametrize(self, pytester: Pytester) -> None:

testing/test_unittest.py

@@ -1,5 +1,4 @@
 # mypy: allow-untyped-defs
-import gc
 import sys
 from typing import List
 
@@ -192,30 +191,35 @@ def test_teardown(pytester: Pytester) -> None:
 def test_teardown_issue1649(pytester: Pytester) -> None:
     """
     Are TestCase objects cleaned up? Often unittest TestCase objects set
-    attributes that are large and expensive during setUp.
+    attributes that are large and expensive during test run or setUp.
 
     The TestCase will not be cleaned up if the test fails, because it
     would then exist in the stackframe.
+
+    Regression test for #1649 (see also #12367).
     """
-    testpath = pytester.makepyfile(
+    pytester.makepyfile(
         """
         import unittest
-        class TestCaseObjectsShouldBeCleanedUp(unittest.TestCase):
-            def setUp(self):
-                self.an_expensive_object = 1
-            def test_demo(self):
-                pass
+        import gc
 
-    """
+        class TestCaseObjectsShouldBeCleanedUp(unittest.TestCase):
+            def test_expensive(self):
+                self.an_expensive_obj = object()
+
+            def test_is_it_still_alive(self):
+                gc.collect()
+                for obj in gc.get_objects():
+                    if type(obj).__name__ == "TestCaseObjectsShouldBeCleanedUp":
+                        assert not hasattr(obj, "an_expensive_obj")
+                        break
+                else:
+                    assert False, "Could not find TestCaseObjectsShouldBeCleanedUp instance"
+        """
     )
 
-    pytester.inline_run("-s", testpath)
-    gc.collect()
-
-    # Either already destroyed, or didn't run setUp.
-    for obj in gc.get_objects():
-        if type(obj).__name__ == "TestCaseObjectsShouldBeCleanedUp":
-            assert not hasattr(obj, "an_expensive_obj")
+    result = pytester.runpytest()
+    assert result.ret == ExitCode.OK
 
 
 def test_unittest_skip_issue148(pytester: Pytester) -> None: