diff --git a/changelog/7672.bugfix.rst b/changelog/7672.bugfix.rst
new file mode 100644
index 000000000..88608e161
--- /dev/null
+++ b/changelog/7672.bugfix.rst
@@ -0,0 +1 @@
+Fixed log-capturing level restored incorrectly if ``caplog.set_level`` is called more than once.
diff --git a/src/_pytest/logging.py b/src/_pytest/logging.py
index 0ee9457ea..6c422647d 100644
--- a/src/_pytest/logging.py
+++ b/src/_pytest/logging.py
@@ -437,7 +437,8 @@ class LogCaptureFixture:
         # save the original log-level to restore it during teardown
         self._initial_logger_levels.setdefault(logger, logger_obj.level)
         logger_obj.setLevel(level)
-        self._initial_handler_level = self.handler.level
+        if self._initial_handler_level is None:
+            self._initial_handler_level = self.handler.level
         self.handler.setLevel(level)
 
     @contextmanager
diff --git a/testing/logging/test_fixture.py b/testing/logging/test_fixture.py
index 6e5e9c2b4..f18d7087d 100644
--- a/testing/logging/test_fixture.py
+++ b/testing/logging/test_fixture.py
@@ -65,6 +65,7 @@ def test_change_level_undos_handler_level(testdir: Testdir) -> None:
 
         def test1(caplog):
             assert caplog.handler.level == 0
+            caplog.set_level(9999)
             caplog.set_level(41)
             assert caplog.handler.level == 41