update-plugin-list: use json, use http caching, use gh action caches

this change
* switches from html index usage to json index usage
* adds using requests-cache as http cache to trim down runtime
* adds gh action caches for pip and the new http cache

scripts/update-plugin-list.py

@@ -5,12 +5,19 @@ from textwrap import dedent
 from textwrap import indent
 
 import packaging.version
-import requests
+import platformdirs
 import tabulate
 import wcwidth
+from requests_cache import CachedResponse
+from requests_cache import CachedSession
+from requests_cache import OriginalResponse
+from requests_cache import SQLiteCache
 from tqdm import tqdm
 
+
 FILE_HEAD = r"""
+.. Note this file is autogenerated by scripts/update-plugin-list.py - usually weekly via github action
+
 .. _plugin-list:
 
 Plugin List
@@ -53,19 +60,47 @@ def escape_rst(text: str) -> str:
     return text
 
 
+def project_response_with_refresh(
+    session: CachedSession, name: str, last_serial: int
+) -> OriginalResponse | CachedResponse:
+    """Get a http cached pypi project
+
+    force refresh in case of last serial mismatch
+    """
+
+    response = session.get(f"https://pypi.org/pypi/{name}/json")
+    if int(response.headers.get("X-PyPI-Last-Serial", -1)) != last_serial:
+        response = session.get(f"https://pypi.org/pypi/{name}/json", refresh=True)
+    return response
+
+
+def get_session() -> CachedSession:
+    """Configures the requests-cache session"""
+    cache_path = platformdirs.user_cache_path("pytest-plugin-list")
+    cache_path.mkdir(exist_ok=True, parents=True)
+    cache_file = cache_path.joinpath("http_cache.sqlite3")
+    return CachedSession(backend=SQLiteCache(cache_file))
+
+
+def pytest_plugin_projects_from_pypi(session: CachedSession) -> dict[str, int]:
+    response = session.get(
+        "https://pypi.org/simple",
+        headers={"Accept": "application/vnd.pypi.simple.v1+json"},
+        refresh=True,
+    )
+    return {
+        name: p["_last-serial"]
+        for p in response.json()["projects"]
+        if (name := p["name"]).startswith("pytest-") or name in ADDITIONAL_PROJECTS
+    }
+
+
 def iter_plugins():
-    regex = r">([\d\w-]*)</a>"
-    response = requests.get("https://pypi.org/simple")
+    session = get_session()
+    name_2_serial = pytest_plugin_projects_from_pypi(session)
 
-    match_names = (match.groups()[0] for match in re.finditer(regex, response.text))
-    plugin_names = [
-        name
-        for name in match_names
-        if name.startswith("pytest-") or name in ADDITIONAL_PROJECTS
-    ]
-
-    for name in tqdm(plugin_names, smoothing=0):
-        response = requests.get(f"https://pypi.org/pypi/{name}/json")
+    for name, last_serial in tqdm(name_2_serial.items(), smoothing=0):
+        response = project_response_with_refresh(session, name, last_serial)
         if response.status_code == 404:
             # Some packages, like pytest-azurepipelines42, are included in https://pypi.org/simple
             # but return 404 on the JSON API. Skip.
@@ -136,7 +171,7 @@ def plugin_definitions(plugins):
 
 
 def main():
-    plugins = list(iter_plugins())
+    plugins = [*iter_plugins()]
 
     reference_dir = pathlib.Path("doc", "en", "reference")