feat(api): Add wiki-data endpoint for coverage help pages

cl/api/tests.py

@@ -68,7 +68,11 @@
     promo_doubling_applies,
     promo_switch_is_active,
 )
-from cl.api.views import build_chart_data, coverage_data, make_court_variable
+from cl.api.views import (
+    build_chart_data,
+    coverage_data,
+    make_court_variable,
+)
 from cl.api.webhooks import send_webhook_event
 from cl.audio.api_views import AudioViewSet
 from cl.audio.audio_sources import AudioSources
@@ -178,14 +182,9 @@
 class BasicAPIPageTest(ESIndexTestCase, TestCase):
     """Test the basic views"""
 
-    fixtures = [
-        "judge_judy.json",
-        "test_court.json",
-        "test_objects_search.json",
-    ]
-
     @classmethod
     def setUpTestData(cls):
+        CourtFactory(id="ca1", jurisdiction=Court.FEDERAL_APPELLATE)
         cls.rebuild_index("search.OpinionCluster")
 
     def setUp(self) -> None:
@@ -264,6 +263,31 @@ async def test_wiki_data_endpoint(self) -> None:
         self.assertIsInstance(data["feeds"]["opinion_courts"], str)
         self.assertIsInstance(data["podcasts"]["oral_argument_courts"], str)
 
+    async def test_wiki_coverage_data_endpoint(self) -> None:
+        """Does the coverage data endpoint return the expected JSON structure?"""
+        await caches["default"].adelete("wiki-coverage-data")
+        r = await self.async_client.get(reverse("wiki_coverage_data"))
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r["Content-Type"], "application/json")
+        data = json.loads(r.content)
+        expected_keys = {"judges", "oral_arguments", "financial_disclosures"}
+        self.assertEqual(set(data.keys()), expected_keys)
+        self.assertIsInstance(data["judges"]["count"], int)
+        self.assertIn("duration_minutes", data["oral_arguments"])
+        financial_disclosures = data["financial_disclosures"]
+        for key in (
+            "disclosures",
+            "investments",
+            "positions",
+            "agreements",
+            "non_investment_income",
+            "spousal_income",
+            "reimbursements",
+            "gifts",
+            "debts",
+        ):
+            self.assertIsInstance(financial_disclosures[key], int)
+
 
 @override_settings(
     CACHES={
@@ -379,36 +403,90 @@ async def test_court_link_list_markdown(self) -> None:
         )
 
     async def test_bust_cache_param(self) -> None:
-        """Does ?bust_cache rebuild the cached response for staff only?"""
-        sentinel = {"sentinel": True}
-        await caches["default"].aset("wiki-data", sentinel)
+        """Does ?bust_cache rebuild the cached response for staff only?
 
-        # Without the param, the cached payload is served.
-        r = await self.async_client.get(reverse("wiki_data"))
-        self.assertEqual(json.loads(r.content), sentinel)
-
-        # Anonymous and non-staff users can't bust the cache.
-        r = await self.async_client.get(
-            reverse("wiki_data"), {"bust_cache": ""}
-        )
-        self.assertEqual(json.loads(r.content), sentinel)
+        Both wiki-data endpoints share the same get_or_build_wiki_json()
+        caching logic, so one parametrized test covers both instead of
+        duplicating it per endpoint.
+        """
         non_staff = await sync_to_async(UserFactory)(is_staff=False)
-        await self.async_client.aforce_login(non_staff)
-        r = await self.async_client.get(
-            reverse("wiki_data"), {"bust_cache": ""}
-        )
-        self.assertEqual(json.loads(r.content), sentinel)
+        staff = await sync_to_async(UserFactory)(is_staff=True)
+        cases = [
+            ("wiki_data", "wiki-data", "rss_feeds"),
+            (
+                "wiki_coverage_data",
+                "wiki-coverage-data",
+                "financial_disclosures",
+            ),
+        ]
+        for url_name, cache_key, marker_key in cases:
+            with self.subTest(url_name=url_name):
+                # A fresh, logged-out client per case, so the previous
+                # case's aforce_login(staff) can't leak into this one's
+                # anonymous/non-staff assertions.
+                self.async_client = AsyncClient()
+                sentinel = {"sentinel": True}
+                await caches["default"].aset(cache_key, sentinel)
+
+                # Without the param, the cached payload is served.
+                r = await self.async_client.get(reverse(url_name))
+                self.assertEqual(json.loads(r.content), sentinel)
+
+                # Anonymous and non-staff users can't bust the cache.
+                r = await self.async_client.get(
+                    reverse(url_name), {"bust_cache": ""}
+                )
+                self.assertEqual(json.loads(r.content), sentinel)
+                await self.async_client.aforce_login(non_staff)
+                r = await self.async_client.get(
+                    reverse(url_name), {"bust_cache": ""}
+                )
+                self.assertEqual(json.loads(r.content), sentinel)
+
+                # Staff can: the response is rebuilt and re-cached.
+                await self.async_client.aforce_login(staff)
+                r = await self.async_client.get(
+                    reverse(url_name), {"bust_cache": ""}
+                )
+                data = json.loads(r.content)
+                self.assertIn(marker_key, data)
+                cached = await caches["default"].aget(cache_key)
+                self.assertIn(marker_key, cached)
+
+    async def test_bust_cache_refreshes_nested_fd_cache(self) -> None:
+        """Does ?bust_cache also refresh get_coverage_data_fds()'s own,
+        separately-cached financial disclosure counts?
+
+        get_coverage_data_fds() caches its counts under "coverage-data.fd3"
+        for a week, independent of the wiki endpoints' own caches. Busting
+        wiki_data/wiki_coverage_data's cache must bust that nested cache
+        too, or staff see up to a week-old FD counts despite ?bust_cache.
+        """
+        stale_fd_data = {
+            "disclosures": -1,
+            "investments": -1,
+            "positions": -1,
+            "agreements": -1,
+            "non_investment_income": -1,
+            "spousal_income": -1,
+            "reimbursements": -1,
+            "gifts": -1,
+            "debts": -1,
+            "private": False,
+        }
+        await caches["default"].aset("coverage-data.fd3", stale_fd_data)
+        await caches["default"].adelete("wiki-coverage-data")
 
-        # Staff can: the response is rebuilt and re-cached.
         staff = await sync_to_async(UserFactory)(is_staff=True)
         await self.async_client.aforce_login(staff)
         r = await self.async_client.get(
-            reverse("wiki_data"), {"bust_cache": ""}
+            reverse("wiki_coverage_data"), {"bust_cache": ""}
         )
-        data = json.loads(r.content)
-        self.assertIn("rss_feeds", data)
-        cached = await caches["default"].aget("wiki-data")
-        self.assertIn("rss_feeds", cached)
+        data = json.loads(r.content)["financial_disclosures"]
+        self.assertNotEqual(data["disclosures"], -1)
+
+        cached_fd_data = await caches["default"].aget("coverage-data.fd3")
+        self.assertNotEqual(cached_fd_data["disclosures"], -1)
 
 
 class CoverageTests(ESIndexTestCase, TestCase):

cl/api/urls.py

@@ -219,6 +219,11 @@
     path("help/api/jurisdictions/", views.court_index, name="court_index"),
     # Live API endpoints
     path("api/rest/v4/wiki-data/", views.wiki_data, name="wiki_data"),
+    path(
+        "api/rest/v4/wiki-data/coverage/",
+        views.wiki_coverage_data,
+        name="wiki_coverage_data",
+    ),
     re_path(
         r"^api/rest/v4/coverage/opinions/",
         views.coverage_data_opinions,

cl/api/views.py

@@ -1,5 +1,6 @@
 import logging
 import re
+from collections.abc import Awaitable, Callable
 from datetime import date, datetime, timedelta
 from http import HTTPStatus
 from typing import TypedDict, cast
@@ -8,7 +9,7 @@
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.cache import cache
-from django.db.models import QuerySet
+from django.db.models import QuerySet, Sum
 from django.http import HttpRequest, HttpResponse, JsonResponse
 from django.shortcuts import aget_object_or_404  # type: ignore[attr-defined]
 from django.template.response import TemplateResponse
@@ -25,6 +26,7 @@
     get_current_throttle_usage,
     get_user_api_usage,
 )
+from cl.audio.models import Audio
 from cl.custom_filters.templatetags.partition_util import columns
 from cl.donate.models import NeonMembership, NeonMembershipLevel
 from cl.favorites.models import Prayer
@@ -34,6 +36,7 @@
     get_opinions_coverage_over_time,
 )
 from cl.lib.url_utils import BASE_URL
+from cl.people_db.models import Person
 from cl.search.documents import (
     OpinionClusterDocument,
 )
@@ -315,17 +318,30 @@ async def make_court_link_list(courts: QuerySet, url_name: str) -> str:
     return "\n".join(lines)
 
 
-async def wiki_data(request: HttpRequest) -> JsonResponse:
-    """Provide data for the external wiki's help pages.
+async def get_or_build_wiki_json(
+    request: HttpRequest,
+    cache_key: str,
+    build_data: Callable[[bool], Awaitable[dict]],
+) -> JsonResponse:
+    """Serve a cached JSON payload for the wiki, rebuilding it on request.
 
-    Returns counts and settings used across several API documentation pages
-    so the wiki can display them via external data connectors.
+    Shared by the wiki-data endpoints so each one only has to describe how
+    to build its own payload, not how to cache it.
 
     Staff users can pass ?bust_cache to skip the cached response and rebuild
-    it, e.g. after court metadata changes. The rebuild is expensive, so the
-    param is ignored for everybody else.
+    it, e.g. after the underlying data changes. The rebuild is expensive, so
+    the param is ignored for everybody else. The flag is also passed to
+    build_data() so it can bust any caches of its own nested in the data it
+    fetches — otherwise a "fresh" rebuild here could still return
+    data that's stale by as much as those inner caches' own TTLs.
+
+    :param request: The request. Only used to check for ?bust_cache + staff.
+    :param cache_key: The cache key this payload is stored under.
+    :param build_data: An async callable that computes a fresh payload. It
+        receives the bust_cache flag so it can propagate it to any caches
+        of its own.
+    :return: The cached or freshly-built payload as a JsonResponse.
     """
-    cache_key = "wiki-data"
     bust_cache = (
         "bust_cache" in request.GET and (await request.auser()).is_staff  # type: ignore[attr-defined]
     )
@@ -334,6 +350,36 @@ async def wiki_data(request: HttpRequest) -> JsonResponse:
         if data is not None:
             return JsonResponse(data)
 
+    data = await build_data(bust_cache)
+    one_day = 60 * 60 * 24
+    await cache.aset(cache_key, data, one_day)
+    return JsonResponse(data)
+
+
+async def wiki_data(request: HttpRequest) -> JsonResponse:
+    """Provide data for the external wiki's help pages.
+
+    Returns counts and settings used across several API documentation pages
+    so the wiki can display them via external data connectors.
+
+    Staff users can pass ?bust_cache to skip the cached response and rebuild
+    it, e.g. after court metadata changes. The rebuild is expensive, so the
+    param is ignored for everybody else.
+    """
+    return await get_or_build_wiki_json(request, "wiki-data", build_wiki_data)
+
+
+async def build_wiki_data(bust_cache: bool = False) -> dict:
+    """Build the payload served by wiki_data().
+
+    Kept separate from the view so get_or_build_wiki_json() can call it only
+    when the cached payload is missing or busted.
+
+    :param bust_cache: Passed through to nested caches (e.g. financial
+        disclosure counts) so busting this endpoint's cache doesn't leave
+        stale data behind in those.
+    :return: The wiki-data payload.
+    """
     court_count = await Court.objects.exclude(
         jurisdiction=Court.TESTING_COURT
     ).acount()
@@ -342,7 +388,7 @@ async def wiki_data(request: HttpRequest) -> JsonResponse:
     rate = settings.REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"]["citations"]  # type: ignore[misc]
     count, period = parse_throttle_rate_for_template(rate)  # type: ignore[misc]
 
-    fd_data = await get_coverage_data_fds()
+    fd_data = await get_coverage_data_fds(bust_cache=bust_cache)
     # Yesterday's alert total; start=1 skips today's still-filling bucket.
     alerts_sent_count = await sync_to_async(get_redis_stat_sum)(
         f"{StatMetric.ALERTS_SENT}.{{date}}", days=1, start=1
@@ -423,9 +469,65 @@ async def wiki_data(request: HttpRequest) -> JsonResponse:
             ),
         },
     }
-    one_day = 60 * 60 * 24
-    await cache.aset(cache_key, data, one_day)
-    return JsonResponse(data)
+    return data
+
+
+async def wiki_coverage_data(request: HttpRequest) -> JsonResponse:
+    """Provide data for the external wiki's coverage help pages.
+
+    Returns counts used across the coverage help pages so the wiki can
+    display them via external data connectors. This is kept separate from
+    wiki_data() so that endpoint doesn't keep growing without bound — new
+    coverage stats belong here instead.
+
+    Staff users can pass ?bust_cache to skip the cached response and rebuild
+    it, e.g. after new financial disclosures land. The rebuild is expensive,
+    so the param is ignored for everybody else.
+    """
+    return await get_or_build_wiki_json(
+        request, "wiki-coverage-data", build_wiki_coverage_data
+    )
+
+
+async def build_wiki_coverage_data(bust_cache: bool = False) -> dict:
+    """Build the payload served by wiki_coverage_data().
+
+    Kept separate from the view so get_or_build_wiki_json() can call it only
+    when the cached payload is missing or busted.
+
+    :param bust_cache: Passed through to get_coverage_data_fds() so busting
+        this endpoint's cache actually refreshes the financial disclosure
+        counts too, instead of leaving up to a week-old counts from that
+        function's own cache in place.
+    :return: The wiki-coverage-data payload.
+    """
+    fd_data = await get_coverage_data_fds(bust_cache=bust_cache)
+    judge_count = await Person.objects.all().acount()
+
+    oa_aggregate = await Audio.objects.aaggregate(Sum("duration"))
+    oa_duration = oa_aggregate["duration__sum"]
+    if oa_duration:
+        oa_duration /= 60  # Avoids a "unsupported operand type" error
+
+    return {
+        "judges": {
+            "count": judge_count,
+        },
+        "oral_arguments": {
+            "duration_minutes": oa_duration,
+        },
+        "financial_disclosures": {
+            "disclosures": fd_data["disclosures"],
+            "investments": fd_data["investments"],
+            "positions": fd_data["positions"],
+            "agreements": fd_data["agreements"],
+            "non_investment_income": fd_data["non_investment_income"],
+            "spousal_income": fd_data["spousal_income"],
+            "reimbursements": fd_data["reimbursements"],
+            "gifts": fd_data["gifts"],
+            "debts": fd_data["debts"],
+        },
+    }
 
 
 class MembershipInfo(TypedDict):

cl/simple_pages/views.py

@@ -73,15 +73,18 @@ async def build_court_dicts(courts: QuerySet) -> list[dict[str, str]]:
     return court_dicts
 
 
-async def get_coverage_data_fds() -> dict[str, int]:
+async def get_coverage_data_fds(bust_cache: bool = False) -> dict[str, int]:
     """Get stats on the disclosure data
 
     Attempt the cache if possible.
 
+    :param bust_cache: If True, skip the cache and recompute fresh counts,
+        e.g. when a caller's own cache was just busted and needs this data
+        to actually be current rather than up to a week stale.
     :return: A dict mapping item types to their counts.
     """
     coverage_key = "coverage-data.fd3"
-    coverage_data = await cache.aget(coverage_key)
+    coverage_data = None if bust_cache else await cache.aget(coverage_key)
     if coverage_data is None:
         coverage_data = {
             "disclosures": FinancialDisclosure,