Skip to content

Commit e0bc9cf

Browse files
authored
fix(backend): serve app search from the shared public-app cache, not a per-request full scan (#11851)
GET /v2/apps/search streamed `approved==True AND private==False` from Firestore on every request. In prod that is 3,247 documents / ~25MB per request. Prod evidence (api.omi.me load-balancer logs, 2026-08-18T12Z..2026-08-19T05Z, image c5b0b5d), broken down by query shape: q=<term>&limit=100 n=29 p50 13.44s p90 30.05s 24/29 over 5s installed_apps=true n=802 p50 0.15s p90 0.40s 43 over 5s my_apps=true n=12 p50 0.13s p90 0.36s 0 over 5s Only the branches that stream the whole public set are slow. `?q=` always does; `installed_apps=true` does only when the user has more than 30 enabled apps (Firestore's `in` limit), which is exactly its 43-request tail. 32 of these terminated as 504 at the 30s edge — the Apps tab search box returning nothing. The marketplace list path (utils.apps.get_approved_available_apps) already serves those same documents from a 10-minute Redis cache under `get_public_approved_apps_data`. Search was the one reader that bypassed it. search_apps_db now reads that shared key for both public-set branches and applies category/capability in Python (they were server-side filters on the read being replaced). my_apps and the `id in [...]` installed-apps fast path are unchanged and still query Firestore, because private/unapproved records are not in the public cache. The cache key moves to one shared constant so a reader cannot drift off the invalidation path. Failure-Class: FC-bounded-read-exceeds-request-budget
1 parent 02b6b1a commit e0bc9cf

4 files changed

Lines changed: 246 additions & 12 deletions

File tree

backend/database/apps.py

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,17 @@
66

77
from ulid import ULID
88

9-
from models.app import UsageHistoryType
9+
from models.app import App, UsageHistoryType
10+
from .redis_db import get_generic_cache, set_generic_cache
1011
from ._client import db
1112
import logging
1213

1314
logger = logging.getLogger(__name__)
1415

16+
# Shared with utils.apps (list + invalidation). Keep every reader and the invalidation path on this
17+
# one constant: a second literal is how a cache ends up populated but never cleared.
18+
PUBLIC_APPROVED_APPS_CACHE_KEY = 'get_public_approved_apps_data'
19+
1520
# BaseCompositeFilter expects Operator enum but accepts 'AND' string at runtime.
1621
# Typed as Any to satisfy pyright without importing StructuredQuery (which fails
1722
# on some google-cloud-firestore versions).
@@ -69,6 +74,20 @@ def get_public_approved_apps_db() -> List[Dict[str, Any]]:
6974
return [_typed_doc(doc) for doc in public_apps]
7075

7176

77+
def get_public_approved_apps_cached_db() -> List[Dict[str, Any]]:
78+
"""The approved+public app set, read through the marketplace's shared 10-minute Redis cache.
79+
80+
Same key, TTL, reduction and invalidation as `utils.apps.get_approved_available_apps`, so a
81+
reader here can never serve a staler view than the list the user just came from.
82+
"""
83+
cached = get_generic_cache(PUBLIC_APPROVED_APPS_CACHE_KEY)
84+
if cached:
85+
return cast(List[Dict[str, Any]], cached)
86+
reduced = [App.reduce_dict(app) for app in get_public_approved_apps_db()]
87+
set_generic_cache(PUBLIC_APPROVED_APPS_CACHE_KEY, reduced, 60 * 10) # 10 minutes cached
88+
return reduced
89+
90+
7291
def get_popular_apps_db() -> List[Dict[str, Any]]:
7392
filters = [FieldFilter('approved', '==', True), FieldFilter('is_popular', '==', True)]
7493
popular_apps = db.collection(apps_collection).where(filter=BaseCompositeFilter(_AND_OP, filters)).stream()
@@ -107,6 +126,10 @@ def search_apps_db(
107126
List of app dictionaries matching the filters
108127
"""
109128
filters: List[FieldFilter] = []
129+
# Whether the primary read is the whole approved+public app set. That set is 3k+ documents and
130+
# streaming it per request is what made `?q=` search a p50-13s / p90-30s endpoint in prod; the
131+
# marketplace list path already serves the same documents from Redis, so read through it here too.
132+
reads_public_set = False
110133

111134
# 1. Apply most restrictive filter first
112135
if my_apps:
@@ -120,16 +143,14 @@ def search_apps_db(
120143
if len(enabled_app_ids) > 30:
121144
# Firestore 'in' limited to 30 items
122145
# Query public approved apps first, then add user's own apps
123-
filters.append(FieldFilter('approved', '==', True))
124-
filters.append(FieldFilter('private', '==', False))
146+
reads_public_set = True
125147
else:
126148
# Query by specific IDs
127149
filters.append(FieldFilter('id', 'in', enabled_app_ids))
128150

129151
else:
130152
# Default: Public approved apps
131-
filters.append(FieldFilter('approved', '==', True))
132-
filters.append(FieldFilter('private', '==', False))
153+
reads_public_set = True
133154

134155
# 2. Add category filter
135156
if category and not my_apps: # Don't add if already filtering by my_apps
@@ -141,7 +162,15 @@ def search_apps_db(
141162

142163
# Execute query with all filters
143164
apps: List[Dict[str, Any]] = []
144-
if filters:
165+
if reads_public_set:
166+
apps = get_public_approved_apps_cached_db()
167+
# category/capability were server-side filters on the Firestore read this replaces; my_apps is
168+
# False on this branch, so both are unconditional here.
169+
if category:
170+
apps = [app for app in apps if app.get('category') == category]
171+
if capability:
172+
apps = [app for app in apps if capability in (app.get('capabilities') or [])]
173+
elif filters:
145174
query = db.collection(apps_collection).where(filter=BaseCompositeFilter(_AND_OP, filters))
146175
apps = [_typed_doc(doc) for doc in query.stream()]
147176

backend/tests/unit/test_app_visibility_missing_doc_guard.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,9 @@ def _mod(name, **attrs):
4949
_mod("google.cloud.firestore_v1.base_query", BaseCompositeFilter=MagicMock(), FieldFilter=MagicMock())
5050
_mod("google.cloud.firestore", ArrayUnion=MagicMock(), ArrayRemove=MagicMock())
5151
_mod("ulid", ULID=lambda: "01HZZTESTULID")
52-
_mod("models.app", UsageHistoryType=MagicMock())
52+
_mod("models.app", App=MagicMock(), UsageHistoryType=MagicMock())
5353
_mod("database._client", db=MagicMock())
54+
_mod("database.redis_db", get_generic_cache=MagicMock(), set_generic_cache=MagicMock())
5455

5556

5657
def _load():
Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
"""GET /v2/apps/search must read the approved+public app set through the shared Redis cache.
2+
3+
Prod signature (api.omi.me load-balancer logs, 2026-08-18/19, image c5b0b5d): `?q=<term>` searches
4+
ran p50 13.4s / p90 30.0s and 24 of 29 requests exceeded 5s, most terminating as 504 at the 30s
5+
edge — app search was effectively unusable in the mobile Apps tab. `installed_apps=true` requests
6+
served by the same branch (>30 enabled apps) carried the same tail; the `id in [...]` fast path
7+
sat at p50 0.15s.
8+
9+
Root cause: `search_apps_db` streamed `approved==True AND private==False` from Firestore on every
10+
request. That is 3,247 documents / ~25MB in prod. The marketplace list path
11+
(`utils.apps.get_approved_available_apps`) already serves exactly those documents from a 10-minute
12+
Redis cache under `get_public_approved_apps_data`; search was the one reader that bypassed it.
13+
14+
These tests drive the real route over HTTP with only the Firestore client and the Redis cache
15+
helpers stubbed, so the router, `search_apps_db` and the cache read all execute.
16+
"""
17+
18+
from __future__ import annotations
19+
20+
import os
21+
22+
os.environ.setdefault('OPENAI_API_KEY', 'sk-test-not-real')
23+
os.environ.setdefault('ENCRYPTION_SECRET', 'omi_ZwB2ZNqB2HHpMK6wStk7sTpavJiPTFg7gXUHnc4tFABPU6pZ2c2DKgehtfgi4RZv')
24+
25+
import pytest # noqa: E402
26+
from fastapi import FastAPI # noqa: E402
27+
from fastapi.testclient import TestClient # noqa: E402
28+
29+
from database import apps as apps_db # noqa: E402
30+
from routers import apps as apps_mod # noqa: E402
31+
32+
UID = 'uid-apps-search'
33+
34+
35+
def _app_doc(app_id: str, name: str, **overrides):
36+
doc = {
37+
'id': app_id,
38+
'name': name,
39+
'category': 'productivity',
40+
'author': 'Someone',
41+
'description': f'{name} description',
42+
'image': 'http://img',
43+
'capabilities': ['chat'],
44+
'approved': True,
45+
'private': False,
46+
'uid': 'author-uid',
47+
# Excluded by App.reduce_dict — present on the Firestore document, absent from the cache.
48+
'chat_prompt': 'x' * 128,
49+
'memory_prompt': 'y' * 128,
50+
}
51+
doc.update(overrides)
52+
return doc
53+
54+
55+
class _FakeDoc:
56+
def __init__(self, data):
57+
self._data = data
58+
59+
def to_dict(self):
60+
return dict(self._data)
61+
62+
63+
class _FakeQuery:
64+
def __init__(self, collection, docs):
65+
self._collection = collection
66+
self._docs = docs
67+
68+
def stream(self):
69+
self._collection.streams += 1
70+
return [_FakeDoc(d) for d in self._docs]
71+
72+
73+
class _FakeCollection:
74+
"""Streams every seeded document; the point of the assertions is *whether* it is streamed."""
75+
76+
def __init__(self, docs):
77+
self.docs = docs
78+
self.streams = 0
79+
80+
def where(self, filter=None): # noqa: A002 - matches the firestore kwarg
81+
return _FakeQuery(self, self.docs)
82+
83+
84+
class _FakeFirestore:
85+
def __init__(self, docs):
86+
self.collection_obj = _FakeCollection(docs)
87+
88+
def collection(self, _name):
89+
return self.collection_obj
90+
91+
92+
@pytest.fixture
93+
def env(monkeypatch):
94+
"""Real router + real search_apps_db; only Firestore and the Redis cache are stubbed."""
95+
docs = [_app_doc('a1', 'Todoist'), _app_doc('a2', 'Grok'), _app_doc('a3', 'Calendar Sync')]
96+
firestore = _FakeFirestore(docs)
97+
cache: dict[str, object] = {}
98+
99+
monkeypatch.setattr(apps_db, 'db', firestore)
100+
# raising=False so the control run against the pre-fix source still executes and fails on the
101+
# behavioural assertion (a Firestore stream per request) rather than erroring at setup.
102+
monkeypatch.setattr(apps_db, 'get_generic_cache', lambda key: cache.get(key), raising=False)
103+
monkeypatch.setattr(
104+
apps_db, 'set_generic_cache', lambda key, data, ttl=None: cache.__setitem__(key, data), raising=False
105+
)
106+
monkeypatch.setattr(apps_mod, 'get_enabled_apps', lambda uid: set())
107+
monkeypatch.setattr(apps_mod, 'get_apps_installs_count', lambda ids: {})
108+
monkeypatch.setattr(apps_mod, 'get_apps_reviews', lambda ids: {})
109+
110+
app = FastAPI()
111+
app.include_router(apps_mod.router)
112+
app.dependency_overrides[apps_mod.auth.get_current_user_uid] = lambda: UID
113+
114+
class Env:
115+
client = TestClient(app)
116+
117+
Env.firestore = firestore
118+
Env.cache = cache
119+
Env.docs = docs
120+
return Env
121+
122+
123+
def test_search_populates_the_shared_cache_on_a_cold_read(env):
124+
response = env.client.get('/v2/apps/search', params={'q': 'todoist', 'limit': 100})
125+
126+
assert response.status_code == 200
127+
assert [a['id'] for a in response.json()['data']] == ['a1']
128+
assert env.firestore.collection_obj.streams == 1
129+
130+
cached = env.cache['get_public_approved_apps_data']
131+
assert [a['id'] for a in cached] == ['a1', 'a2', 'a3']
132+
# Cached as reduced records, matching the marketplace list path.
133+
assert 'chat_prompt' not in cached[0]
134+
assert 'description' in cached[0]
135+
136+
137+
def test_warm_cache_serves_search_without_streaming_the_collection(env):
138+
assert env.client.get('/v2/apps/search', params={'q': 'grok', 'limit': 100}).status_code == 200
139+
env.firestore.collection_obj.streams = 0
140+
141+
response = env.client.get('/v2/apps/search', params={'q': 'grok', 'limit': 100})
142+
143+
assert response.status_code == 200
144+
assert [a['id'] for a in response.json()['data']] == ['a2']
145+
# The regression: this is the read that cost 13s+ per request in prod.
146+
assert env.firestore.collection_obj.streams == 0
147+
148+
149+
def test_default_browse_and_filters_are_served_from_the_cache(env):
150+
env.client.get('/v2/apps/search', params={'limit': 100})
151+
env.firestore.collection_obj.streams = 0
152+
153+
browse = env.client.get('/v2/apps/search', params={'limit': 100})
154+
assert [a['id'] for a in browse.json()['data']] == ['a3', 'a2', 'a1'] # name_asc default
155+
156+
matching = env.client.get('/v2/apps/search', params={'category': 'productivity', 'limit': 100})
157+
assert [a['id'] for a in matching.json()['data']] == ['a3', 'a2', 'a1']
158+
other = env.client.get('/v2/apps/search', params={'category': 'entertainment', 'limit': 100})
159+
assert other.json()['data'] == []
160+
161+
has_cap = env.client.get('/v2/apps/search', params={'capability': 'chat', 'limit': 100})
162+
assert len(has_cap.json()['data']) == 3
163+
no_cap = env.client.get('/v2/apps/search', params={'capability': 'persona', 'limit': 100})
164+
assert no_cap.json()['data'] == []
165+
166+
assert env.firestore.collection_obj.streams == 0
167+
168+
169+
def test_my_apps_still_reads_firestore(env):
170+
"""Private/unapproved records are not in the public cache — that branch must keep querying."""
171+
env.client.get('/v2/apps/search', params={'limit': 100}) # warm the cache
172+
env.firestore.collection_obj.streams = 0
173+
174+
response = env.client.get('/v2/apps/search', params={'my_apps': 'true', 'limit': 100})
175+
176+
assert response.status_code == 200
177+
assert env.firestore.collection_obj.streams == 1
178+
179+
180+
def test_installed_apps_under_the_in_limit_still_reads_firestore(env, monkeypatch):
181+
monkeypatch.setattr(apps_mod, 'get_enabled_apps', lambda uid: {'a1'})
182+
env.client.get('/v2/apps/search', params={'limit': 100}) # warm the cache
183+
env.firestore.collection_obj.streams = 0
184+
185+
response = env.client.get('/v2/apps/search', params={'installed_apps': 'true', 'limit': 100})
186+
187+
assert response.status_code == 200
188+
assert env.firestore.collection_obj.streams == 1
189+
190+
191+
def test_installed_apps_over_the_in_limit_uses_the_cache_for_the_public_set(env, monkeypatch):
192+
""">30 enabled ids: the public set comes from the cache, the user's own apps still from Firestore."""
193+
enabled = {'a1', 'a2'} | {f'x{i}' for i in range(30)}
194+
monkeypatch.setattr(apps_mod, 'get_enabled_apps', lambda uid: enabled)
195+
env.client.get('/v2/apps/search', params={'limit': 100}) # warm the cache
196+
env.firestore.collection_obj.streams = 0
197+
198+
response = env.client.get('/v2/apps/search', params={'installed_apps': 'true', 'limit': 100})
199+
200+
assert response.status_code == 200
201+
assert sorted(a['id'] for a in response.json()['data']) == ['a1', 'a2']
202+
# Exactly one stream: the uid-scoped query for the user's own private/unapproved apps.
203+
assert env.firestore.collection_obj.streams == 1

backend/utils/apps.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from database.cache import get_memory_cache, get_pubsub_manager
1414
from database.redis_db import delete_generic_cache
1515
from database.apps import (
16+
PUBLIC_APPROVED_APPS_CACHE_KEY,
1617
get_private_apps_db,
1718
get_public_unapproved_apps_db,
1819
get_public_approved_apps_db,
@@ -318,7 +319,7 @@ def fetch_and_process() -> List[App]:
318319

319320

320321
def get_available_apps(uid: str, include_reviews: bool = False) -> List[App]:
321-
cache_key = 'get_public_approved_apps_data'
322+
cache_key = PUBLIC_APPROVED_APPS_CACHE_KEY
322323
memory_cache = get_memory_cache()
323324

324325
# Cache tester flag per user (30s TTL) to avoid Firestore lookup every 1s (#5439 sub-task 3)
@@ -472,23 +473,23 @@ def invalidate_approved_apps_cache() -> None:
472473
pubsub_manager = get_pubsub_manager()
473474

474475
# Invalidate both cache key variants (with and without reviews)
475-
cache_keys = ['get_public_approved_apps_data:reviews=0', 'get_public_approved_apps_data:reviews=1']
476+
cache_keys = [f'{PUBLIC_APPROVED_APPS_CACHE_KEY}:reviews={n}' for n in (0, 1)]
476477

477478
# Clear local memory cache
478479
for key in cache_keys:
479480
memory_cache.delete(key)
480481

481482
# Clear Redis cache
482-
delete_generic_cache('get_public_approved_apps_data')
483+
delete_generic_cache(PUBLIC_APPROVED_APPS_CACHE_KEY)
483484

484485
# Notify all other instances to clear their memory cache
485486
pubsub_manager.publish_invalidation(cache_keys)
486487

487488

488489
def get_approved_available_apps(include_reviews: bool = False) -> list[App]:
489490
# Use separate cache keys for with/without reviews
490-
cache_key = f'get_public_approved_apps_data:reviews={int(include_reviews)}'
491-
redis_cache_key = 'get_public_approved_apps_data'
491+
cache_key = f'{PUBLIC_APPROVED_APPS_CACHE_KEY}:reviews={int(include_reviews)}'
492+
redis_cache_key = PUBLIC_APPROVED_APPS_CACHE_KEY
492493
memory_cache = get_memory_cache()
493494

494495
def fetch_and_process() -> List[App]:

0 commit comments

Comments
 (0)