diff --git a/hub/agents/email/python/gaia_agent_email/tools/read_tools.py b/hub/agents/email/python/gaia_agent_email/tools/read_tools.py index 3713db2fd..ce3251e45 100644 --- a/hub/agents/email/python/gaia_agent_email/tools/read_tools.py +++ b/hub/agents/email/python/gaia_agent_email/tools/read_tools.py @@ -172,6 +172,38 @@ def _format_message_for_llm( } +def _format_message_metadata_for_llm(msg: Dict[str, Any]) -> Dict[str, Any]: + """Reduce a metadata-format Gmail message (no body) to fields the LLM + can act on for a counting/listing question (#2763). + + Companion to ``_format_message_for_llm``: that one decodes and wraps a + body, which costs up to ``DEFAULT_BODY_LIMIT_CHARS`` per message and is + the entire payload cost for a question like "how many emails from X" + that never reads message content. This formatter never touches + ``payload.body``/``payload.parts`` — a ``format="metadata"`` fetch + doesn't populate them (see ``GmailBackend.get_message``'s docstring), + so there is nothing to decode. No per-message or envelope budget check + is needed here: at the tool's 100-message ceiling, a metadata row (a + handful of headers + a ~200-char snippet) stays orders of magnitude + below any device profile's context budget. + """ + payload = msg.get("payload") or {} + headers = { + (h.get("name") or "").lower(): h.get("value", "") + for h in payload.get("headers", []) + } + return { + "id": msg.get("id"), + "thread_id": msg.get("threadId"), + "subject": headers.get("subject", ""), + "from": headers.get("from", ""), + "to": headers.get("to", ""), + "date": headers.get("date", ""), + "label_ids": list(msg.get("labelIds", [])), + "snippet": msg.get("snippet", ""), + } + + # --------------------------------------------------------------------------- # Pure tool implementations (testable without the agent class) # --------------------------------------------------------------------------- @@ -850,11 +882,32 @@ def search_messages_impl( debug: bool = False, operator_retry: bool = True, budget_tokens: Optional[int] = None, + include_bodies: bool = False, ) -> Dict[str, Any]: + """``include_bodies`` defaults to ``False`` (#2763): metadata-only (no + body decode, no per-message/envelope budget check needed -- see + ``_format_message_metadata_for_llm``). Live-hardware evidence showed a + docstring-only opt-IN (default ``True``, model sets ``False`` for a + counting question) is not reliable enough: a 4B-class local model did + not choose it on the very probe this issue is about, reproducing the + original overflow byte-for-byte (measured ``n_prompt_tokens`` within 1% + of the pre-fix run). Defaulting to the cheap, safe path and requiring an + explicit ``include_bodies=True`` opt-in for the expensive one means the + fix does not depend on the model reliably choosing a new parameter on + the failure path that actually destroys the conversation -- the + asymmetry matters: a content question that forgets to opt in gets a + recoverable "no body available" rather than a context-ending overflow. + Full bodies via ``_format_messages_within_budget`` are still available + with ``include_bodies=True``. + """ query = normalize_gmail_date_operators(query) with log_tool_call( "search_messages", - {"query": query, "max_results": max_results}, + { + "query": query, + "max_results": max_results, + "include_bodies": include_bodies, + }, debug=debug, ) as st: listing = gmail.list_messages(query=query, max_results=max_results) @@ -871,13 +924,29 @@ def search_messages_impl( query=retried_query, max_results=max_results ) stubs = listing.get("messages", []) - full_msgs = [gmail.get_message(stub["id"]) for stub in stubs] - out = _format_messages_within_budget( - full_msgs, - tool_name="search_messages", - max_results=max_results, - budget_tokens=budget_tokens, - ) + if include_bodies: + full_msgs = [gmail.get_message(stub["id"]) for stub in stubs] + out = _format_messages_within_budget( + full_msgs, + tool_name="search_messages", + max_results=max_results, + budget_tokens=budget_tokens, + ) + else: + # Metadata-only: fetch in as few round-trips as the backend + # supports (batch when available), then re-walk ``stubs`` to + # preserve the backend's own ordering -- _fetch_messages returns + # an id-keyed dict, not a list (mirrors triage_inbox_impl's + # phase-1 pattern, read_tools.py:~1264). + stub_ids = [stub["id"] for stub in stubs] + metadata_by_id, _dropped_ids = _fetch_messages( + gmail, stub_ids, format="metadata" + ) + out = [ + _format_message_metadata_for_llm(metadata_by_id[sid]) + for sid in stub_ids + if sid in metadata_by_id + ] # Real cursor only -- never len(stubs) == max_results (see # _list_all_stubs's scan_truncated docstring above for why that # heuristic is wrong the moment a mailbox's true size matches the ask). @@ -1790,9 +1859,7 @@ def _remember(mid: Optional[str]) -> None: candidates.append({**item, "kind": kind}) _remember(item.get("message_id")) for item in actionable: - kind = ( - "meeting_request" if item.get("is_meeting_request") else "needs_response" - ) + kind = "meeting_request" if item.get("is_meeting_request") else "needs_response" candidates.append({**item, "kind": kind}) _remember(item.get("message_id")) for item in needs_review: @@ -2114,7 +2181,10 @@ def pre_scan_inbox_impl( def _drop_internal_date(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: # ``internal_date`` is a needs_you-only working field (#2743) — # never part of the public PreScanItem shape (extra="forbid"). - return [{k: v for k, v in item.items() if k != "internal_date"} for item in items] + return [ + {k: v for k, v in item.items() if k != "internal_date"} + for item in items + ] scanned = len(triage["results"]) inbox_counts = _fetch_inbox_counts(gmail) @@ -2133,7 +2203,9 @@ def _drop_internal_date(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: suggested_archives[: max(0, archive_cap)] ), "suggested_drafts": [], - "needs_review": _drop_internal_date(needs_review[: max(0, needs_review_cap)]), + "needs_review": _drop_internal_date( + needs_review[: max(0, needs_review_cap)] + ), "preferences_applied": { "priority_senders": sorted(prefs.get("priority_senders") or []), "low_priority_senders": sorted(prefs.get("low_priority_senders") or []), @@ -2597,7 +2669,9 @@ def summarize_thread(thread_id: str, mailbox: str = "") -> str: return _envelope_err(f"{type(exc).__name__}: {exc}") @tool - def search_messages(query: str, max_results: int = 25) -> str: + def search_messages( + query: str, max_results: int = 25, include_bodies: bool = False + ) -> str: """Search across ALL connected mailboxes. When multiple mailboxes are connected, searches both with a shared @@ -2625,13 +2699,25 @@ def search_messages(query: str, max_results: int = 25) -> str: query automatically, but forming the operator query yourself is more reliable. - A large ``max_results`` may shrink every hit's body TOGETHER (never - independently, never dropping a hit) so the whole result stays - within the model's context window — shrunk messages report - ``body_truncated: true``. If even the smallest usable body can't fit - every requested hit, the tool returns an actionable error instead of - silently returning fewer hits than asked for — retry with a smaller - ``max_results``. + By DEFAULT this returns METADATA ONLY — id/subject/from/to/date/ + label_ids/snippet, no body text — which is all a counting or + listing question needs ("how many emails from X", "list the + emails from Y this week", "do I have anything from Z"), at a + small fraction of the cost of a full search, so a large or + long-bodied result set never risks the model's context window. + Set ``include_bodies=True`` ONLY when the question needs what a + message actually SAYS — summarizing, quoting, or answering about + body content — since fetching bodies costs far more context and + can force the tool to shrink or refuse a large request. + + When ``include_bodies=True``, a large ``max_results`` may shrink + every hit's body TOGETHER (never independently, never dropping a + hit) so the whole result stays within the model's context window + — shrunk messages report ``body_truncated: true``. If even the + smallest usable body can't fit every requested hit, the tool + returns an actionable error instead of silently returning fewer + hits than asked for — retry with a smaller ``max_results`` or + drop back to the metadata-only default. Returns: JSON envelope with ``{"messages": [...]}`` plus ``count`` (the @@ -2641,7 +2727,11 @@ def search_messages(query: str, max_results: int = 25) -> str: ``max_results`` — say "at least N", never present N as the total). REPORT EVERY ENTRY in ``messages`` individually — do not summarize, merge, or quietly drop entries from a long - list. If ``operator_retry`` is present, the literal query you + list. With ``include_bodies=False`` each entry has no ``body`` + field at all — never claim to quote or summarize content from + a metadata-only result; re-call with ``include_bodies=True`` + (narrowing the query first) if content is actually needed. + If ``operator_retry`` is present, the literal query you passed found nothing and this is the broadened operator query that was retried instead — say the search was broadened before stating the count, since it may include hits (e.g. a @@ -2678,6 +2768,7 @@ def search_messages(query: str, max_results: int = 25) -> str: query=query, max_results=per_backend, debug=debug_flag, + include_bodies=include_bodies, ) except ConnectorsError as exc: msg = format_connector_error(exc) @@ -2769,7 +2860,8 @@ def triage_inbox(max_messages: int = DEFAULT_INBOX_SCAN_MESSAGES) -> str: """ try: max_messages = max( - 1, min(int(max_messages or DEFAULT_INBOX_SCAN_MESSAGES), scan_ceiling) + 1, + min(int(max_messages or DEFAULT_INBOX_SCAN_MESSAGES), scan_ceiling), ) # Phase 2 (#1603): scan every connected mailbox, tag each item @@ -2892,7 +2984,8 @@ def pre_scan_inbox( """ try: max_messages = max( - 1, min(int(max_messages or DEFAULT_INBOX_SCAN_MESSAGES), scan_ceiling) + 1, + min(int(max_messages or DEFAULT_INBOX_SCAN_MESSAGES), scan_ceiling), ) # Phase 2 (#1603): pre-scan every connected mailbox, tag each # section item with its source mailbox, split the budget, merge. diff --git a/hub/agents/email/python/tests/test_read_tools_list_inbox_budget_2514.py b/hub/agents/email/python/tests/test_read_tools_list_inbox_budget_2514.py index baea7f7b3..e9f11b45a 100644 --- a/hub/agents/email/python/tests/test_read_tools_list_inbox_budget_2514.py +++ b/hub/agents/email/python/tests/test_read_tools_list_inbox_budget_2514.py @@ -363,6 +363,8 @@ def test_all_messages_present_and_shrunk_to_a_shared_limit(self): max_results=n, budget_tokens=npu_budget, operator_retry=False, + include_bodies=True, # this test exercises the full-body shrink + # contract specifically; include_bodies now defaults to False (#2763) ) assert len(result["messages"]) == n @@ -391,6 +393,9 @@ def test_raises_when_even_the_floor_cannot_fit(self): max_results=n, budget_tokens=tiny_budget, operator_retry=False, + include_bodies=True, # exercises the full-body fail-loud path; + # include_bodies now defaults to False (#2763), which has no + # budget check to raise (metadata rows never need shrinking) ) message = str(exc_info.value) assert str(tiny_budget) in message diff --git a/hub/agents/email/python/tests/test_search_messages_metadata_only_2763.py b/hub/agents/email/python/tests/test_search_messages_metadata_only_2763.py new file mode 100644 index 000000000..d89a53516 --- /dev/null +++ b/hub/agents/email/python/tests/test_search_messages_metadata_only_2763.py @@ -0,0 +1,434 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""``search_messages(include_bodies=False)`` metadata-only path (#2763). + +Issue #2763 (P0): a counting/listing question against a long-bodied sender +("how many emails from X in the last two weeks?") returned NO answer at +all -- the search succeeded, but the model then blew the context window +retelling itself the full 4000-char-per-message envelope and emitted the +canned "I had to trim the conversation..." apology, deterministically, +every time. + +A counting/listing question needs zero body bytes. This file pins the +metadata-only contract added to ``search_messages``: + +- ``include_bodies=False`` fetches via the backend's existing + ``format="metadata"`` primitive (#2643) -- no body decode, no per-message + truncation -- and returns id/subject/from/to/date/label_ids/snippet ONLY. + No message in the result carries a ``body``, ``body_truncated``, + ``body_chars_dropped``, or ``attachments`` field. +- The resulting envelope is AT LEAST AN ORDER OF MAGNITUDE smaller than the + same query's full-body envelope (the acceptance criterion's own wording), + measured at the REGISTERED ``@tool`` layer -- not ``search_messages_impl`` + in isolation -- so a wrapper-introduced regression (e.g. a mailbox tag or + merge step re-adding bulk) would be caught. +- The envelope is asserted against the ACTUAL computed budget + (``envelope_budget_tokens``, imported -- never a hardcoded literal), not + merely "the call returned". +- ``include_bodies=True`` (the default, unchanged) still shrinks/behaves + exactly as before #2763 -- this file adds a new opt-in path, it does not + change the existing one. + +Hermetic: ``FakeGmailBackend`` only, no Lemonade, no network. Long bodies +mirror the failing probe's shape: 15 messages from one sender +(``from:Every newer_than:14d``, true count 15 in the real issue), each with +a raw body far longer than ``DEFAULT_BODY_LIMIT_CHARS``. +""" + +from __future__ import annotations + +import base64 +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, List, Tuple + +import pytest + +# --------------------------------------------------------------------------- +# Path / import bootstrap (mirrors test_read_tools_list_inbox_budget_2514.py) +# --------------------------------------------------------------------------- + +# parents[0] = tests/, [1] = python/, [2] = email/, [3] = agents/, [4] = hub/, +# [5] = repo-root +_REPO_ROOT = Path(__file__).resolve().parents[5] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +pytest.importorskip("gaia_agent_email") + +from gaia_agent_email.context_budget import ( # noqa: E402 + envelope_budget_tokens, + estimate_tokens_json, +) +from gaia_agent_email.tools.read_tools import ( # noqa: E402 + DEFAULT_BODY_LIMIT_CHARS, + ReadToolsMixin, + _format_message_metadata_for_llm, + search_messages_impl, +) + +from gaia.agents.base.tools import _TOOL_REGISTRY # noqa: E402 +from gaia.llm.lemonade_client import GPU_CTX_SIZE, NPU_CTX_SIZE # noqa: E402 +from tests.fixtures.email.fake_gmail import FakeGmailBackend # noqa: E402 + +# --------------------------------------------------------------------------- +# Fixture helpers (adapted from test_read_tools_list_inbox_budget_2514.py) +# --------------------------------------------------------------------------- + + +def _b64url(text: str) -> str: + return base64.urlsafe_b64encode(text.encode("utf-8")).decode("ascii").rstrip("=") + + +def _long_body_msg(msg_id: str, body_text: str, **overrides: Any) -> Dict[str, Any]: + msg: Dict[str, Any] = { + "id": msg_id, + "threadId": msg_id, + "labelIds": ["INBOX"], + "snippet": body_text[:200], + "internalDate": "1750000000000", + "payload": { + "mimeType": "text/plain", + "filename": "", + "headers": [ + {"name": "Subject", "value": "Every: newsletter issue"}, + {"name": "From", "value": "Every "}, + {"name": "To", "value": "user@example.com"}, + {"name": "Date", "value": "Mon, 1 Jan 2026 00:00:00 +0000"}, + ], + "body": { + "data": _b64url(body_text), + "size": len(body_text.encode("utf-8")), + }, + }, + "sizeEstimate": len(body_text), + } + msg.update(overrides) + return msg + + +def _build_long_body_sender_inbox( + n: int = 15, *, raw_body_chars: int = 12000 +) -> Tuple[FakeGmailBackend, List[Dict[str, Any]]]: + """N messages from one sender with bodies far longer than + ``DEFAULT_BODY_LIMIT_CHARS`` -- mirrors #2763's real repro shape + (``from:Every newer_than:14d``, true count 15). + """ + gmail = FakeGmailBackend(user_email="user@example.com") + base_date = 1_800_000_000_000 + msgs: List[Dict[str, Any]] = [] + for i in range(n): + body = "x" * raw_body_chars + msg = _long_body_msg( + f"every{i}", body, threadId=f"every{i}", internalDate=str(base_date - i) + ) + gmail.add_message(msg) + msgs.append(msg) + return gmail, msgs + + +# --------------------------------------------------------------------------- +# Minimal tool-hosting stand-in (established pattern -- copied from +# test_search_messages_count_2756.py / test_read_tools_list_inbox_budget_2514.py) +# --------------------------------------------------------------------------- + + +class _Host(ReadToolsMixin): + """Minimal stand-in for EmailTriageAgent's tool-hosting surface.""" + + def __init__(self, backend: FakeGmailBackend): + self._gmail = backend + self._backends = {"google": backend} + self._message_mailbox: Dict[str, str] = {} + self.config = SimpleNamespace(debug=False) + + def _remember_message_mailbox(self, message_id, provider): + if message_id: + self._message_mailbox[message_id] = provider + + def _backend_for_message(self, message_id, explicit_mailbox=None): + provider = explicit_mailbox or self._message_mailbox.get(message_id) + if provider is None: + if len(self._backends) == 1: + return next(iter(self._backends.values())) + raise ValueError("ambiguous mailbox in test stub") + backend = self._backends.get(provider) + if backend is None: + raise ValueError("mailbox not connected in test stub") + return backend + + +def _registered_search_messages(host: _Host): + _TOOL_REGISTRY.clear() + host._register_read_tools() + assert "search_messages" in _TOOL_REGISTRY + return _TOOL_REGISTRY["search_messages"]["function"] + + +def _call(search_messages, **kwargs) -> Dict[str, Any]: + payload = json.loads(search_messages(**kwargs)) + assert payload["ok"] is True, payload + return payload["data"] + + +# --------------------------------------------------------------------------- +# No body content at all (AC: "the tool payload must contain no message +# body content") +# --------------------------------------------------------------------------- + + +class TestMetadataOnlyCarriesNoBodyContent: + def test_no_message_has_a_body_field(self): + gmail, msgs = _build_long_body_sender_inbox(n=15) + host = _Host(gmail) + search_messages = _registered_search_messages(host) + + data = _call( + search_messages, + query="from:every", + max_results=25, + include_bodies=False, + ) + + assert len(data["messages"]) == 15 + for m in data["messages"]: + assert "body" not in m + assert "body_truncated" not in m + assert "body_chars_dropped" not in m + assert "attachments" not in m + # Still carries what a counting/listing answer needs. + assert m["subject"] + assert m["from"] + + def test_metadata_formatter_matches_registered_output(self): + """Wire-level parity: the registered tool's per-message shape must + equal ``_format_message_metadata_for_llm`` plus the wrapper's own + ``mailbox`` tag -- not a divergent ad hoc shape.""" + gmail, msgs = _build_long_body_sender_inbox(n=3) + host = _Host(gmail) + search_messages = _registered_search_messages(host) + + data = _call( + search_messages, query="from:every", max_results=25, include_bodies=False + ) + + by_id = {m["id"]: m for m in msgs} + for out_msg in data["messages"]: + # Re-fetch metadata-format directly to build the expected shape. + meta_msg = gmail.get_message(out_msg["id"], format="metadata") + expected = { + **_format_message_metadata_for_llm(meta_msg), + "mailbox": "google", + } + assert out_msg == expected + + +# --------------------------------------------------------------------------- +# Order-of-magnitude envelope reduction (AC: "the envelope size drops by at +# least an order of magnitude versus your measured value from step 1"), +# measured at the REGISTERED tool layer +# --------------------------------------------------------------------------- + + +class TestMetadataOnlyEnvelopeShrinksAnOrderOfMagnitude: + def test_registered_tool_envelope_is_at_least_10x_smaller(self): + n = 15 + gmail, msgs = _build_long_body_sender_inbox(n=n, raw_body_chars=12000) + host = _Host(gmail) + search_messages = _registered_search_messages(host) + + full_body_data = _call( + search_messages, query="from:every", max_results=25, include_bodies=True + ) + metadata_data = _call( + search_messages, query="from:every", max_results=25, include_bodies=False + ) + + assert len(full_body_data["messages"]) == n + assert len(metadata_data["messages"]) == n + + full_serialized = json.dumps(full_body_data["messages"], default=str) + metadata_serialized = json.dumps(metadata_data["messages"], default=str) + + full_tokens = estimate_tokens_json(full_serialized) + metadata_tokens = estimate_tokens_json(metadata_serialized) + + assert metadata_tokens > 0 + reduction_factor = full_tokens / metadata_tokens + assert reduction_factor >= 10, ( + f"metadata-only envelope must be at least an order of magnitude " + f"smaller than the full-body envelope for the identical query -- " + f"got {full_tokens} -> {metadata_tokens} tokens " + f"({reduction_factor:.1f}x)" + ) + + char_reduction_factor = len(full_serialized) / len(metadata_serialized) + assert char_reduction_factor >= 10 + + +# --------------------------------------------------------------------------- +# Envelope size asserted against the ACTUAL computed budget -- not merely +# "the call returned" (run-contract requirement) +# --------------------------------------------------------------------------- + + +class TestMetadataOnlyEnvelopeAgainstComputedBudget: + def test_metadata_envelope_fits_gpu_budget_with_wide_margin(self): + n = 15 + gmail, msgs = _build_long_body_sender_inbox(n=n, raw_body_chars=12000) + host = _Host(gmail) + search_messages = _registered_search_messages(host) + + data = _call( + search_messages, query="from:every", max_results=25, include_bodies=False + ) + serialized = json.dumps(data["messages"], default=str) + tokens = estimate_tokens_json(serialized) + + gpu_budget = envelope_budget_tokens(ctx_size=GPU_CTX_SIZE) + # Not just "fits" -- comfortably so: metadata rows are cheap enough + # that even 100 of them (the tool's max_results ceiling) must stay + # under 10% of the real device budget, or the metadata formatter has + # regressed toward carrying real content again. + assert tokens <= gpu_budget * 0.10, ( + f"metadata-only envelope ({tokens} tokens) should be a small " + f"fraction of the GPU budget ({gpu_budget} tokens) -- got " + f"{tokens / gpu_budget:.1%}" + ) + + def test_metadata_envelope_fits_npu_budget_with_wide_margin(self): + n = 15 + gmail, msgs = _build_long_body_sender_inbox(n=n, raw_body_chars=12000) + host = _Host(gmail) + search_messages = _registered_search_messages(host) + + data = _call( + search_messages, query="from:every", max_results=25, include_bodies=False + ) + serialized = json.dumps(data["messages"], default=str) + tokens = estimate_tokens_json(serialized) + + npu_budget = envelope_budget_tokens(ctx_size=NPU_CTX_SIZE) + # Empirically ~23% of the NPU budget for 15 rows -- assert well + # under half, not a hand-picked number tighter than reality. + assert tokens <= npu_budget * 0.5, ( + f"metadata-only envelope ({tokens} tokens) should stay well " + f"under half the smaller NPU budget ({npu_budget} tokens) -- " + f"got {tokens / npu_budget:.1%}" + ) + + def test_100_metadata_rows_still_fits_comfortably(self): + """The tool's own ceiling (``max_results`` clamped to 100) is the + worst case -- even at 100 long-bodied-sender hits, metadata-only + must still FIT, with room to spare, inside the GPU budget. (Still + dramatically cheaper than 100 full bodies would be -- see + ``TestMetadataOnlyEnvelopeShrinksAnOrderOfMagnitude`` for that + comparison; empirically ~62% of budget here, so "comfortable" means + clearly under it, not vanishingly small in absolute terms.) + """ + n = 100 + gmail, msgs = _build_long_body_sender_inbox(n=n, raw_body_chars=12000) + host = _Host(gmail) + search_messages = _registered_search_messages(host) + + data = _call( + search_messages, query="from:every", max_results=100, include_bodies=False + ) + assert len(data["messages"]) == n + serialized = json.dumps(data["messages"], default=str) + tokens = estimate_tokens_json(serialized) + gpu_budget = envelope_budget_tokens(ctx_size=GPU_CTX_SIZE) + assert tokens <= gpu_budget * 0.8, ( + f"100-row metadata-only envelope ({tokens} tokens) should still " + f"fit the GPU budget ({gpu_budget} tokens) with room to spare " + f"-- got {tokens / gpu_budget:.1%}" + ) + + +# --------------------------------------------------------------------------- +# include_bodies defaults to False -- live-hardware evidence (see the commit +# that made this the default) showed a docstring-only opt-in with +# include_bodies=True as the default was not reliable: a 4B-class local +# model did not choose include_bodies=False on the exact failing probe this +# issue is about, reproducing the original overflow. Defaulting to the +# cheap, safe path removes the dependency on the model choosing a new +# parameter correctly on the failure-prone case. +# --------------------------------------------------------------------------- + + +class TestIncludeBodiesDefaultsToFalse: + def test_omitting_include_bodies_matches_explicit_false(self): + gmail, msgs = _build_long_body_sender_inbox(n=5, raw_body_chars=500) + host = _Host(gmail) + search_messages = _registered_search_messages(host) + + default_result = search_messages(query="from:every", max_results=25) + explicit_false_result = search_messages( + query="from:every", max_results=25, include_bodies=False + ) + assert default_result == explicit_false_result + + data = json.loads(default_result)["data"] + assert len(data["messages"]) == 5 + for m in data["messages"]: + assert "body" not in m + + def test_small_inbox_full_body_still_carries_body_field(self): + gmail, msgs = _build_long_body_sender_inbox(n=3, raw_body_chars=500) + host = _Host(gmail) + search_messages = _registered_search_messages(host) + + data = _call( + search_messages, query="from:every", max_results=25, include_bodies=True + ) + assert len(data["messages"]) == 3 + for m in data["messages"]: + assert "body" in m + assert m["body_truncated"] is False # 500 chars < DEFAULT_BODY_LIMIT_CHARS + + +# --------------------------------------------------------------------------- +# search_messages_impl layer (pure function, no registered-tool wrapper) -- +# nice-to-have, mirrors TestSearchMessagesSharesTheEnvelopeBudgetContract's +# level of coverage in the sibling #2514 budget test file +# --------------------------------------------------------------------------- + + +class TestSearchMessagesImplMetadataOnly: + def test_impl_metadata_only_preserves_stub_order_and_count(self): + n = 6 + gmail, msgs = _build_long_body_sender_inbox(n=n, raw_body_chars=12000) + + result = search_messages_impl( + gmail, + query="from:every", + max_results=25, + operator_retry=False, + include_bodies=False, + ) + + assert len(result["messages"]) == n + # FakeGmailBackend.list_messages sorts newest-first by internalDate; + # _build_long_body_sender_inbox assigns descending internalDate as + # i increases, so ids must come back in exactly seeded (m0..m_{n-1}) + # order -- proves the id-keyed _fetch_messages dict lookup didn't + # silently reorder anything. + assert [m["id"] for m in result["messages"]] == [f"every{i}" for i in range(n)] + + def test_impl_metadata_only_respects_max_results_via_list_messages(self): + gmail, msgs = _build_long_body_sender_inbox(n=15, raw_body_chars=12000) + + result = search_messages_impl( + gmail, + query="from:every", + max_results=5, + operator_retry=False, + include_bodies=False, + ) + assert len(result["messages"]) == 5 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/scripts/email_context_budget_measurement_2763.py b/scripts/email_context_budget_measurement_2763.py new file mode 100644 index 000000000..22331760f --- /dev/null +++ b/scripts/email_context_budget_measurement_2763.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +""" +Manual diagnostic script for issue #2763 (email agent context-overflow bug). + +NOT a pytest test suite -- this constructs a real ``EmailTriageAgent`` (with a +minimal in-memory backend, no live Lemonade/Gmail needed) and a +``FakeGmailBackend`` seeded with long-body messages from one sender, then +measures the actual composed system_prompt, the actual OpenAI tool-calling +schema (``_openai_tools``), and the actual ``search_messages`` envelope -- +using this repo's own ``context_budget.py`` estimator functions, so the +printed numbers are directly comparable to what the production code itself +computes when it decides whether to shrink a tool result. + +Kept here (not thrown away after the investigation) because the same +question -- "does the fixed per-turn overhead / envelope size assumption +still match reality" -- will recur as the email agent's tool registry grows. +Re-run this after any change to the tool registry, the system prompt, or +``context_budget.py``'s constants. + +Usage: + python scripts/email_context_budget_measurement_2763.py + python scripts/email_context_budget_measurement_2763.py --dump-dir /tmp/payloads + # also writes the exact payload strings (system_prompt, the + # _openai_tools JSON, the search_messages envelope JSON) to + # , e.g. for a real-tokenizer cross-check: + # llama-tokenize -m -f /openai_tools.json --show-count + +Requires: gaia-agent-email installed editable (``uv pip install -e +hub/agents/email/python``) in the active environment. No live Lemonade or +Gmail connection needed -- this is fully hermetic. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict +from unittest.mock import MagicMock, patch + +# Add project root to path (mirrors scripts/jira_smoke.py's convention). +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(_PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(_PROJECT_ROOT)) + +from gaia_agent_email.context_budget import ( # noqa: E402 + envelope_budget_tokens, + estimate_tokens, + estimate_tokens_json, +) +from gaia_agent_email.tools.read_tools import ( # noqa: E402 + DEFAULT_BODY_LIMIT_CHARS, + search_messages_impl, +) + +from gaia.llm.lemonade_client import ( # noqa: E402 + GPU_CTX_SIZE, + NPU_CTX_SIZE, + is_tool_calling_model, +) +from tests.fixtures.email.fake_gmail import FakeGmailBackend # noqa: E402 + + +def _b64url(text: str) -> str: + return base64.urlsafe_b64encode(text.encode("utf-8")).decode("ascii").rstrip("=") + + +def _msg_with_body( + msg_id: str, body_text: str, subject: str, sender: str, **overrides: Any +) -> Dict[str, Any]: + msg: Dict[str, Any] = { + "id": msg_id, + "threadId": msg_id, + "labelIds": ["INBOX"], + "snippet": body_text[:200], + "internalDate": "1750000000000", + "payload": { + "mimeType": "text/plain", + "filename": "", + "headers": [ + {"name": "Subject", "value": subject}, + {"name": "From", "value": sender}, + {"name": "To", "value": "user@example.com"}, + {"name": "Date", "value": "Mon, 1 Jan 2026 00:00:00 +0000"}, + ], + "body": { + "data": _b64url(body_text), + "size": len(body_text.encode("utf-8")), + }, + }, + "sizeEstimate": len(body_text), + } + msg.update(overrides) + return msg + + +def build_long_body_sender_inbox( + n: int = 15, raw_body_chars: int = 12000 +) -> FakeGmailBackend: + """N messages from one sender, each with a body far longer than + DEFAULT_BODY_LIMIT_CHARS -- issue #2763's stated repro shape ("a sender + whose messages have long bodies"), modeled after the real probe + (``from:Every newer_than:14d``, true count 15). + """ + gmail = FakeGmailBackend(user_email="user@example.com") + base_date = 1_800_000_000_000 + paragraph = ( + "Every is a media company that publishes essays on technology, " + "startups, and the future of work. This edition covers several " + "topics in depth, with extended analysis and multiple sections. " + ) + for i in range(n): + body = (paragraph * (raw_body_chars // len(paragraph) + 1))[:raw_body_chars] + msg = _msg_with_body( + f"every{i}", + body, + subject=f"Every: Issue #{100 + i}", + sender="Every ", + threadId=f"every{i}", + internalDate=str(base_date - i), + ) + gmail.add_message(msg) + return gmail + + +def build_agent(model_id: str = "Gemma-4-E4B-it-GGUF"): + """Instantiate a real EmailTriageAgent with a minimal in-memory backend + (no live Lemonade/Gmail) and return it, so ``agent.system_prompt`` and + ``agent._openai_tools`` reflect the REAL, currently-registered tool set. + """ + from gaia_agent_email.agent import EmailTriageAgent + from gaia_agent_email.config import EmailAgentConfig + + class _MinimalMailBackend: + pass + + class _MinimalCalendarBackend: + pass + + tmp = tempfile.mkdtemp() + cfg = EmailAgentConfig( + gmail_backend=_MinimalMailBackend(), + calendar_backend=_MinimalCalendarBackend(), + db_path=str(Path(tmp) / "state.db"), + memory_db_path=str(Path(tmp) / "memory.db"), + silent_mode=True, + debug=False, + model_id=model_id, + ) + with patch("gaia.agents.base.agent.AgentSDK") as mock_sdk: + mock_sdk.return_value = MagicMock() + return EmailTriageAgent(config=cfg) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dump-dir", + type=Path, + default=None, + help="Write the exact payload strings here for a real-tokenizer cross-check " + "(e.g. llama-tokenize against the actual GGUF).", + ) + args = parser.parse_args() + + print("=" * 78) + print("GPU profile (Gemma-4-E4B-it-GGUF, tool_calling=True)") + print("=" * 78) + gpu_agent = build_agent("Gemma-4-E4B-it-GGUF") + gpu_system_prompt = gpu_agent.system_prompt + print( + f"model_id: {gpu_agent.model_id} tool_calling: {is_tool_calling_model(gpu_agent.model_id)}" + ) + print( + f"system_prompt: {len(gpu_system_prompt)} chars, {estimate_tokens(gpu_system_prompt)} est tokens" + ) + openai_tools = gpu_agent._openai_tools + openai_tools_json = json.dumps(openai_tools, default=str) if openai_tools else "" + if openai_tools: + print( + f"_openai_tools: {len(openai_tools)} tool schemas, " + f"{len(openai_tools_json)} chars, " + f"{estimate_tokens_json(openai_tools_json)} est tokens" + ) + else: + print("_openai_tools: None (no separate tools= payload)") + + gpu_budget = envelope_budget_tokens(ctx_size=GPU_CTX_SIZE) + npu_budget = envelope_budget_tokens(ctx_size=NPU_CTX_SIZE) + print() + print(f"envelope_budget_tokens(GPU, ctx={GPU_CTX_SIZE}) = {gpu_budget}") + print(f"envelope_budget_tokens(NPU, ctx={NPU_CTX_SIZE}) = {npu_budget}") + + print() + print("=" * 78) + print("NPU profile (gemma4-it-e2b-FLM, tool_calling=False -- embedded-JSON path)") + print("=" * 78) + npu_agent = build_agent("gemma4-it-e2b-FLM") + npu_system_prompt = npu_agent.system_prompt + print( + f"model_id: {npu_agent.model_id} tool_calling: {is_tool_calling_model(npu_agent.model_id)}" + ) + print( + f"system_prompt: {len(npu_system_prompt)} chars, {estimate_tokens(npu_system_prompt)} est tokens" + ) + print(f"_openai_tools: {npu_agent._openai_tools!r}") + + print() + print("=" * 78) + print("search_messages_impl on 15 long-body messages from one sender") + print("(mirrors the failing probe: from:Every newer_than:14d, true count 15)") + print("=" * 78) + for max_results in (25, 50, 100): + gmail = build_long_body_sender_inbox(n=15, raw_body_chars=12000) + result = search_messages_impl( + gmail, + query="from:every", + max_results=max_results, + debug=False, + operator_retry=False, + budget_tokens=None, # production default: active_profile_ctx_size() + include_bodies=True, # this script measures the full-body shrink + # contract specifically; include_bodies now defaults to False (#2763) + ) + messages = result["messages"] + serialized = json.dumps({"messages": messages}, default=str) + env_tokens = estimate_tokens_json(serialized) + dropped = sorted({m["body_chars_dropped"] for m in messages}) + default_cap_drop_only = 12000 - DEFAULT_BODY_LIMIT_CHARS + shrink_fired = any(d > default_cap_drop_only for d in dropped) + print( + f"max_results={max_results}: {len(messages)} messages, " + f"{len(serialized)} chars, {env_tokens} est tokens, " + f"shrink_fired={shrink_fired}, fits_gpu_budget={env_tokens <= gpu_budget}" + ) + + if args.dump_dir: + args.dump_dir.mkdir(parents=True, exist_ok=True) + gmail = build_long_body_sender_inbox(n=15, raw_body_chars=12000) + result = search_messages_impl( + gmail, + query="from:every", + max_results=100, + debug=False, + operator_retry=False, + budget_tokens=None, + include_bodies=True, # same reasoning as above -- this dump is for + # the full-body real-tokenizer cross-check, not the metadata path + ) + tool_result_json = json.dumps( + { + "ok": True, + "data": { + "messages": result["messages"], + "count": len(result["messages"]), + "truncated": False, + }, + }, + default=str, + ) + (args.dump_dir / "system_prompt.txt").write_text(gpu_system_prompt) + (args.dump_dir / "openai_tools.json").write_text(openai_tools_json) + (args.dump_dir / "tool_result.json").write_text(tool_result_json) + (args.dump_dir / "npu_system_prompt.txt").write_text(npu_system_prompt) + print() + print( + f"Wrote payload files to {args.dump_dir} for a real-tokenizer cross-check." + ) + + +if __name__ == "__main__": + main() diff --git a/src/gaia/agents/base/agent.py b/src/gaia/agents/base/agent.py index 39a6df0be..9c97d6fa2 100644 --- a/src/gaia/agents/base/agent.py +++ b/src/gaia/agents/base/agent.py @@ -182,6 +182,21 @@ class HardwareRequirement: _SD_CAPABILITY_TOOLS: Tuple[str, ...] = ("generate_image",) +# Final answer when a turn still overflows the model's context window after +# the one-shot shrink-and-retry (#2763) -- shared across every agent, so it +# names the constraint generically (no tool- or domain-specific vocabulary) +# rather than assuming a search/date-range shape. The prior copy ("re-ask in +# a fresh chat with just the essentials") never said WHAT was too big or HOW +# to shrink it, so every occurrence read identically regardless of cause -- +# this repo's fail-loud rule requires naming the constraint and a next step. +_CONTEXT_STILL_OVERFLOWING_MESSAGE = ( + "This request needs more than fits in the model's context window, even " + "after trimming older results. Try narrowing it — fewer results, a " + "shorter date range, or a more specific query — or start a fresh " + "conversation and ask again." +) + + # Tools that mutate external state (mark read, archive, star, …). A small # model that loses track of sequential state may re-issue an identical # mutation (same tool + same id). Unlike query dedup we key on the *args*, @@ -3751,12 +3766,11 @@ def _process_query_impl( } ) if is_ctx_overflow: - final_answer = ( - "I had to trim the conversation to fit my " - "memory but I'm still not making progress. " - "Could you re-ask in a fresh chat with just " - "the essentials?" - ) + # Name the actual constraint and a next step + # (#2763) -- "re-ask with just the essentials" told + # the user nothing about WHAT was too big or HOW to + # shrink it, so every retry looked identical. + final_answer = _CONTEXT_STILL_OVERFLOWING_MESSAGE else: final_answer = ( f"Sorry, I ran into a problem while processing your request. " @@ -3900,12 +3914,11 @@ def _process_query_impl( } ) if is_ctx_overflow: - final_answer = ( - "I had to trim the conversation to fit my " - "memory but I'm still not making progress. " - "Could you re-ask in a fresh chat with just " - "the essentials?" - ) + # Name the actual constraint and a next step + # (#2763) -- "re-ask with just the essentials" told + # the user nothing about WHAT was too big or HOW to + # shrink it, so every retry looked identical. + final_answer = _CONTEXT_STILL_OVERFLOWING_MESSAGE else: # If we have a typed Lemonade error in the # cause-chain (e.g. ``LemonadeUpstreamTimeoutError`` diff --git a/tests/unit/agents/test_parse_error_recovery.py b/tests/unit/agents/test_parse_error_recovery.py index 1bb3a7dfb..b8f6a0db9 100644 --- a/tests/unit/agents/test_parse_error_recovery.py +++ b/tests/unit/agents/test_parse_error_recovery.py @@ -19,7 +19,7 @@ import pytest -from gaia.agents.base.agent import Agent +from gaia.agents.base.agent import _CONTEXT_STILL_OVERFLOWING_MESSAGE, Agent class _DummyAgent(Agent): @@ -195,10 +195,10 @@ def _send(*_, **__): def test_flm_context_overflow_after_retry_gives_friendly_fallback(self, agent): """#2513 work item 3: once the FastFlowLM 400 is reachable, an - exhausted retry must render the SAME "I had to trim the - conversation" message the llama.cpp path already has -- not the - generic "Sorry, I ran into an unexpected problem" wrapper, and not - a leaked "Max length reached!" backend string. + exhausted retry must render the SAME actionable overflow message + the llama.cpp path already has (``_CONTEXT_STILL_OVERFLOWING_MESSAGE``, + #2763) -- not the generic "Sorry, I ran into an unexpected problem" + wrapper, and not a leaked "Max length reached!" backend string. """ agent.streaming = False agent._is_loaded_ctx_too_small = lambda: False @@ -221,7 +221,7 @@ def _send(*_, **__): # ``process_query`` returns ``{"status": ..., "result": , ...}`` text = result["result"] if isinstance(result, dict) else str(result) assert text, "expected the friendly trim-exhausted fallback text" - assert "I had to trim the conversation" in text + assert text == _CONTEXT_STILL_OVERFLOWING_MESSAGE assert "Max length reached" not in text assert "Sorry, I ran into" not in text @@ -327,6 +327,34 @@ def _send_stream(*_, **__): text = result["result"] if isinstance(result, dict) else str(result) assert text, "expected the retried streamed answer to reach the user" assert "Max length reached" not in text + + def test_context_overflow_streaming_after_retry_gives_actionable_fallback( + self, agent + ): + """#2763: the streaming path's exhausted-retry fallback must be the + SAME actionable ``_CONTEXT_STILL_OVERFLOWING_MESSAGE`` the + non-streaming path renders (``test_flm_context_overflow_after_retry_ + gives_friendly_fallback`` above) -- not a leaked exception string. + Previously untested: only the streaming SUCCESS-after-retry case + (the test above) had coverage; the streaming STILL-overflowing case + did not. + """ + agent.streaming = True + agent._is_loaded_ctx_too_small = lambda: False + call_count = {"n": 0} + + def _send_stream(*_, **__): + call_count["n"] += 1 + raise RuntimeError("exceeds the available context size") + + agent.chat.send_messages_stream = MagicMock(side_effect=_send_stream) + + result = agent.process_query("anything", max_steps=5) + + assert call_count["n"] == 2 # initial attempt + one trim-and-retry + text = result["result"] if isinstance(result, dict) else str(result) + assert text == _CONTEXT_STILL_OVERFLOWING_MESSAGE + assert "exceeds the available context size" not in text assert "Sorry, I ran into" not in text