From 45c9f53d00f1972cff29c26a1e3dd21e2ca55378 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Sat, 11 Jul 2026 15:38:21 +0700 Subject: [PATCH] feat(ai): production agent templates + golden-transcript eval harness Adds dogeapi.ai.templates - drop-in, MIT agents built on build_agent(), one level up from dogeapi.ai.examples: - support_triage: typed TriageResult with a deterministic severity floor (severity-4 always needs_human=True via output validator) and a fetch_recent_tickets tool wired through SupportContext deps. - rag_answerer: CitedAnswer over caller-retrieved chunks; an output validator rejects uncited answers or citations outside the retrieval set (ModelRetry), and forces insufficient_context=true on empty retrieval instead of improvised answers. - evals: golden-transcript harness (GoldenCase/EvalReport, JSON loader, run_golden_cases) so customized templates can be regression-checked; ships sample goldens for the triage agent. Follows the ai-module conventions: pydantic_ai stays a lazy import (the 'ai' extra is only needed when a build_* factory is called), respx-mocked offline tests, mypy --strict clean. Co-Authored-By: Claude Fable 5 --- README.md | 2 +- backend/src/dogeapi/ai/templates/__init__.py | 53 ++++ backend/src/dogeapi/ai/templates/evals.py | 178 +++++++++++ .../ai/templates/golden/support_triage.json | 27 ++ .../src/dogeapi/ai/templates/rag_answerer.py | 144 +++++++++ .../dogeapi/ai/templates/support_triage.py | 127 ++++++++ backend/tests/test_ai_templates.py | 298 ++++++++++++++++++ 7 files changed, 828 insertions(+), 1 deletion(-) create mode 100644 backend/src/dogeapi/ai/templates/__init__.py create mode 100644 backend/src/dogeapi/ai/templates/evals.py create mode 100644 backend/src/dogeapi/ai/templates/golden/support_triage.json create mode 100644 backend/src/dogeapi/ai/templates/rag_answerer.py create mode 100644 backend/src/dogeapi/ai/templates/support_triage.py create mode 100644 backend/tests/test_ai_templates.py diff --git a/README.md b/README.md index 51df748..98dcea3 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ - **JWT scoped to org** — switch orgs and the access token is re-issued with the new `org_id` and computed scopes. - **API keys per organization** with scoped permissions, validated via `X-API-Key`. - **Email invitations** — or token-link only when email delivery is disabled. -- Optional, all gated by `FEATURE_*` env vars: OAuth (Google/GitHub), magic-link, real email delivery (Resend/SMTP), Stripe billing, Pydantic AI + LLM Gateway (chat + demo agents), Logfire observability, audit log, rate limiting. +- Optional, all gated by `FEATURE_*` env vars: OAuth (Google/GitHub), magic-link, real email delivery (Resend/SMTP), Stripe billing, Pydantic AI + LLM Gateway (chat, demo agents + production agent templates with a golden-transcript eval harness), Logfire observability, audit log, rate limiting. ## Repository layout diff --git a/backend/src/dogeapi/ai/templates/__init__.py b/backend/src/dogeapi/ai/templates/__init__.py new file mode 100644 index 0000000..9fb6412 --- /dev/null +++ b/backend/src/dogeapi/ai/templates/__init__.py @@ -0,0 +1,53 @@ +"""Production agent templates for the LLM Gateway (FEATURE_AI_CHAT). + +Drop-in, MIT-licensed agents built on :func:`dogeapi.ai.agents.build_agent`. +Unlike :mod:`dogeapi.ai.examples` (minimal teaching demos), these are the +reliability layer: typed outputs, deterministic guardrail validators, and a +golden-transcript eval harness so you can customize a template and verify +nothing broke. + +- :mod:`dogeapi.ai.templates.support_triage` — classify inbound support + messages with a severity floor that always escalates to a human. +- :mod:`dogeapi.ai.templates.rag_answerer` — answer over retrieved chunks, + refusing to emit claims without a citation from the retrieval set. +- :mod:`dogeapi.ai.templates.evals` — run golden transcripts against any + agent in this package (or your own). + +Like the rest of the ``ai`` module, nothing here imports ``pydantic_ai`` at +module scope — the optional ``ai`` extra is only required once you call +a ``build_*`` factory. +""" + +from dogeapi.ai.templates.evals import ( + CaseResult, + EvalReport, + GoldenCase, + load_golden_cases, + run_golden_cases, +) +from dogeapi.ai.templates.rag_answerer import ( + CitedAnswer, + RagContext, + RetrievedChunk, + build_rag_answerer, +) +from dogeapi.ai.templates.support_triage import ( + SupportContext, + TriageResult, + build_support_triage_agent, +) + +__all__ = ( + "CaseResult", + "CitedAnswer", + "EvalReport", + "GoldenCase", + "RagContext", + "RetrievedChunk", + "SupportContext", + "TriageResult", + "build_rag_answerer", + "build_support_triage_agent", + "load_golden_cases", + "run_golden_cases", +) diff --git a/backend/src/dogeapi/ai/templates/evals.py b/backend/src/dogeapi/ai/templates/evals.py new file mode 100644 index 0000000..1f955be --- /dev/null +++ b/backend/src/dogeapi/ai/templates/evals.py @@ -0,0 +1,178 @@ +"""Golden-transcript eval harness. + +Runs a list of :class:`GoldenCase` prompts against any Pydantic AI agent +(the templates in this package, the examples, or your own) and reports +which cases still pass. The point is regression safety: customize a +template's prompt or tools, re-run the goldens, and know immediately +whether you broke behaviour you relied on. + +Two kinds of checks per case, both optional: + +- ``expected`` — exact-match assertions against fields of a structured + output (dotted paths supported, e.g. ``"category"`` or ``"user.email"``). +- ``contains`` — substrings that must appear in the rendered output + (useful for plain-text agents). + +Cases can live in code or in JSON files (see ``golden/support_triage.json`` +next to this module):: + + from dogeapi.ai.templates import build_support_triage_agent, load_golden_cases, run_golden_cases + from dogeapi.settings import get_settings + + agent = build_support_triage_agent(get_settings()) + cases = load_golden_cases("src/dogeapi/ai/templates/golden/support_triage.json") + report = await run_golden_cases(agent, cases, deps_factory=lambda case: SupportContext(user_id="eval")) + assert report.ok, report.summary() + +The harness never touches the network itself — whatever model the agent +is built against (live gateway, or a mock in tests) is what gets evaluated. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from pydantic import BaseModel, Field + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from pydantic_ai import Agent + + +class GoldenCase(BaseModel): + """One golden transcript: a prompt plus assertions about the output.""" + + name: str = Field(description="Unique, human-readable case id.") + prompt: str = Field(description="The user message sent to the agent.") + expected: dict[str, Any] = Field( + default_factory=dict, + description="Dotted output-field path -> exact expected value.", + ) + contains: list[str] = Field( + default_factory=list, + description="Substrings that must appear in the rendered output.", + ) + + +class CaseResult(BaseModel): + """Outcome of one golden case.""" + + name: str + passed: bool + failures: list[str] = Field(default_factory=list) + output: str = Field(default="", description="Rendered agent output, for debugging failures.") + + +class EvalReport(BaseModel): + """Aggregate outcome of a golden run.""" + + results: list[CaseResult] = Field(default_factory=list) + + @property + def passed(self) -> int: + return sum(1 for r in self.results if r.passed) + + @property + def failed(self) -> int: + return len(self.results) - self.passed + + @property + def ok(self) -> bool: + return self.failed == 0 + + def summary(self) -> str: + """One line per case, failures first — suitable for CI logs.""" + lines = [f"golden run: {self.passed}/{len(self.results)} passed"] + for result in sorted(self.results, key=lambda r: r.passed): + status = "PASS" if result.passed else "FAIL" + lines.append(f" [{status}] {result.name}") + lines.extend(f" - {failure}" for failure in result.failures) + return "\n".join(lines) + + +def load_golden_cases(path: str | Path) -> list[GoldenCase]: + """Load cases from a JSON file containing a list of GoldenCase objects.""" + raw = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ValueError(f"{path}: expected a top-level JSON list of golden cases.") + return [GoldenCase.model_validate(item) for item in raw] + + +def _resolve(output: Any, dotted_path: str) -> Any: + """Walk a dotted attribute path on the output (raises AttributeError if absent).""" + value = output + for part in dotted_path.split("."): + value = getattr(value, part) + return value + + +def _render(output: Any) -> str: + """Human/CI-readable form of an agent output.""" + if isinstance(output, BaseModel): + return output.model_dump_json() + return str(output) + + +def _check_case(case: GoldenCase, output: Any) -> list[str]: + """Return the list of assertion failures for one case (empty = pass).""" + failures: list[str] = [] + for path, expected in case.expected.items(): + try: + actual = _resolve(output, path) + except AttributeError: + failures.append(f"expected.{path}: field not present on output") + continue + if actual != expected: + failures.append(f"expected.{path}: wanted {expected!r}, got {actual!r}") + rendered = _render(output) + failures.extend(f"contains: {needle!r} not found in output" for needle in case.contains if needle not in rendered) + return failures + + +async def run_golden_cases( + agent: Agent[Any, Any], + cases: Sequence[GoldenCase], + *, + deps_factory: Callable[[GoldenCase], Any] | None = None, +) -> EvalReport: + """Run every case against the agent and collect a report. + + Args: + agent: Any Pydantic AI agent (typed output or plain text). + cases: The golden cases to run. + deps_factory: Builds the ``deps`` for each case when the agent + requires them (e.g. ``lambda case: SupportContext(user_id="eval")``). + + Agent errors don't abort the run; they fail the individual case so one + regression can't hide the others. + """ + results: list[CaseResult] = [] + for case in cases: + deps = deps_factory(case) if deps_factory is not None else None + try: + run_result = await agent.run(case.prompt, deps=deps) + except Exception as exc: # a crashed case is a failed case; keep running the rest + results.append(CaseResult(name=case.name, passed=False, failures=[f"agent raised {exc!r}"])) + continue + failures = _check_case(case, run_result.output) + results.append( + CaseResult( + name=case.name, + passed=not failures, + failures=failures, + output=_render(run_result.output), + ) + ) + return EvalReport(results=results) + + +__all__ = ( + "CaseResult", + "EvalReport", + "GoldenCase", + "load_golden_cases", + "run_golden_cases", +) diff --git a/backend/src/dogeapi/ai/templates/golden/support_triage.json b/backend/src/dogeapi/ai/templates/golden/support_triage.json new file mode 100644 index 0000000..cc31f36 --- /dev/null +++ b/backend/src/dogeapi/ai/templates/golden/support_triage.json @@ -0,0 +1,27 @@ +[ + { + "name": "double-charge-is-billing-and-escalates", + "prompt": "I was charged twice this month and I want my money back immediately.", + "expected": { + "category": "billing", + "needs_human": true + } + }, + { + "name": "how-do-i-invite-teammates-is-low-severity", + "prompt": "Hey, how do I invite my teammates to my workspace?", + "expected": { + "category": "account", + "severity": 1, + "needs_human": false + } + }, + { + "name": "data-loss-hits-severity-floor", + "prompt": "All of our organization's documents disappeared after the last sync. Nothing in trash either.", + "expected": { + "severity": 4, + "needs_human": true + } + } +] diff --git a/backend/src/dogeapi/ai/templates/rag_answerer.py b/backend/src/dogeapi/ai/templates/rag_answerer.py new file mode 100644 index 0000000..4c93ccd --- /dev/null +++ b/backend/src/dogeapi/ai/templates/rag_answerer.py @@ -0,0 +1,144 @@ +"""Citation-enforcing RAG answerer. + +Answers a question over a set of retrieved chunks and *refuses to emit +claims without a citation*: an output validator rejects any answer whose +``citations`` list is empty or references a chunk id that was not actually +retrieved, forcing the model to retry. When retrieval came back empty (or +irrelevant), the model must set ``insufficient_context=True`` instead of +improvising. + +Retrieval is deliberately out of scope — run your vector/keyword search +first, then hand the chunks in via :class:`RagContext`. That keeps the +template storage-agnostic (pgvector, Qdrant, a SQL ``LIKE`` — anything). + +Usage:: + + from dogeapi.ai.templates import RagContext, RetrievedChunk, build_rag_answerer + from dogeapi.settings import get_settings + + agent = build_rag_answerer(get_settings()) + result = await agent.run( + "How do refunds work?", + deps=RagContext(chunks=[RetrievedChunk(id="c1", source="docs/billing.md", text="...")]), + ) + result.output # CitedAnswer with citations drawn from the retrieved ids + +Note: this module deliberately avoids ``from __future__ import annotations`` +— see :mod:`dogeapi.ai.templates.support_triage` for why. +""" + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from pydantic import BaseModel, Field + +from dogeapi.ai.agents import build_agent + +if TYPE_CHECKING: + from pydantic_ai import Agent + + from dogeapi.settings import Settings + + +SYSTEM_PROMPT = ( + "You answer questions using ONLY the context chunks provided in this conversation. " + "Every factual claim must be supported by at least one chunk; list the supporting " + "chunk ids in citations. If the chunks do not contain the answer, set " + "insufficient_context=true, leave citations empty, and say briefly that the " + "available documentation does not cover the question. Never use outside knowledge." +) + + +class RetrievedChunk(BaseModel): + """One chunk returned by your retrieval step.""" + + id: str = Field(description="Stable identifier the model cites, e.g. a chunk PK.") + source: str = Field(description="Human-readable origin, e.g. 'docs/billing.md#refunds'.") + text: str = Field(description="The chunk content shown to the model.") + + +@dataclass +class RagContext: + """Per-request dependencies: the chunks your retriever found.""" + + chunks: Sequence[RetrievedChunk] = () + + +class CitedAnswer(BaseModel): + """An answer that is only trusted as far as its citations.""" + + answer: str = Field(description="The answer, grounded in the cited chunks.") + citations: list[str] = Field( + default_factory=list, + description="Ids of the retrieved chunks that support the answer.", + ) + insufficient_context: bool = Field( + default=False, + description="True when the retrieved chunks cannot answer the question.", + ) + + +def build_rag_answerer( + settings: "Settings", + *, + model: str | None = None, +) -> "Agent[RagContext, CitedAnswer]": + """Build the citation-enforcing RAG agent against the LLM Gateway. + + Raises :class:`dogeapi.ai.agents.GatewayNotConfiguredError` when + ``LLM_GATEWAY_API_KEY`` is not configured. + """ + from pydantic_ai import ModelRetry, RunContext + + agent: Agent[RagContext, CitedAnswer] = build_agent( + settings, + model=model, + system_prompt=SYSTEM_PROMPT, + deps_type=RagContext, + output_type=CitedAnswer, + retries=2, + ) + + @agent.system_prompt + async def inject_chunks(ctx: RunContext[RagContext]) -> str: + """Render the retrieved chunks into the prompt, ids first so they are easy to cite.""" + if not ctx.deps.chunks: + return "No context chunks were retrieved for this question." + rendered = "\n\n".join(f"[{chunk.id}] ({chunk.source})\n{chunk.text}" for chunk in ctx.deps.chunks) + return f"Context chunks:\n\n{rendered}" + + @agent.output_validator + async def require_citations(ctx: RunContext[RagContext], output: CitedAnswer) -> CitedAnswer: + """Reject uncited or mis-cited answers so they never reach the caller.""" + known = {chunk.id for chunk in ctx.deps.chunks} + + if not known and not output.insufficient_context: + raise ModelRetry("No chunks were retrieved: set insufficient_context=true instead of answering.") + + if output.insufficient_context: + if output.citations: + raise ModelRetry("insufficient_context=true must not carry citations.") + return output + + if not output.citations: + raise ModelRetry( + "Every answer must cite at least one retrieved chunk id, or set insufficient_context=true." + ) + + unknown = sorted(set(output.citations) - known) + if unknown: + raise ModelRetry(f"Unknown citation ids {unknown}; cite only retrieved chunk ids: {sorted(known)}.") + + return output + + return agent + + +__all__ = ( + "SYSTEM_PROMPT", + "CitedAnswer", + "RagContext", + "RetrievedChunk", + "build_rag_answerer", +) diff --git a/backend/src/dogeapi/ai/templates/support_triage.py b/backend/src/dogeapi/ai/templates/support_triage.py new file mode 100644 index 0000000..41623d7 --- /dev/null +++ b/backend/src/dogeapi/ai/templates/support_triage.py @@ -0,0 +1,127 @@ +"""Support-ticket triage agent. + +Classifies an inbound support message into a category + severity, drafts a +suggested reply, and decides whether a human must take over. Two guardrails +make it production-grade rather than a demo: + +- The system prompt tells the model to classify conservatively (pick the + higher severity when torn) and to escalate billing disputes, security and + legal topics. +- A deterministic output validator enforces a *severity floor*: severity-4 + results are always ``needs_human=True`` no matter what the model said. + +Usage:: + + from dogeapi.ai.templates import SupportContext, build_support_triage_agent + from dogeapi.settings import get_settings + + agent = build_support_triage_agent(get_settings()) + result = await agent.run( + "I was charged twice this month and need a refund NOW", + deps=SupportContext(user_id="u_123", recent_tickets=["Duplicate charge in May"]), + ) + result.output # TriageResult + +``SupportContext.recent_tickets`` is intentionally a plain sequence: wire it +to your real ticket store when you drop this in (see ``fetch_recent_tickets`` +below), and the agent will spot repeat issues. + +Note: this module deliberately avoids ``from __future__ import annotations`` +— tool signatures must hold real types at runtime for Pydantic AI to +build their schemas, while ``pydantic_ai`` itself stays a lazy import so the +optional ``ai`` extra is only needed once you call the factory. +""" + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +from pydantic import BaseModel, Field + +from dogeapi.ai.agents import build_agent + +if TYPE_CHECKING: + from pydantic_ai import Agent + + from dogeapi.settings import Settings + + +TriageCategory = Literal["billing", "bug", "feature_request", "account", "other"] + +SYSTEM_PROMPT = ( + "You triage inbound support messages for a multi-tenant SaaS product. " + "Classify conservatively: if unsure between two severities, pick the higher. " + "Severity scale: 1=question/info, 2=degraded experience, 3=feature broken, " + "4=data loss, security incident, or money at risk. " + "Set needs_human=true for anything touching billing disputes, security, or legal. " + "The suggested_reply must be polite, specific to the message, and never promise " + "refunds, credits, or timelines. " + "Call fetch_recent_tickets before deciding: repeat issues raise severity." +) + + +class TriageResult(BaseModel): + """Structured verdict for one inbound support message.""" + + category: TriageCategory = Field(description="Primary topic of the message.") + severity: int = Field(ge=1, le=4, description="1=info, 2=degraded, 3=broken, 4=data-loss/security/money.") + summary: str = Field(max_length=200, description="One-line neutral restatement of the issue.") + suggested_reply: str = Field(description="Draft first response an agent could send as-is.") + needs_human: bool = Field(description="True when a human must review before any reply is sent.") + + +@dataclass +class SupportContext: + """Per-request dependencies for the triage agent. + + Replace ``recent_tickets`` with a query against your ticket store; the + default keeps the template runnable with zero infrastructure. + """ + + user_id: str + recent_tickets: Sequence[str] = () + + +def build_support_triage_agent( + settings: "Settings", + *, + model: str | None = None, +) -> "Agent[SupportContext, TriageResult]": + """Build the support-triage agent against the LLM Gateway. + + Raises :class:`dogeapi.ai.agents.GatewayNotConfiguredError` when + ``LLM_GATEWAY_API_KEY`` is not configured. + """ + from pydantic_ai import RunContext + + agent: Agent[SupportContext, TriageResult] = build_agent( + settings, + model=model, + system_prompt=SYSTEM_PROMPT, + deps_type=SupportContext, + output_type=TriageResult, + retries=2, + ) + + @agent.tool + async def fetch_recent_tickets(ctx: RunContext[SupportContext]) -> list[str]: + """Recent ticket subjects for this user — lets the model spot repeat issues.""" + return list(ctx.deps.recent_tickets) + + @agent.output_validator + async def enforce_severity_floor(ctx: RunContext[SupportContext], output: TriageResult) -> TriageResult: + """Deterministic guardrail: severity-4 results always require a human.""" + if output.severity >= 4 and not output.needs_human: + return output.model_copy(update={"needs_human": True}) + return output + + return agent + + +__all__ = ( + "SYSTEM_PROMPT", + "SupportContext", + "TriageCategory", + "TriageResult", + "build_support_triage_agent", +) diff --git a/backend/tests/test_ai_templates.py b/backend/tests/test_ai_templates.py new file mode 100644 index 0000000..07ee9d8 --- /dev/null +++ b/backend/tests/test_ai_templates.py @@ -0,0 +1,298 @@ +"""Tests for the production agent templates (dogeapi.ai.templates). + +Same approach as ``test_pydantic_agent.py``: the gateway is mocked over HTTP +with respx so everything runs offline. Each template's guardrail is exercised +directly: + +- support triage: structured output parses, and the deterministic severity + floor flips ``needs_human`` on severity-4 results. +- RAG answerer: an uncited/mis-cited answer is rejected and retried; an empty + retrieval set forces ``insufficient_context``. +- eval harness: golden cases pass/fail independently, agent crashes fail the + single case, and the shipped sample goldens load. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import httpx +import pytest +import respx + +import dogeapi.ai.templates as templates_pkg +from dogeapi.ai.templates import ( + CitedAnswer, + GoldenCase, + RagContext, + RetrievedChunk, + SupportContext, + TriageResult, + build_rag_answerer, + build_support_triage_agent, + load_golden_cases, + run_golden_cases, +) +from dogeapi.settings import Settings + +GATEWAY_BASE = "https://api.llmgateway.io/v1" + + +def _settings(api_key: str = "test-key", base_url: str = GATEWAY_BASE) -> Settings: + return Settings( + JWT_SECRET_KEY="x" * 32, + LLM_GATEWAY_URL=base_url, + LLM_GATEWAY_API_KEY=api_key, + AI_DEFAULT_MODEL="gpt-5-mini", + ) + + +def _structured_response(arguments: dict[str, Any], *, response_id: str = "resp-1") -> httpx.Response: + """A chat completion whose only content is a ``final_result`` tool call.""" + return httpx.Response( + 200, + json={ + "id": response_id, + "object": "chat.completion", + "created": 1_700_000_000, + "model": "gpt-5-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": f"call-{response_id}", + "type": "function", + "function": { + "name": "final_result", + "arguments": json.dumps(arguments), + }, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": { + "prompt_tokens": 50, + "completion_tokens": 20, + "total_tokens": 70, + }, + }, + ) + + +# ─── Support triage ────────────────────────────────────────────────────── + + +@respx.mock +async def test_support_triage_parses_structured_output() -> None: + settings = _settings() + respx.post(f"{GATEWAY_BASE}/chat/completions").mock( + return_value=_structured_response( + { + "category": "billing", + "severity": 3, + "summary": "Customer was charged twice this month.", + "suggested_reply": "Sorry about that - we're looking into the duplicate charge now.", + "needs_human": True, + } + ) + ) + + agent = build_support_triage_agent(settings) + result = await agent.run( + "I was charged twice this month!", + deps=SupportContext(user_id="u_1", recent_tickets=["Duplicate charge in May"]), + ) + + assert isinstance(result.output, TriageResult) + assert result.output.category == "billing" + assert result.output.severity == 3 + assert result.output.needs_human is True + + +@respx.mock +async def test_support_triage_severity_floor_forces_human() -> None: + """A severity-4 verdict with needs_human=false must be corrected deterministically.""" + settings = _settings() + respx.post(f"{GATEWAY_BASE}/chat/completions").mock( + return_value=_structured_response( + { + "category": "bug", + "severity": 4, + "summary": "All org documents disappeared after sync.", + "suggested_reply": "We are escalating this immediately.", + "needs_human": False, + } + ) + ) + + agent = build_support_triage_agent(settings) + result = await agent.run( + "All of our documents disappeared after the last sync.", + deps=SupportContext(user_id="u_2"), + ) + + assert result.output.severity == 4 + assert result.output.needs_human is True + + +# ─── RAG answerer ──────────────────────────────────────────────────────── + +CHUNKS = [ + RetrievedChunk(id="c1", source="docs/billing.md#refunds", text="Refunds are issued within 5 business days."), + RetrievedChunk(id="c2", source="docs/billing.md#invoices", text="Invoices are emailed monthly."), +] + + +@respx.mock +async def test_rag_answerer_accepts_cited_answer() -> None: + settings = _settings() + respx.post(f"{GATEWAY_BASE}/chat/completions").mock( + return_value=_structured_response( + { + "answer": "Refunds are issued within 5 business days.", + "citations": ["c1"], + "insufficient_context": False, + } + ) + ) + + agent = build_rag_answerer(settings) + result = await agent.run("How long do refunds take?", deps=RagContext(chunks=CHUNKS)) + + assert isinstance(result.output, CitedAnswer) + assert result.output.citations == ["c1"] + + +@respx.mock +async def test_rag_answerer_retries_unknown_citation() -> None: + """A citation outside the retrieved set is rejected; the retry must succeed.""" + settings = _settings() + responses = iter( + [ + _structured_response( + {"answer": "Refunds take 5 days.", "citations": ["made-up-id"], "insufficient_context": False}, + response_id="resp-bad", + ), + _structured_response( + {"answer": "Refunds take 5 business days.", "citations": ["c1"], "insufficient_context": False}, + response_id="resp-good", + ), + ] + ) + route = respx.post(f"{GATEWAY_BASE}/chat/completions").mock(side_effect=lambda _request: next(responses)) + + agent = build_rag_answerer(settings) + result = await agent.run("How long do refunds take?", deps=RagContext(chunks=CHUNKS)) + + assert route.call_count == 2 + assert result.output.citations == ["c1"] + + +@respx.mock +async def test_rag_answerer_requires_insufficient_context_when_no_chunks() -> None: + """With an empty retrieval set, an improvised answer is rejected until the model concedes.""" + settings = _settings() + responses = iter( + [ + _structured_response( + {"answer": "Refunds take about a week, usually.", "citations": [], "insufficient_context": False}, + response_id="resp-improvised", + ), + _structured_response( + { + "answer": "The available documentation does not cover refunds.", + "citations": [], + "insufficient_context": True, + }, + response_id="resp-conceded", + ), + ] + ) + route = respx.post(f"{GATEWAY_BASE}/chat/completions").mock(side_effect=lambda _request: next(responses)) + + agent = build_rag_answerer(settings) + result = await agent.run("How long do refunds take?", deps=RagContext(chunks=[])) + + assert route.call_count == 2 + assert result.output.insufficient_context is True + assert result.output.citations == [] + + +# ─── Eval harness ──────────────────────────────────────────────────────── + + +@respx.mock +async def test_eval_harness_reports_pass_and_fail_independently() -> None: + settings = _settings() + respx.post(f"{GATEWAY_BASE}/chat/completions").mock( + return_value=_structured_response( + { + "category": "billing", + "severity": 3, + "summary": "Duplicate charge reported.", + "suggested_reply": "We're checking the duplicate charge now.", + "needs_human": True, + } + ) + ) + + agent = build_support_triage_agent(settings) + cases = [ + GoldenCase( + name="passes", + prompt="I was charged twice", + expected={"category": "billing", "needs_human": True}, + contains=["duplicate charge"], + ), + GoldenCase(name="fails", prompt="I was charged twice", expected={"category": "bug"}), + ] + + report = await run_golden_cases(agent, cases, deps_factory=lambda case: SupportContext(user_id="eval")) + + assert report.passed == 1 + assert report.failed == 1 + assert not report.ok + by_name = {r.name: r for r in report.results} + assert by_name["passes"].passed + assert by_name["fails"].failures == ["expected.category: wanted 'bug', got 'billing'"] + assert "[FAIL] fails" in report.summary() + + +@respx.mock +async def test_eval_harness_survives_agent_errors() -> None: + """A gateway blow-up fails that one case instead of aborting the run.""" + settings = _settings() + respx.post(f"{GATEWAY_BASE}/chat/completions").mock(return_value=httpx.Response(500, text="boom")) + + agent = build_support_triage_agent(settings) + report = await run_golden_cases( + agent, + [GoldenCase(name="crashes", prompt="hello", expected={"category": "other"})], + deps_factory=lambda case: SupportContext(user_id="eval"), + ) + + assert report.failed == 1 + assert "agent raised" in report.results[0].failures[0] + + +def test_shipped_golden_sample_loads() -> None: + path = Path(templates_pkg.__file__).parent / "golden" / "support_triage.json" + cases = load_golden_cases(path) + + assert len(cases) == 3 + assert len({case.name for case in cases}) == 3 + + +def test_load_golden_cases_rejects_non_list(tmp_path: Path) -> None: + bogus = tmp_path / "bogus.json" + bogus.write_text('{"name": "not-a-list"}', encoding="utf-8") + with pytest.raises(ValueError, match="top-level JSON list"): + load_golden_cases(bogus)