feat(evaluation) 5/15: generate the two promptfoo configs a run executes

openrag/core/evaluation/__init__.py

@@ -2,9 +2,12 @@
 
 from core.evaluation.identity import sanitize_file_id
 from core.evaluation.metrics import extract_results, indexing_metrics, summarize
+from core.evaluation.promptfoo_config import build_answer_config, build_retrieval_config
 from core.evaluation.testset import parse_testset
 
 __all__ = [
+    "build_answer_config",
+    "build_retrieval_config",
     "extract_results",
     "indexing_metrics",
     "parse_testset",

openrag/core/evaluation/promptfoo_config.py

@@ -0,0 +1,171 @@
+"""Generation of the promptfoo configs a run executes.
+
+A run produces two configs rather than one, because the two questions need
+different endpoints:
+
+* **retrieval** hits ``GET /search/partition/{partition}``, whose documents
+  carry the chunk ``content`` — the text that ``context-relevance`` grades and
+  whose ``metadata.file_id`` feeds hit rate / MRR / recall.
+* **answer** hits ``POST /v1/chat/completions``, whose ``extra.sources`` carry
+  source metadata but no chunk text, and whose message content is what
+  ``factuality`` and ``llm-rubric`` grade.
+
+Keeping them separate means every assertion in a config applies to that
+config's single provider, so no assertion ever runs against an output shape it
+cannot read.
+
+This module is pure: it returns plain dicts. Serialisation and execution live
+in the worker.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from core.models.evaluation import EvalTestCase
+
+#: promptfoo templates with Nunjucks; ``urlencode`` keeps a question
+#: containing ``&`` or ``?`` from corrupting the search query string.
+_QUERY_TEMPLATE = "{{ query | urlencode }}"
+
+#: Extract the ``documents`` array from the search response.
+_SEARCH_TRANSFORM = "json.documents || []"
+
+#: ``transformResponse`` must be a single JavaScript expression — statements
+#: and IIFEs are rejected — so this extracts the answer text and nothing more.
+#: Retrieved sources come from the retrieval pass instead.
+_CHAT_TRANSFORM = "json.choices[0].message.content"
+
+_RUBRIC = (
+    "The response must answer the question using the retrieved documents. "
+    "Grade it against this reference answer: {{expected_answer}}. "
+    "Pass if the response conveys the same facts, even if worded differently. "
+    "Fail if it contradicts the reference, is empty, or refuses to answer."
+)
+
+
+def _grader(model: str, base_url: str, api_key: str | None) -> dict[str, Any]:
+    """The provider promptfoo uses for model-graded assertions.
+
+    Points at OpenRAG's own OpenAI-compatible LLM endpoint so an eval needs no
+    third-party credentials.
+    """
+    config: dict[str, Any] = {"apiBaseUrl": base_url}
+    # vLLM ignores the key but the OpenAI client refuses to send without one.
+    config["apiKey"] = api_key or "sk-no-key-required"
+    return {"id": f"openai:chat:{model}", "config": config}
+
+
+def _tests(cases: Sequence[EvalTestCase], asserts: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """One promptfoo test per case, all sharing the same assertions.
+
+    ``expected_file_ids`` is deliberately absent from ``vars``: no assertion
+    reads it. The ranking metrics are computed from the retrieved ids in
+    ``metrics.summarize``, not by promptfoo.
+    """
+    return [
+        {
+            "vars": {"query": case.query, "expected_answer": case.expected_answer},
+            # A fresh copy per test: a shared list would serialise as a YAML
+            # anchor plus aliases.
+            "assert": [dict(assertion) for assertion in asserts],
+        }
+        for case in cases
+    ]
+
+
+def build_retrieval_config(
+    *,
+    cases: Sequence[EvalTestCase],
+    api_base_url: str,
+    partition: str,
+    token: str,
+    grader_model: str,
+    grader_base_url: str,
+    grader_api_key: str | None = None,
+    top_k: int = 5,
+    relevance_threshold: float = 0.0,
+) -> dict[str, Any]:
+    """Config that measures what the retriever returns for each question.
+
+    ``relevance_threshold`` defaults to 0 so ``context-relevance`` records a
+    score without failing the run; the deterministic ranking metrics are
+    computed from the same responses afterwards.
+    """
+    url = f"{api_base_url.rstrip('/')}/search/partition/{partition}?text={_QUERY_TEMPLATE}&top_k={top_k}"
+    return {
+        "description": f"OpenRAG retrieval eval ({partition})",
+        "prompts": ["{{query}}"],
+        "providers": [
+            {
+                "id": "https",
+                "label": "openrag-retrieval",
+                "config": {
+                    "url": url,
+                    "method": "GET",
+                    "headers": {"Authorization": f"Bearer {token}"},
+                    "transformResponse": _SEARCH_TRANSFORM,
+                },
+            }
+        ],
+        "defaultTest": {"options": {"provider": _grader(grader_model, grader_base_url, grader_api_key)}},
+        "tests": _tests(
+            cases,
+            [
+                {
+                    "type": "context-relevance",
+                    "contextTransform": "output.map(d => d.content).join('\\n\\n')",
+                    "threshold": relevance_threshold,
+                }
+            ],
+        ),
+    }
+
+
+def build_answer_config(
+    *,
+    cases: Sequence[EvalTestCase],
+    api_base_url: str,
+    partition: str,
+    token: str,
+    grader_model: str,
+    grader_base_url: str,
+    grader_api_key: str | None = None,
+) -> dict[str, Any]:
+    """Config that grades the generated answer against the expected one."""
+    return {
+        "description": f"OpenRAG answer eval ({partition})",
+        "prompts": ["{{query}}"],
+        "providers": [
+            {
+                "id": "https",
+                "label": "openrag-chat",
+                "config": {
+                    "url": f"{api_base_url.rstrip('/')}/v1/chat/completions",
+                    "method": "POST",
+                    "headers": {
+                        "Authorization": f"Bearer {token}",
+                        "Content-Type": "application/json",
+                    },
+                    "body": {
+                        "model": f"openrag-{partition}",
+                        "messages": [{"role": "user", "content": "{{query}}"}],
+                        "stream": False,
+                    },
+                    "transformResponse": _CHAT_TRANSFORM,
+                },
+            }
+        ],
+        "defaultTest": {"options": {"provider": _grader(grader_model, grader_base_url, grader_api_key)}},
+        "tests": _tests(
+            cases,
+            [
+                {"type": "factuality", "value": "{{expected_answer}}"},
+                {"type": "llm-rubric", "value": _RUBRIC},
+            ],
+        ),
+    }
+
+
+__all__ = ["build_answer_config", "build_retrieval_config"]

tests/unit/core/evaluation/test_promptfoo_config.py

@@ -0,0 +1,99 @@
+"""Tests for the generated promptfoo configs."""
+
+from __future__ import annotations
+
+from core.evaluation.promptfoo_config import build_answer_config, build_retrieval_config
+from core.models.evaluation import EvalTestCase
+
+CASES = [
+    EvalTestCase(query="What is the refund window?", expected_answer="30 days", expected_file_ids=("p.pdf",)),
+    EvalTestCase(query="Who approves?", expected_answer="The CFO"),
+]
+
+COMMON = {
+    "api_base_url": "http://openrag:8080/",
+    "partition": "__eval_abc",
+    "token": "or-secret",
+    "grader_model": "qwen",
+    "grader_base_url": "http://vllm:8000/v1",
+}
+
+
+def test_retrieval_provider_targets_the_single_partition_search_route():
+    config = build_retrieval_config(cases=CASES, **COMMON, top_k=7)
+    url = config["providers"][0]["config"]["url"]
+    assert url.startswith("http://openrag:8080/search/partition/__eval_abc")
+    assert "top_k=7" in url
+
+
+def test_retrieval_query_is_url_encoded():
+    """A question containing '&' would otherwise truncate the query string."""
+    config = build_retrieval_config(cases=CASES, **COMMON)
+    assert "{{ query | urlencode }}" in config["providers"][0]["config"]["url"]
+
+
+def test_retrieval_asserts_on_the_chunk_text():
+    config = build_retrieval_config(cases=CASES, **COMMON)
+    assertion = config["tests"][0]["assert"][0]
+    assert assertion["type"] == "context-relevance"
+    assert "d.content" in assertion["contextTransform"]
+
+
+def test_answer_provider_posts_to_the_partition_scoped_model():
+    config = build_answer_config(cases=CASES, **COMMON)
+    body = config["providers"][0]["config"]["body"]
+    assert config["providers"][0]["config"]["url"] == "http://openrag:8080/v1/chat/completions"
+    assert body["model"] == "openrag-__eval_abc"
+    assert body["stream"] is False
+
+
+def test_answer_transform_is_a_single_expression():
+    """promptfoo evaluates transformResponse as an expression — a statement or
+    an IIFE fails at runtime with a transform error, which manifests as every
+    answer scoring zero."""
+    transform = build_answer_config(cases=CASES, **COMMON)["providers"][0]["config"]["transformResponse"]
+    assert transform == "json.choices[0].message.content"
+    assert "return" not in transform
+    assert ";" not in transform
+
+
+def test_answer_grades_against_the_expected_answer():
+    config = build_answer_config(cases=CASES, **COMMON)
+    types = [assertion["type"] for assertion in config["tests"][0]["assert"]]
+    assert types == ["factuality", "llm-rubric"]
+    assert config["tests"][0]["assert"][0]["value"] == "{{expected_answer}}"
+
+
+def test_both_configs_send_the_bearer_token():
+    for config in (
+        build_retrieval_config(cases=CASES, **COMMON),
+        build_answer_config(cases=CASES, **COMMON),
+    ):
+        headers = config["providers"][0]["config"]["headers"]
+        assert headers["Authorization"] == "Bearer or-secret"
+
+
+def test_grader_points_at_the_configured_openrag_llm():
+    """Model-graded assertions must not silently fall back to OpenAI."""
+    config = build_answer_config(cases=CASES, **COMMON)
+    grader = config["defaultTest"]["options"]["provider"]
+    assert grader["id"] == "openai:chat:qwen"
+    assert grader["config"]["apiBaseUrl"] == "http://vllm:8000/v1"
+    assert grader["config"]["apiKey"]
+
+
+def test_every_case_becomes_a_test_with_its_vars():
+    """Only the vars an assertion actually templates are emitted — the ranking
+    metrics read expected_file_ids from the test set, not from promptfoo."""
+    config = build_retrieval_config(cases=CASES, **COMMON)
+    assert len(config["tests"]) == 2
+    assert config["tests"][0]["vars"] == {
+        "query": "What is the refund window?",
+        "expected_answer": "30 days",
+    }
+
+
+def test_assertions_are_not_shared_between_tests():
+    """A shared list would serialise as a YAML anchor plus aliases."""
+    tests = build_answer_config(cases=CASES, **COMMON)["tests"]
+    assert tests[0]["assert"][0] is not tests[1]["assert"][0]