feat(evaluation): admin System tab for indexing speed and RAG quality

.gitignore

@@ -65,6 +65,8 @@ volumes/*
 !/services/.gitkeep # Keep the placeholder
 
 *.csv
+# The committed evaluation test set is source, not scratch data.
+!tests/evaluation/*.csv
 *.pkl
 
 #helm

CLAUDE.md

@@ -242,6 +242,59 @@ Optional web search augmentation via the Staan API, allowing the LLM to combine
 - `openrag/services/orchestrators/query_service.py` — `_prepare_for_web_only()`, web search logic in `_prepare_for_chat_completion()`
 - `openrag/api/routers/user/chat.py` — `__prepare_sources()` merges document and web sources
 
+### Evaluation (admin System page → Evaluation tab)
+
+On-demand benchmarking of indexing speed, retrieval quality and answer quality.
+
+**Flow** (`EvalRunner` Ray actor, `openrag/services/workers/eval_runner.py`): create the
+throwaway partition `__eval_<run_id>` → upload and time each corpus file over the real HTTP
+API → shell out to `promptfoo eval` twice → fold the outputs into metrics → drop the partition.
+
+- **Layering**: `EvaluationService` never touches Ray. It dispatches through the
+  `EvaluationRunner` port (`openrag/core/evaluation/runner.py`), implemented by
+  `RayEvaluationRunner` (`openrag/services/workers/eval_dispatcher.py`) and injected by the
+  container — the same port/adapter shape as `IndexingDispatcher`. The adapter resolves its
+  detached actor on first use, so building the service does not spawn a worker.
+- **Datasets** are admin-uploaded: a corpus plus a CSV test set
+  (`question,expected_answer,expected_file_ids`; the last column is optional and
+  `;`-separated). Files live under `<data_dir>/eval/<dataset_id>/`. The corpus is
+  streamed to disk rather than buffered, so its cap is a disk cost, not a RAM one.
+- **`file_id` sanitisation** (`sanitize_file_id`, `openrag/core/evaluation/identity.py`):
+  the indexing API accepts only `[A-Za-z0-9._:-]` in a `file_id`, so a corpus file cannot be
+  uploaded under a raw human filename. The runner uploads a sanitised id, and
+  `metrics.summarize` sanitises **both** sides of the ground-truth comparison, so a test set
+  naming `A B.pdf` still matches the stored `A_B.pdf`. Note `metadata.source` is the
+  server's storage path, not the original name — the `file_id` is what a human sees.
+- **Two promptfoo configs, not one** (`openrag/core/evaluation/promptfoo_config.py`):
+  retrieval hits `GET /search/partition/{partition}` (documents carry the chunk under
+  `content`, plus `metadata.file_id`), answers hit `POST /v1/chat/completions`. Each config
+  has one provider, so no assertion runs against an output shape it cannot read.
+  `transformResponse` must be a single JavaScript **expression** — an IIFE or any statement
+  makes promptfoo error every row before grading.
+- **Metrics** (`openrag/core/evaluation/metrics.py`): throughput from wall-clock, plus
+  hit rate / MRR / recall using the definitions in
+  `tests/load/automatic-evaluation-pipeline/README.md`. Rows without `expected_file_ids`
+  are reported as `skipped_cases`, never as misses. Percentiles are nearest-rank
+  (`ceil`) — `round` would break ties to even and report the wrong observation.
+- **Auth**: runs authenticate as the non-admin service user `__openrag_eval__`, whose token
+  is regenerated at the start of every run, so no usable plaintext token is stored at rest.
+- **Concurrency**: one run at a time. Enforced by the partial unique index
+  `ux_eval_runs_single_active`, not by a read-then-insert — the run row is created *before*
+  the token is regenerated, so two racing starts cannot revoke each other's credentials.
+  `POST /evaluation/runs` returns 409 on the loser, and 503 if the runner cannot be pinged.
+  A run orphaned by an actor restart is reaped by cancelling it, which writes the terminal
+  status directly; a failed provision releases the row the same way.
+- **Config** (`openrag/core/config/evaluation.py`, `evaluation:` in `conf/config.yaml`):
+  limits and timeouts are env-overridable (`EVAL_*`, `PROMPTFOO_BIN`). The API base URL the
+  runner calls back on is `server.internal_url` (env: `OPENRAG_INTERNAL_URL`) — it is a
+  server property, not an eval one. The reserved partition prefix, CSV column names and the
+  `file_id` alphabet are deliberately *not* config — they are contracts with stored datasets.
+- Requires **Node 22** (from NodeSource; distro packages predate promptfoo's floor) + a
+  pinned promptfoo in **both** `infra/docker/api.Dockerfile` (compose runs Ray inside the
+  API container) and `infra/docker/ray.Dockerfile` (separate Ray cluster). Dataset files are
+  read from disk by the runner, so a **separate** Ray cluster needs `<data_dir>` on shared
+  storage.
+
 ### File Quota System
 
 Per-user file quota enforcement tracked via the `file_count` and `file_quota` columns on `users`, and `created_by` on `files`.

conf/config.yaml

@@ -122,9 +122,13 @@ verbose:
   level: INFO
 
 # --- Server ---
-# Env: PREFERRED_URL_SCHEME
+# Env: PREFERRED_URL_SCHEME, OPENRAG_INTERNAL_URL
 server:
   preferred_url_scheme: null
+  # How out-of-process workers (e.g. the evaluation runner) reach the API from
+  # inside the deployment. Change only if the API is not reachable under the
+  # compose service name.
+  internal_url: http://openrag:8080
 
 # --- LLM Context ---
 # Env: MAX_LLM_CONTEXT_SIZE, MAX_OUTPUT_TOKENS
@@ -363,3 +367,19 @@ mcp:
   similarity_threshold: 0.8
   download_timeout: 30.0
   max_download_bytes: 104857600  # 100 MiB
+
+# --- Evaluation ---
+# Env: PROMPTFOO_BIN, EVAL_MAX_CORPUS_MB, EVAL_MAX_TESTSET_MB,
+#      EVAL_MAX_TESTSET_ROWS, EVAL_TOP_K, EVAL_TASK_TIMEOUT,
+#      EVAL_TASK_POLL_SECONDS, EVAL_HTTP_TIMEOUT, EVAL_PROMPTFOO_TIMEOUT
+# The runner reaches the API via server.internal_url.
+evaluation:
+  promptfoo_bin: promptfoo
+  max_corpus_mb: 512
+  max_testset_mb: 5
+  max_testset_rows: 500
+  top_k: 5
+  task_timeout_seconds: 1800.0
+  task_poll_seconds: 1.0
+  http_timeout_seconds: 300.0
+  promptfoo_timeout_seconds: 3600.0

docs/content/docs/documentation/env_vars.md

@@ -572,6 +572,16 @@ The following environment variables configure the FastAPI server and control acc
 | `DEFAULT_FILE_QUOTA` | `int` | `-1` | Default per-user file quota. `<0` disables quotas globally; `>=0` sets the default limit when a user has no explicit quota. |
 | `PREFERRED_URL_SCHEME` | `string` | `null` | URL scheme (`http` or `https`) used when generating URLs in API responses (e.g., `task_status_url`). When running behind a reverse proxy that terminates SSL, set this to `https` to ensure generated URLs use the correct scheme. If unset, the scheme from the incoming request is used. |
 | `CORS_EXTRA_ORIGINS` | `string` | _(unset)_ | Semicolon-separated list of additional origins allowed by CORS (e.g. `https://app.example.com;https://other.example.com`). Extends the default list without replacing it. |
+| `OPENRAG_INTERNAL_URL` | `string` | `http://openrag:8080` | Overrides `server.internal_url`: the base URL out-of-process workers use to reach the API from inside the deployment. Used by the evaluation runner, which uploads the corpus and drives promptfoo over HTTP from the Ray container. Change only if the API is not reachable under the compose service name. |
+| `PROMPTFOO_BIN` | `string` | `promptfoo` | Executable the evaluation runner shells out to. The Ray image installs a pinned promptfoo on `PATH`; set this only for a custom location. |
+| `EVAL_MAX_CORPUS_MB` | `int` | `512` | Maximum total size of one dataset's corpus upload, in MB. Enforced while streaming to disk, so an inflated `Content-Length` cannot get past it. A dataset is re-indexed on every run, so an oversized corpus costs far more than the upload itself. |
+| `EVAL_MAX_TESTSET_MB` | `int` | `5` | Maximum size of the test-set CSV upload, in MB. |
+| `EVAL_MAX_TESTSET_ROWS` | `int` | `500` | Maximum number of questions in a test set. Each row costs one retrieval call plus one LLM-graded generation per run. |
+| `EVAL_TOP_K` | `int` | `5` | Chunks retrieved per question when measuring retrieval quality. Raising it makes hit rate and recall more forgiving, so compare runs only at a fixed value. |
+| `EVAL_TASK_TIMEOUT` | `float` | `1800` | Seconds to wait for one corpus file's indexing task before the run gives up on it. Raise it for slow parsers (large scanned PDFs through Marker). |
+| `EVAL_TASK_POLL_SECONDS` | `float` | `1.0` | How often the runner polls a file's indexing task status. |
+| `EVAL_HTTP_TIMEOUT` | `float` | `300` | Per-request timeout for the runner's own HTTP calls to the API. |
+| `EVAL_PROMPTFOO_TIMEOUT` | `float` | `3600` | Seconds allowed for one `promptfoo eval` invocation. Every row is graded by an LLM, so raise it for a slow grader or a large test set. |
 | `UVICORN_FORWARDED_ALLOW_IPS` | `string` | `127.0.0.1` | Comma-separated CIDRs/IPs (or `*`) whose `X-Forwarded-*` headers uvicorn trusts. **Required when OpenRAG runs behind a reverse proxy that lives outside loopback** (typical docker-compose / k8s — including the bundled admin-ui proxy). Otherwise `X-Forwarded-Proto` is dropped and OIDC cookies ship with `Secure=False` even over HTTPS, and `X-Forwarded-For` is dropped so per-user rate limits collapse onto the proxy's single IP. **Set this to your proxy's subnet, not `*`** — see the proxy-trust caution under [Rate Limiting](#rate-limiting) for why `*` can be spoofed. |
 | `MAX_UPLOAD_SIZE_MB` | `int` | `1024` | Maximum accepted upload size, in MB. `0` or a negative value means unlimited. |
 | `MAX_PARTITIONS_PER_USER` | `int` | `100` | Maximum number of partitions a non-admin user may own. `-1` disables the cap (unlimited). Admin users always bypass it. |

infra/docker/api.Dockerfile

@@ -18,7 +18,21 @@ RUN apt-get update && apt-get install -y \
 
 # install ffmpeg
 RUN apt update && \
-    apt install -y ffmpeg 
+    apt install -y ffmpeg
+
+# Node + promptfoo back the admin evaluation page, where EvalRunner shells out
+# to `promptfoo eval`. Also installed in ray.Dockerfile: Ray runs inside this
+# container unless the deployment uses a separate cluster. Pinned rather than
+# resolved at run time so a run never depends on npm reachability.
+ARG PROMPTFOO_VERSION=0.121.19
+# Node comes from NodeSource: the distro package predates promptfoo's floor.
+RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && npm install -g promptfoo@${PROMPTFOO_VERSION} \
+    && npm cache clean --force \
+    && rm -rf /var/lib/apt/lists/*
+ENV PROMPTFOO_DISABLE_TELEMETRY=1 \
+    PROMPTFOO_DISABLE_UPDATE=1
 
 # Set environment variables for Hugging Face cache location
 ENV XDG_CACHE_HOME=${XDG_CACHE_HOME:-/app/model_weights}

infra/docker/ray.Dockerfile

@@ -19,7 +19,20 @@ RUN apt-get update && apt-get install -y \
 
 # install ffmpeg
 RUN apt update && \
-    apt install -y ffmpeg 
+    apt install -y ffmpeg
+
+# Node + promptfoo back the admin evaluation page, where EvalRunner shells out
+# to `promptfoo eval`. Pinned rather than resolved at run time so a run never
+# depends on npm reachability.
+ARG PROMPTFOO_VERSION=0.121.19
+# Node comes from NodeSource: the distro package predates promptfoo's floor.
+RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && npm install -g promptfoo@${PROMPTFOO_VERSION} \
+    && npm cache clean --force \
+    && rm -rf /var/lib/apt/lists/*
+ENV PROMPTFOO_DISABLE_TELEMETRY=1 \
+    PROMPTFOO_DISABLE_UPDATE=1
 
 
 # Set environment variables for Hugging Face cache location

openrag/api/main.py

@@ -40,6 +40,7 @@
     SecurityHeadersMiddleware,
 )
 from api.routers.admin.cluster import router as actors_router
+from api.routers.admin.evaluation import router as evaluation_router
 from api.routers.admin.indexing import router as indexer_router
 from api.routers.admin.jobs import router as queue_router
 from api.routers.admin.model_endpoints import router as model_endpoints_router
@@ -117,6 +118,7 @@ class Tags(Enum):
     PARTITION = "Partitions & files"
     MODEL_ENDPOINTS = "Model Endpoints"
     PRESETS = "Presets"
+    EVALUATION = "Evaluation"
     QUEUE = "Queue management"
     ACTORS = "Ray Actors"
     USERS = "User management"
@@ -357,6 +359,7 @@ def get_config():
 app.include_router(partition_router, prefix="/partition", tags=[Tags.PARTITION])
 app.include_router(model_endpoints_router, prefix="/model-endpoints", tags=[Tags.MODEL_ENDPOINTS])
 app.include_router(presets_router, prefix="/presets", tags=[Tags.PRESETS])
+app.include_router(evaluation_router, prefix="/evaluation", tags=[Tags.EVALUATION])
 app.include_router(queue_router, prefix="/queue", tags=[Tags.QUEUE])
 app.include_router(actors_router, prefix="/actors", tags=[Tags.ACTORS])
 app.include_router(users_router, prefix="/users", tags=[Tags.USERS])

openrag/api/routers/admin/evaluation.py

@@ -0,0 +1,130 @@
+"""Admin routes for the evaluation page.
+
+Datasets are uploaded once and replayed by runs. Every route is admin-only:
+a run indexes a corpus, spends grader tokens, and occupies the single runner
+slot, so it is not something a partition editor should be able to trigger.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict
+from typing import Any
+
+from api.dependencies.auth import current_user, require_admin
+from api.schemas.admin.evaluation_schemas import (
+    EvalDatasetResponse,
+    EvalRunResponse,
+    EvalRunSummaryResponse,
+    StartRunRequest,
+)
+from di.providers import get_evaluation_service
+from fastapi import APIRouter, Depends, File, Form, UploadFile, status
+
+router = APIRouter(dependencies=[Depends(require_admin)])
+
+
+def _run_summary(run: Any) -> EvalRunSummaryResponse:
+    return EvalRunSummaryResponse(
+        id=run.id,
+        dataset_id=run.dataset_id,
+        status=run.status.value,
+        started_at=run.started_at,
+        finished_at=run.finished_at,
+        hit_rate=run.retrieval.hit_rate if run.retrieval else None,
+        mrr=run.retrieval.mrr if run.retrieval else None,
+        answer_pass_rate=run.answer.pass_rate if run.answer else None,
+        files_per_minute=run.indexing.files_per_minute if run.indexing else None,
+        error=run.error,
+    )
+
+
+def _run_detail(run: Any) -> EvalRunResponse:
+    return EvalRunResponse(
+        id=run.id,
+        dataset_id=run.dataset_id,
+        status=run.status.value,
+        started_at=run.started_at,
+        finished_at=run.finished_at,
+        indexing=asdict(run.indexing) if run.indexing else None,
+        retrieval=asdict(run.retrieval) if run.retrieval else None,
+        answer=asdict(run.answer) if run.answer else None,
+        cases=[asdict(case) for case in run.cases],
+        error=run.error,
+        created_by=run.created_by,
+    )
+
+
+@router.get("/datasets", response_model=list[EvalDatasetResponse])
+async def list_datasets(service=Depends(get_evaluation_service)):
+    """List stored evaluation datasets, newest first."""
+    return [asdict(dataset) for dataset in await service.list_datasets()]
+
+
+@router.post(
+    "/datasets",
+    response_model=EvalDatasetResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+async def create_dataset(
+    name: str = Form(..., description="Human-readable dataset name"),
+    testset: UploadFile = File(..., description="CSV: question,expected_answer,expected_file_ids"),
+    corpus: list[UploadFile] = File(..., description="Documents to index for the run"),
+    user=Depends(current_user),
+    service=Depends(get_evaluation_service),
+):
+    """Upload a corpus and its test set.
+
+    The CSV is validated here, so a bad test set fails now rather than after a
+    run has already indexed the corpus.
+    """
+    # Pass the open streams, not the bytes: large uploads are already spooled
+    # to disk, and reading them here would pull them into memory unbounded.
+    dataset = await service.create_dataset(
+        name=name,
+        corpus=[(upload.filename or "unnamed", upload.file) for upload in corpus],
+        testset=testset.file,
+        user_id=user.get("id") if isinstance(user, dict) else None,
+    )
+    return asdict(dataset)
+
+
+@router.delete("/datasets/{dataset_id}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_dataset(dataset_id: str, service=Depends(get_evaluation_service)):
+    """Delete a dataset and its stored files."""
+    await service.delete_dataset(dataset_id)
+
+
+@router.get("/runs", response_model=list[EvalRunSummaryResponse])
+async def list_runs(limit: int = 50, service=Depends(get_evaluation_service)):
+    """Run history, newest first."""
+    return [_run_summary(run) for run in await service.list_runs(limit)]
+
+
+@router.post("/runs", response_model=EvalRunResponse, status_code=status.HTTP_202_ACCEPTED)
+async def start_run(
+    body: StartRunRequest,
+    user=Depends(current_user),
+    service=Depends(get_evaluation_service),
+):
+    """Queue a run against a dataset.
+
+    Returns ``409`` when a run is already in flight — runs execute one at a
+    time so that indexing timings stay comparable between them.
+    """
+    run = await service.start_run(
+        body.dataset_id,
+        user.get("id") if isinstance(user, dict) else None,
+    )
+    return _run_detail(run)
+
+
+@router.get("/runs/{run_id}", response_model=EvalRunResponse)
+async def get_run(run_id: str, service=Depends(get_evaluation_service)):
+    """One run with its metrics and per-question detail."""
+    return _run_detail(await service.get_run(run_id))
+
+
+@router.post("/runs/{run_id}/cancel", response_model=EvalRunResponse)
+async def cancel_run(run_id: str, service=Depends(get_evaluation_service)):
+    """Ask the runner to abandon an in-flight run."""
+    return _run_detail(await service.cancel_run(run_id))