diff --git a/.gitignore b/.gitignore index 00b6d2cf1..4153768c4 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,8 @@ volumes/* !/services/.gitkeep # Keep the placeholder *.csv +# The committed evaluation test set is source, not scratch data. +!tests/evaluation/*.csv *.pkl #helm diff --git a/CLAUDE.md b/CLAUDE.md index 07590973b..59891eed4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -242,6 +242,59 @@ Optional web search augmentation via the Staan API, allowing the LLM to combine - `openrag/services/orchestrators/query_service.py` — `_prepare_for_web_only()`, web search logic in `_prepare_for_chat_completion()` - `openrag/api/routers/user/chat.py` — `__prepare_sources()` merges document and web sources +### Evaluation (admin System page → Evaluation tab) + +On-demand benchmarking of indexing speed, retrieval quality and answer quality. + +**Flow** (`EvalRunner` Ray actor, `openrag/services/workers/eval_runner.py`): create the +throwaway partition `__eval_` → upload and time each corpus file over the real HTTP +API → shell out to `promptfoo eval` twice → fold the outputs into metrics → drop the partition. + +- **Layering**: `EvaluationService` never touches Ray. It dispatches through the + `EvaluationRunner` port (`openrag/core/evaluation/runner.py`), implemented by + `RayEvaluationRunner` (`openrag/services/workers/eval_dispatcher.py`) and injected by the + container — the same port/adapter shape as `IndexingDispatcher`. The adapter resolves its + detached actor on first use, so building the service does not spawn a worker. +- **Datasets** are admin-uploaded: a corpus plus a CSV test set + (`question,expected_answer,expected_file_ids`; the last column is optional and + `;`-separated). Files live under `/eval//`. The corpus is + streamed to disk rather than buffered, so its cap is a disk cost, not a RAM one. +- **`file_id` sanitisation** (`sanitize_file_id`, `openrag/core/evaluation/identity.py`): + the indexing API accepts only `[A-Za-z0-9._:-]` in a `file_id`, so a corpus file cannot be + uploaded under a raw human filename. The runner uploads a sanitised id, and + `metrics.summarize` sanitises **both** sides of the ground-truth comparison, so a test set + naming `A B.pdf` still matches the stored `A_B.pdf`. Note `metadata.source` is the + server's storage path, not the original name — the `file_id` is what a human sees. +- **Two promptfoo configs, not one** (`openrag/core/evaluation/promptfoo_config.py`): + retrieval hits `GET /search/partition/{partition}` (documents carry the chunk under + `content`, plus `metadata.file_id`), answers hit `POST /v1/chat/completions`. Each config + has one provider, so no assertion runs against an output shape it cannot read. + `transformResponse` must be a single JavaScript **expression** — an IIFE or any statement + makes promptfoo error every row before grading. +- **Metrics** (`openrag/core/evaluation/metrics.py`): throughput from wall-clock, plus + hit rate / MRR / recall using the definitions in + `tests/load/automatic-evaluation-pipeline/README.md`. Rows without `expected_file_ids` + are reported as `skipped_cases`, never as misses. Percentiles are nearest-rank + (`ceil`) — `round` would break ties to even and report the wrong observation. +- **Auth**: runs authenticate as the non-admin service user `__openrag_eval__`, whose token + is regenerated at the start of every run, so no usable plaintext token is stored at rest. +- **Concurrency**: one run at a time. Enforced by the partial unique index + `ux_eval_runs_single_active`, not by a read-then-insert — the run row is created *before* + the token is regenerated, so two racing starts cannot revoke each other's credentials. + `POST /evaluation/runs` returns 409 on the loser, and 503 if the runner cannot be pinged. + A run orphaned by an actor restart is reaped by cancelling it, which writes the terminal + status directly; a failed provision releases the row the same way. +- **Config** (`openrag/core/config/evaluation.py`, `evaluation:` in `conf/config.yaml`): + limits and timeouts are env-overridable (`EVAL_*`, `PROMPTFOO_BIN`). The API base URL the + runner calls back on is `server.internal_url` (env: `OPENRAG_INTERNAL_URL`) — it is a + server property, not an eval one. The reserved partition prefix, CSV column names and the + `file_id` alphabet are deliberately *not* config — they are contracts with stored datasets. +- Requires **Node 22** (from NodeSource; distro packages predate promptfoo's floor) + a + pinned promptfoo in **both** `infra/docker/api.Dockerfile` (compose runs Ray inside the + API container) and `infra/docker/ray.Dockerfile` (separate Ray cluster). Dataset files are + read from disk by the runner, so a **separate** Ray cluster needs `` on shared + storage. + ### File Quota System Per-user file quota enforcement tracked via the `file_count` and `file_quota` columns on `users`, and `created_by` on `files`. diff --git a/conf/config.yaml b/conf/config.yaml index 11d605a73..5c28fc757 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -122,9 +122,13 @@ verbose: level: INFO # --- Server --- -# Env: PREFERRED_URL_SCHEME +# Env: PREFERRED_URL_SCHEME, OPENRAG_INTERNAL_URL server: preferred_url_scheme: null + # How out-of-process workers (e.g. the evaluation runner) reach the API from + # inside the deployment. Change only if the API is not reachable under the + # compose service name. + internal_url: http://openrag:8080 # --- LLM Context --- # Env: MAX_LLM_CONTEXT_SIZE, MAX_OUTPUT_TOKENS @@ -363,3 +367,19 @@ mcp: similarity_threshold: 0.8 download_timeout: 30.0 max_download_bytes: 104857600 # 100 MiB + +# --- Evaluation --- +# Env: PROMPTFOO_BIN, EVAL_MAX_CORPUS_MB, EVAL_MAX_TESTSET_MB, +# EVAL_MAX_TESTSET_ROWS, EVAL_TOP_K, EVAL_TASK_TIMEOUT, +# EVAL_TASK_POLL_SECONDS, EVAL_HTTP_TIMEOUT, EVAL_PROMPTFOO_TIMEOUT +# The runner reaches the API via server.internal_url. +evaluation: + promptfoo_bin: promptfoo + max_corpus_mb: 512 + max_testset_mb: 5 + max_testset_rows: 500 + top_k: 5 + task_timeout_seconds: 1800.0 + task_poll_seconds: 1.0 + http_timeout_seconds: 300.0 + promptfoo_timeout_seconds: 3600.0 diff --git a/docs/content/docs/documentation/env_vars.md b/docs/content/docs/documentation/env_vars.md index bced96799..1f4247b24 100644 --- a/docs/content/docs/documentation/env_vars.md +++ b/docs/content/docs/documentation/env_vars.md @@ -572,6 +572,16 @@ The following environment variables configure the FastAPI server and control acc | `DEFAULT_FILE_QUOTA` | `int` | `-1` | Default per-user file quota. `<0` disables quotas globally; `>=0` sets the default limit when a user has no explicit quota. | | `PREFERRED_URL_SCHEME` | `string` | `null` | URL scheme (`http` or `https`) used when generating URLs in API responses (e.g., `task_status_url`). When running behind a reverse proxy that terminates SSL, set this to `https` to ensure generated URLs use the correct scheme. If unset, the scheme from the incoming request is used. | | `CORS_EXTRA_ORIGINS` | `string` | _(unset)_ | Semicolon-separated list of additional origins allowed by CORS (e.g. `https://app.example.com;https://other.example.com`). Extends the default list without replacing it. | +| `OPENRAG_INTERNAL_URL` | `string` | `http://openrag:8080` | Overrides `server.internal_url`: the base URL out-of-process workers use to reach the API from inside the deployment. Used by the evaluation runner, which uploads the corpus and drives promptfoo over HTTP from the Ray container. Change only if the API is not reachable under the compose service name. | +| `PROMPTFOO_BIN` | `string` | `promptfoo` | Executable the evaluation runner shells out to. The Ray image installs a pinned promptfoo on `PATH`; set this only for a custom location. | +| `EVAL_MAX_CORPUS_MB` | `int` | `512` | Maximum total size of one dataset's corpus upload, in MB. Enforced while streaming to disk, so an inflated `Content-Length` cannot get past it. A dataset is re-indexed on every run, so an oversized corpus costs far more than the upload itself. | +| `EVAL_MAX_TESTSET_MB` | `int` | `5` | Maximum size of the test-set CSV upload, in MB. | +| `EVAL_MAX_TESTSET_ROWS` | `int` | `500` | Maximum number of questions in a test set. Each row costs one retrieval call plus one LLM-graded generation per run. | +| `EVAL_TOP_K` | `int` | `5` | Chunks retrieved per question when measuring retrieval quality. Raising it makes hit rate and recall more forgiving, so compare runs only at a fixed value. | +| `EVAL_TASK_TIMEOUT` | `float` | `1800` | Seconds to wait for one corpus file's indexing task before the run gives up on it. Raise it for slow parsers (large scanned PDFs through Marker). | +| `EVAL_TASK_POLL_SECONDS` | `float` | `1.0` | How often the runner polls a file's indexing task status. | +| `EVAL_HTTP_TIMEOUT` | `float` | `300` | Per-request timeout for the runner's own HTTP calls to the API. | +| `EVAL_PROMPTFOO_TIMEOUT` | `float` | `3600` | Seconds allowed for one `promptfoo eval` invocation. Every row is graded by an LLM, so raise it for a slow grader or a large test set. | | `UVICORN_FORWARDED_ALLOW_IPS` | `string` | `127.0.0.1` | Comma-separated CIDRs/IPs (or `*`) whose `X-Forwarded-*` headers uvicorn trusts. **Required when OpenRAG runs behind a reverse proxy that lives outside loopback** (typical docker-compose / k8s — including the bundled admin-ui proxy). Otherwise `X-Forwarded-Proto` is dropped and OIDC cookies ship with `Secure=False` even over HTTPS, and `X-Forwarded-For` is dropped so per-user rate limits collapse onto the proxy's single IP. **Set this to your proxy's subnet, not `*`** — see the proxy-trust caution under [Rate Limiting](#rate-limiting) for why `*` can be spoofed. | | `MAX_UPLOAD_SIZE_MB` | `int` | `1024` | Maximum accepted upload size, in MB. `0` or a negative value means unlimited. | | `MAX_PARTITIONS_PER_USER` | `int` | `100` | Maximum number of partitions a non-admin user may own. `-1` disables the cap (unlimited). Admin users always bypass it. | diff --git a/infra/docker/api.Dockerfile b/infra/docker/api.Dockerfile index 0e3669f95..d6b934219 100644 --- a/infra/docker/api.Dockerfile +++ b/infra/docker/api.Dockerfile @@ -18,7 +18,21 @@ RUN apt-get update && apt-get install -y \ # install ffmpeg RUN apt update && \ - apt install -y ffmpeg + apt install -y ffmpeg + +# Node + promptfoo back the admin evaluation page, where EvalRunner shells out +# to `promptfoo eval`. Also installed in ray.Dockerfile: Ray runs inside this +# container unless the deployment uses a separate cluster. Pinned rather than +# resolved at run time so a run never depends on npm reachability. +ARG PROMPTFOO_VERSION=0.121.19 +# Node comes from NodeSource: the distro package predates promptfoo's floor. +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && npm install -g promptfoo@${PROMPTFOO_VERSION} \ + && npm cache clean --force \ + && rm -rf /var/lib/apt/lists/* +ENV PROMPTFOO_DISABLE_TELEMETRY=1 \ + PROMPTFOO_DISABLE_UPDATE=1 # Set environment variables for Hugging Face cache location ENV XDG_CACHE_HOME=${XDG_CACHE_HOME:-/app/model_weights} diff --git a/infra/docker/ray.Dockerfile b/infra/docker/ray.Dockerfile index 2029065cf..94c54f0ff 100644 --- a/infra/docker/ray.Dockerfile +++ b/infra/docker/ray.Dockerfile @@ -19,7 +19,20 @@ RUN apt-get update && apt-get install -y \ # install ffmpeg RUN apt update && \ - apt install -y ffmpeg + apt install -y ffmpeg + +# Node + promptfoo back the admin evaluation page, where EvalRunner shells out +# to `promptfoo eval`. Pinned rather than resolved at run time so a run never +# depends on npm reachability. +ARG PROMPTFOO_VERSION=0.121.19 +# Node comes from NodeSource: the distro package predates promptfoo's floor. +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && npm install -g promptfoo@${PROMPTFOO_VERSION} \ + && npm cache clean --force \ + && rm -rf /var/lib/apt/lists/* +ENV PROMPTFOO_DISABLE_TELEMETRY=1 \ + PROMPTFOO_DISABLE_UPDATE=1 # Set environment variables for Hugging Face cache location diff --git a/openrag/api/main.py b/openrag/api/main.py index e3255a5d8..543ce22c0 100644 --- a/openrag/api/main.py +++ b/openrag/api/main.py @@ -40,6 +40,7 @@ SecurityHeadersMiddleware, ) from api.routers.admin.cluster import router as actors_router +from api.routers.admin.evaluation import router as evaluation_router from api.routers.admin.indexing import router as indexer_router from api.routers.admin.jobs import router as queue_router from api.routers.admin.model_endpoints import router as model_endpoints_router @@ -117,6 +118,7 @@ class Tags(Enum): PARTITION = "Partitions & files" MODEL_ENDPOINTS = "Model Endpoints" PRESETS = "Presets" + EVALUATION = "Evaluation" QUEUE = "Queue management" ACTORS = "Ray Actors" USERS = "User management" @@ -357,6 +359,7 @@ def get_config(): app.include_router(partition_router, prefix="/partition", tags=[Tags.PARTITION]) app.include_router(model_endpoints_router, prefix="/model-endpoints", tags=[Tags.MODEL_ENDPOINTS]) app.include_router(presets_router, prefix="/presets", tags=[Tags.PRESETS]) +app.include_router(evaluation_router, prefix="/evaluation", tags=[Tags.EVALUATION]) app.include_router(queue_router, prefix="/queue", tags=[Tags.QUEUE]) app.include_router(actors_router, prefix="/actors", tags=[Tags.ACTORS]) app.include_router(users_router, prefix="/users", tags=[Tags.USERS]) diff --git a/openrag/api/routers/admin/evaluation.py b/openrag/api/routers/admin/evaluation.py new file mode 100644 index 000000000..c5ced0cb9 --- /dev/null +++ b/openrag/api/routers/admin/evaluation.py @@ -0,0 +1,130 @@ +"""Admin routes for the evaluation page. + +Datasets are uploaded once and replayed by runs. Every route is admin-only: +a run indexes a corpus, spends grader tokens, and occupies the single runner +slot, so it is not something a partition editor should be able to trigger. +""" + +from __future__ import annotations + +from dataclasses import asdict +from typing import Any + +from api.dependencies.auth import current_user, require_admin +from api.schemas.admin.evaluation_schemas import ( + EvalDatasetResponse, + EvalRunResponse, + EvalRunSummaryResponse, + StartRunRequest, +) +from di.providers import get_evaluation_service +from fastapi import APIRouter, Depends, File, Form, UploadFile, status + +router = APIRouter(dependencies=[Depends(require_admin)]) + + +def _run_summary(run: Any) -> EvalRunSummaryResponse: + return EvalRunSummaryResponse( + id=run.id, + dataset_id=run.dataset_id, + status=run.status.value, + started_at=run.started_at, + finished_at=run.finished_at, + hit_rate=run.retrieval.hit_rate if run.retrieval else None, + mrr=run.retrieval.mrr if run.retrieval else None, + answer_pass_rate=run.answer.pass_rate if run.answer else None, + files_per_minute=run.indexing.files_per_minute if run.indexing else None, + error=run.error, + ) + + +def _run_detail(run: Any) -> EvalRunResponse: + return EvalRunResponse( + id=run.id, + dataset_id=run.dataset_id, + status=run.status.value, + started_at=run.started_at, + finished_at=run.finished_at, + indexing=asdict(run.indexing) if run.indexing else None, + retrieval=asdict(run.retrieval) if run.retrieval else None, + answer=asdict(run.answer) if run.answer else None, + cases=[asdict(case) for case in run.cases], + error=run.error, + created_by=run.created_by, + ) + + +@router.get("/datasets", response_model=list[EvalDatasetResponse]) +async def list_datasets(service=Depends(get_evaluation_service)): + """List stored evaluation datasets, newest first.""" + return [asdict(dataset) for dataset in await service.list_datasets()] + + +@router.post( + "/datasets", + response_model=EvalDatasetResponse, + status_code=status.HTTP_201_CREATED, +) +async def create_dataset( + name: str = Form(..., description="Human-readable dataset name"), + testset: UploadFile = File(..., description="CSV: question,expected_answer,expected_file_ids"), + corpus: list[UploadFile] = File(..., description="Documents to index for the run"), + user=Depends(current_user), + service=Depends(get_evaluation_service), +): + """Upload a corpus and its test set. + + The CSV is validated here, so a bad test set fails now rather than after a + run has already indexed the corpus. + """ + # Pass the open streams, not the bytes: large uploads are already spooled + # to disk, and reading them here would pull them into memory unbounded. + dataset = await service.create_dataset( + name=name, + corpus=[(upload.filename or "unnamed", upload.file) for upload in corpus], + testset=testset.file, + user_id=user.get("id") if isinstance(user, dict) else None, + ) + return asdict(dataset) + + +@router.delete("/datasets/{dataset_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_dataset(dataset_id: str, service=Depends(get_evaluation_service)): + """Delete a dataset and its stored files.""" + await service.delete_dataset(dataset_id) + + +@router.get("/runs", response_model=list[EvalRunSummaryResponse]) +async def list_runs(limit: int = 50, service=Depends(get_evaluation_service)): + """Run history, newest first.""" + return [_run_summary(run) for run in await service.list_runs(limit)] + + +@router.post("/runs", response_model=EvalRunResponse, status_code=status.HTTP_202_ACCEPTED) +async def start_run( + body: StartRunRequest, + user=Depends(current_user), + service=Depends(get_evaluation_service), +): + """Queue a run against a dataset. + + Returns ``409`` when a run is already in flight — runs execute one at a + time so that indexing timings stay comparable between them. + """ + run = await service.start_run( + body.dataset_id, + user.get("id") if isinstance(user, dict) else None, + ) + return _run_detail(run) + + +@router.get("/runs/{run_id}", response_model=EvalRunResponse) +async def get_run(run_id: str, service=Depends(get_evaluation_service)): + """One run with its metrics and per-question detail.""" + return _run_detail(await service.get_run(run_id)) + + +@router.post("/runs/{run_id}/cancel", response_model=EvalRunResponse) +async def cancel_run(run_id: str, service=Depends(get_evaluation_service)): + """Ask the runner to abandon an in-flight run.""" + return _run_detail(await service.cancel_run(run_id)) diff --git a/openrag/api/schemas/admin/evaluation_schemas.py b/openrag/api/schemas/admin/evaluation_schemas.py new file mode 100644 index 000000000..f88dada89 --- /dev/null +++ b/openrag/api/schemas/admin/evaluation_schemas.py @@ -0,0 +1,113 @@ +"""Response models for the admin evaluation endpoints.""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, Field + + +class EvalDatasetResponse(BaseModel): + """A stored corpus + test set.""" + + id: str + name: str + corpus_file_count: int + testset_row_count: int + created_at: datetime | None = None + created_by: int | None = None + + +class FileIndexingSampleResponse(BaseModel): + filename: str + size_bytes: int + duration_seconds: float + failed: bool = False + + +class IndexingMetricsResponse(BaseModel): + files_total: int + files_failed: int + bytes_total: int + wall_seconds: float + files_per_minute: float + megabytes_per_second: float + p50_seconds: float + p95_seconds: float + by_extension: dict[str, dict[str, float]] = Field(default_factory=dict) + samples: list[FileIndexingSampleResponse] = Field(default_factory=list) + + +class RetrievalMetricsResponse(BaseModel): + scored_cases: int + skipped_cases: int + hit_rate: float + mrr: float + recall: float + context_relevance: float | None = None + + +class AnswerMetricsResponse(BaseModel): + scored_cases: int + pass_rate: float + factuality: float | None = None + rubric_score: float | None = None + + +class EvalCaseResponse(BaseModel): + query: str + retrieved_file_ids: list[str] = Field(default_factory=list) + expected_file_ids: list[str] = Field(default_factory=list) + hit: bool | None = None + reciprocal_rank: float | None = None + answer: str | None = None + answer_passed: bool | None = None + grader_reason: str | None = None + + +class EvalRunResponse(BaseModel): + """A run, with metrics once it has finished.""" + + id: str + dataset_id: str + status: str + started_at: datetime | None = None + finished_at: datetime | None = None + indexing: IndexingMetricsResponse | None = None + retrieval: RetrievalMetricsResponse | None = None + answer: AnswerMetricsResponse | None = None + cases: list[EvalCaseResponse] = Field(default_factory=list) + error: str | None = None + created_by: int | None = None + + +class EvalRunSummaryResponse(BaseModel): + """Run history row — metrics headline only, no per-case detail.""" + + id: str + dataset_id: str + status: str + started_at: datetime | None = None + finished_at: datetime | None = None + hit_rate: float | None = None + mrr: float | None = None + answer_pass_rate: float | None = None + files_per_minute: float | None = None + error: str | None = None + + +class StartRunRequest(BaseModel): + dataset_id: str + + +__all__ = [ + "AnswerMetricsResponse", + "EvalCaseResponse", + "EvalDatasetResponse", + "EvalRunResponse", + "EvalRunSummaryResponse", + "FileIndexingSampleResponse", + "IndexingMetricsResponse", + "RetrievalMetricsResponse", + "StartRunRequest", +] diff --git a/openrag/core/config/evaluation.py b/openrag/core/config/evaluation.py new file mode 100644 index 000000000..def0dbc02 --- /dev/null +++ b/openrag/core/config/evaluation.py @@ -0,0 +1,51 @@ +"""Configuration for the admin evaluation feature. + +Operational limits live here rather than as constants in the code that uses +them, so a deployment can be retuned without a rebuild. + +Domain contracts stay out of this file: the reserved partition prefix, the CSV +column names and the ``file_id`` alphabet are not settings, and changing them +would invalidate stored datasets. +""" + +from __future__ import annotations + +from .base import ConfigMixin + + +class EvaluationConfig(ConfigMixin): + """Limits and timeouts for evaluation datasets and runs.""" + + #: Executable the runner shells out to; the images install it on PATH. + promptfoo_bin: str = "promptfoo" + + #: Upload caps. A dataset is re-indexed on every run, so an oversized + #: corpus costs far more than the upload itself. + max_corpus_mb: int = 512 + max_testset_mb: int = 5 + #: Each test-set row costs one retrieval call plus one graded generation. + max_testset_rows: int = 500 + + #: Chunks retrieved per question by the retrieval config. + top_k: int = 5 + + #: How long to wait for one file's indexing task, and how often to poll it. + task_timeout_seconds: float = 1800.0 + task_poll_seconds: float = 1.0 + + #: Per-request timeout for the runner's own HTTP calls. + http_timeout_seconds: float = 300.0 + + #: promptfoo grades every row with an LLM, so allow for a slow grader. + promptfoo_timeout_seconds: float = 3600.0 + + @property + def max_corpus_bytes(self) -> int: + return self.max_corpus_mb * 1024 * 1024 + + @property + def max_testset_bytes(self) -> int: + return self.max_testset_mb * 1024 * 1024 + + +__all__ = ["EvaluationConfig"] diff --git a/openrag/core/config/infrastructure.py b/openrag/core/config/infrastructure.py index ecc4052dd..a2e41aaf9 100644 --- a/openrag/core/config/infrastructure.py +++ b/openrag/core/config/infrastructure.py @@ -98,6 +98,11 @@ class PathsConfig(ConfigMixin): class ServerConfig(ConfigMixin): preferred_url_scheme: str | None = None + # Base URL under which the API reaches itself from inside the deployment. + # Used by out-of-process workers (e.g. the evaluation runner, which uploads + # a corpus and drives promptfoo over HTTP): they run in their own container + # and cannot reuse whatever host the admin's browser happened to use. + internal_url: str = "http://openrag:8080" # --------------------------------------------------------------------------- diff --git a/openrag/core/config/loader.py b/openrag/core/config/loader.py index 0e0dcfe1c..d6f0a69f7 100644 --- a/openrag/core/config/loader.py +++ b/openrag/core/config/loader.py @@ -86,6 +86,7 @@ ("LOG_LEVEL", "verbose.level", str), # Server ("PREFERRED_URL_SCHEME", "server.preferred_url_scheme", str), + ("OPENRAG_INTERNAL_URL", "server.internal_url", str), # LLM Context ("MAX_LLM_CONTEXT_SIZE", "llm_context.max_llm_context_size", int), ("MAX_OUTPUT_TOKENS", "llm_context.max_output_tokens", int), @@ -183,6 +184,16 @@ ("OPENRAG_MCP_SIMILARITY_THRESHOLD", "mcp.similarity_threshold", float), ("OPENRAG_MCP_DOWNLOAD_TIMEOUT", "mcp.download_timeout", float), ("OPENRAG_MCP_MAX_DOWNLOAD_BYTES", "mcp.max_download_bytes", int), + # Evaluation + ("PROMPTFOO_BIN", "evaluation.promptfoo_bin", str), + ("EVAL_MAX_CORPUS_MB", "evaluation.max_corpus_mb", int), + ("EVAL_MAX_TESTSET_MB", "evaluation.max_testset_mb", int), + ("EVAL_MAX_TESTSET_ROWS", "evaluation.max_testset_rows", int), + ("EVAL_TOP_K", "evaluation.top_k", int), + ("EVAL_TASK_TIMEOUT", "evaluation.task_timeout_seconds", float), + ("EVAL_TASK_POLL_SECONDS", "evaluation.task_poll_seconds", float), + ("EVAL_HTTP_TIMEOUT", "evaluation.http_timeout_seconds", float), + ("EVAL_PROMPTFOO_TIMEOUT", "evaluation.promptfoo_timeout_seconds", float), ] _AUDIO_EXTENSIONS = ("mp3", "flac", "ogg", "aac", "flv", "wma", "mp4") diff --git a/openrag/core/config/root.py b/openrag/core/config/root.py index 73adf8e72..a8dcf67ea 100644 --- a/openrag/core/config/root.py +++ b/openrag/core/config/root.py @@ -14,6 +14,7 @@ SemaphoreConfig, VLMConfig, ) +from .evaluation import EvaluationConfig from .indexation import LoaderConfig from .infrastructure import ( PathsConfig, @@ -66,6 +67,22 @@ class Settings(ConfigMixin): rag: RAGConfig = Field(default_factory=RAGConfig) websearch: WebSearchConfig = Field(default_factory=StaanWebSearchConfig) mcp: MCPServerConfig = Field(default_factory=MCPServerConfig) + evaluation: EvaluationConfig = Field(default_factory=EvaluationConfig) models: ModelsConfig = Field(default_factory=ModelsConfig) presets: PresetsConfig = Field(default_factory=PresetsConfig) partitions: dict[str, PartitionConfig] = Field(default_factory=dict) + + def resolved_rdb(self) -> RDBConfig: + """``rdb`` with its database name filled in. + + ``rdb.database`` is optional: historically the name is derived from the + Milvus collection. Any process opening its own Postgres connection — + the API's catalog store, or a Ray worker such as ``EvalRunner`` — must + resolve it the same way, so the derivation lives here rather than in + the callers. + """ + if self.rdb.database is not None: + return self.rdb + return self.rdb.model_copy( + update={"database": f"partitions_for_collection_{self.vectordb.collection_name}"}, + ) diff --git a/openrag/core/evaluation/__init__.py b/openrag/core/evaluation/__init__.py new file mode 100644 index 000000000..840afc1f9 --- /dev/null +++ b/openrag/core/evaluation/__init__.py @@ -0,0 +1,16 @@ +"""Pure evaluation logic: test-set parsing, promptfoo config, metric math.""" + +from core.evaluation.identity import sanitize_file_id +from core.evaluation.metrics import extract_results, indexing_metrics, summarize +from core.evaluation.promptfoo_config import build_answer_config, build_retrieval_config +from core.evaluation.testset import parse_testset + +__all__ = [ + "build_answer_config", + "build_retrieval_config", + "extract_results", + "indexing_metrics", + "parse_testset", + "sanitize_file_id", + "summarize", +] diff --git a/openrag/core/evaluation/identity.py b/openrag/core/evaluation/identity.py new file mode 100644 index 000000000..a3816eb2c --- /dev/null +++ b/openrag/core/evaluation/identity.py @@ -0,0 +1,20 @@ +"""Filename to ``file_id`` normalisation, shared by the runner and the metrics. + +The indexing API accepts only ``[A-Za-z0-9._:-]`` in a ``file_id``, while a test +set names its ground truth by real filename. Both sides go through this function +so the two still match. +""" + +from __future__ import annotations + +import re + +_DISALLOWED = re.compile(r"[^A-Za-z0-9._:-]") + + +def sanitize_file_id(filename: str) -> str: + """Map a corpus filename onto an id the indexing API accepts.""" + return _DISALLOWED.sub("_", filename) + + +__all__ = ["sanitize_file_id"] diff --git a/openrag/core/evaluation/metrics.py b/openrag/core/evaluation/metrics.py new file mode 100644 index 000000000..60347eebb --- /dev/null +++ b/openrag/core/evaluation/metrics.py @@ -0,0 +1,274 @@ +"""Metric computation for an evaluation run. + +Two jobs live here, both pure: + +* aggregate the per-file indexing timings the worker collected; +* turn promptfoo's ``results.json`` into ranking and answer-quality numbers. + +The ranking definitions (hit rate, MRR, recall) follow the write-up in +``tests/load/automatic-evaluation-pipeline/README.md`` so the numbers this +page reports mean the same thing as the ones the offline pipeline produced. + +promptfoo's output envelope varies by release, so :func:`extract_results` +accepts either the ``{"results": {"results": [...]}}`` shape or a bare list, and +every field read from a row is treated as optional. +""" + +from __future__ import annotations + +import math +import statistics +from collections.abc import Iterable, Mapping, Sequence +from pathlib import Path +from typing import Any + +from core.evaluation.identity import sanitize_file_id +from core.models.evaluation import ( + AnswerMetrics, + EvalCaseResult, + EvalTestCase, + FileIndexingSample, + IndexingMetrics, + RetrievalMetrics, +) + +_BYTES_PER_MB = 1024 * 1024 + + +def _mean(values: Sequence[float]) -> float: + return float(statistics.fmean(values)) if values else 0.0 + + +def _percentile(values: Sequence[float], fraction: float) -> float: + """Nearest-rank percentile. + + ``statistics.quantiles`` needs at least two points and interpolates; + nearest-rank keeps a single-file run meaningful and always returns a + duration that was actually observed. + """ + if not values: + return 0.0 + ordered = sorted(values) + # ceil, not round: the rank is ceil(fraction * n) by definition, and round() + # breaks ties to even, selecting the wrong element on an odd integer rank. + rank = math.ceil(fraction * len(ordered)) + return float(ordered[min(max(rank, 1), len(ordered)) - 1]) + + +def indexing_metrics(samples: Sequence[FileIndexingSample], wall_seconds: float) -> IndexingMetrics: + """Aggregate per-file timings into throughput figures. + + ``wall_seconds`` is the measured end-to-end duration of the indexing + phase, which is what throughput is derived from — summing per-file + durations would overstate speed whenever files are indexed concurrently. + """ + succeeded = [s for s in samples if not s.failed] + durations = [s.duration_seconds for s in succeeded] + total_bytes = sum(s.size_bytes for s in succeeded) + + by_extension: dict[str, dict[str, float]] = {} + for sample in succeeded: + extension = (Path(sample.filename).suffix or "(none)").lower() + bucket = by_extension.setdefault(extension, {"files": 0.0, "seconds": 0.0}) + bucket["files"] += 1 + bucket["seconds"] += sample.duration_seconds + for bucket in by_extension.values(): + bucket["mean_seconds"] = round(bucket["seconds"] / bucket["files"], 3) + + return IndexingMetrics( + files_total=len(samples), + files_failed=sum(1 for s in samples if s.failed), + bytes_total=total_bytes, + wall_seconds=round(wall_seconds, 3), + files_per_minute=round(len(succeeded) / wall_seconds * 60, 2) if wall_seconds > 0 else 0.0, + megabytes_per_second=(round(total_bytes / _BYTES_PER_MB / wall_seconds, 3) if wall_seconds > 0 else 0.0), + p50_seconds=round(_percentile(durations, 0.50), 3), + p95_seconds=round(_percentile(durations, 0.95), 3), + by_extension=by_extension, + samples=list(samples), + ) + + +def extract_results(payload: Any) -> list[dict[str, Any]]: + """Pull the per-test rows out of a promptfoo output file.""" + if isinstance(payload, list): + return [row for row in payload if isinstance(row, Mapping)] + if not isinstance(payload, Mapping): + return [] + results = payload.get("results") + if isinstance(results, Mapping): + results = results.get("results") + if isinstance(results, list): + return [row for row in results if isinstance(row, Mapping)] + return [] + + +def _row_query(row: Mapping[str, Any]) -> str: + variables = row.get("vars") + if isinstance(variables, Mapping): + return str(variables.get("query", "")) + return "" + + +def _row_output(row: Mapping[str, Any]) -> Any: + response = row.get("response") + if isinstance(response, Mapping) and "output" in response: + return response["output"] + return row.get("output") + + +def _index_by_query(rows: Iterable[Mapping[str, Any]]) -> dict[str, Mapping[str, Any]]: + """Map each question to its row, keeping the first when a query repeats.""" + indexed: dict[str, Mapping[str, Any]] = {} + for row in rows: + query = _row_query(row) + if query and query not in indexed: + indexed[query] = row + return indexed + + +def _retrieved_documents(output: Any) -> list[tuple[str, set[str]]]: + """Rank-ordered ``(display_name, identifiers)`` from a ``/search`` response. + + Matching accepts either identifier a document carries, ``metadata.source`` + or ``metadata.file_id``, since a test set may name ground truth by either. + """ + if not isinstance(output, list): + return [] + documents: list[tuple[str, set[str]]] = [] + for document in output: + if not isinstance(document, Mapping): + continue + metadata = document.get("metadata") + if not isinstance(metadata, Mapping): + continue + source_name = Path(str(metadata.get("source") or "")).name + file_id = str(metadata.get("file_id") or "") + # Compare on the sanitised form: a test set naming "A B.pdf" has to + # match the "A_B.pdf" the indexer stored. + identifiers = {sanitize_file_id(value) for value in (source_name, file_id) if value} + if identifiers: + # `source` is a server-side storage path, so `file_id` is the + # name worth displaying. + documents.append((file_id or source_name, identifiers)) + return documents + + +def _grading_score(row: Mapping[str, Any], assertion_type: str | None = None) -> float | None: + """Score for a row, optionally narrowed to one assertion type.""" + grading = row.get("gradingResult") + if not isinstance(grading, Mapping): + return None + if assertion_type is None: + score = grading.get("score") + return float(score) if isinstance(score, int | float) else None + + components = grading.get("componentResults") + if not isinstance(components, list): + return None + for component in components: + if not isinstance(component, Mapping): + continue + assertion = component.get("assertion") + if isinstance(assertion, Mapping) and assertion.get("type") == assertion_type: + score = component.get("score") + if isinstance(score, int | float): + return float(score) + return None + + +def _grading_reason(row: Mapping[str, Any]) -> str | None: + grading = row.get("gradingResult") + if isinstance(grading, Mapping): + reason = grading.get("reason") + return str(reason) if reason else None + return None + + +def summarize( + *, + cases: Sequence[EvalTestCase], + retrieval_payload: Any, + answer_payload: Any, +) -> tuple[RetrievalMetrics, AnswerMetrics, list[EvalCaseResult]]: + """Fold both promptfoo outputs into metrics plus per-question detail. + + Test cases with no ``expected_file_ids`` are counted in ``skipped_cases`` + and left out of hit rate / MRR / recall — scoring them as misses would + make a sparsely-annotated test set look like a broken retriever. + """ + retrieval_rows = _index_by_query(extract_results(retrieval_payload)) + answer_rows = _index_by_query(extract_results(answer_payload)) + + hits: list[float] = [] + reciprocal_ranks: list[float] = [] + recalls: list[float] = [] + relevance_scores: list[float] = [] + answer_passes: list[float] = [] + factuality_scores: list[float] = [] + rubric_scores: list[float] = [] + details: list[EvalCaseResult] = [] + + for case in cases: + retrieval_row = retrieval_rows.get(case.query) + answer_row = answer_rows.get(case.query) + + documents = _retrieved_documents(_row_output(retrieval_row)) if retrieval_row else [] + detail = EvalCaseResult( + query=case.query, + retrieved_file_ids=[name for name, _ in documents], + expected_file_ids=list(case.expected_file_ids), + ) + + if retrieval_row is not None: + relevance = _grading_score(retrieval_row, "context-relevance") + if relevance is not None: + relevance_scores.append(relevance) + + if case.has_ground_truth_sources: + expected = {sanitize_file_id(name) for name in case.expected_file_ids} + matched = [rank for rank, (_, identifiers) in enumerate(documents, start=1) if identifiers & expected] + detail.hit = bool(matched) + detail.reciprocal_rank = 1.0 / matched[0] if matched else 0.0 + hits.append(1.0 if matched else 0.0) + reciprocal_ranks.append(detail.reciprocal_rank) + found = {name for name in expected if any(name in ids for _, ids in documents)} + recalls.append(len(found) / len(expected)) + + if answer_row is not None: + output = _row_output(answer_row) + if isinstance(output, Mapping): + output = output.get("answer") + detail.answer = str(output) if output is not None else None + detail.answer_passed = bool(answer_row.get("success")) + detail.grader_reason = _grading_reason(answer_row) + answer_passes.append(1.0 if detail.answer_passed else 0.0) + for assertion_type, sink in ( + ("factuality", factuality_scores), + ("llm-rubric", rubric_scores), + ): + score = _grading_score(answer_row, assertion_type) + if score is not None: + sink.append(score) + + details.append(detail) + + scored = len(hits) + retrieval = RetrievalMetrics( + scored_cases=scored, + skipped_cases=len(cases) - scored, + hit_rate=round(_mean(hits), 4), + mrr=round(_mean(reciprocal_ranks), 4), + recall=round(_mean(recalls), 4), + context_relevance=round(_mean(relevance_scores), 4) if relevance_scores else None, + ) + answer = AnswerMetrics( + scored_cases=len(answer_passes), + pass_rate=round(_mean(answer_passes), 4), + factuality=round(_mean(factuality_scores), 4) if factuality_scores else None, + rubric_score=round(_mean(rubric_scores), 4) if rubric_scores else None, + ) + return retrieval, answer, details + + +__all__ = ["extract_results", "indexing_metrics", "summarize"] diff --git a/openrag/core/evaluation/promptfoo_config.py b/openrag/core/evaluation/promptfoo_config.py new file mode 100644 index 000000000..bee85ed43 --- /dev/null +++ b/openrag/core/evaluation/promptfoo_config.py @@ -0,0 +1,171 @@ +"""Generation of the promptfoo configs a run executes. + +A run produces two configs rather than one, because the two questions need +different endpoints: + +* **retrieval** hits ``GET /search/partition/{partition}``, whose documents + carry the chunk ``content`` — the text that ``context-relevance`` grades and + whose ``metadata.file_id`` feeds hit rate / MRR / recall. +* **answer** hits ``POST /v1/chat/completions``, whose ``extra.sources`` carry + source metadata but no chunk text, and whose message content is what + ``factuality`` and ``llm-rubric`` grade. + +Keeping them separate means every assertion in a config applies to that +config's single provider, so no assertion ever runs against an output shape it +cannot read. + +This module is pure: it returns plain dicts. Serialisation and execution live +in the worker. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from core.models.evaluation import EvalTestCase + +#: promptfoo templates with Nunjucks; ``urlencode`` keeps a question +#: containing ``&`` or ``?`` from corrupting the search query string. +_QUERY_TEMPLATE = "{{ query | urlencode }}" + +#: Extract the ``documents`` array from the search response. +_SEARCH_TRANSFORM = "json.documents || []" + +#: ``transformResponse`` must be a single JavaScript expression — statements +#: and IIFEs are rejected — so this extracts the answer text and nothing more. +#: Retrieved sources come from the retrieval pass instead. +_CHAT_TRANSFORM = "json.choices[0].message.content" + +_RUBRIC = ( + "The response must answer the question using the retrieved documents. " + "Grade it against this reference answer: {{expected_answer}}. " + "Pass if the response conveys the same facts, even if worded differently. " + "Fail if it contradicts the reference, is empty, or refuses to answer." +) + + +def _grader(model: str, base_url: str, api_key: str | None) -> dict[str, Any]: + """The provider promptfoo uses for model-graded assertions. + + Points at OpenRAG's own OpenAI-compatible LLM endpoint so an eval needs no + third-party credentials. + """ + config: dict[str, Any] = {"apiBaseUrl": base_url} + # vLLM ignores the key but the OpenAI client refuses to send without one. + config["apiKey"] = api_key or "sk-no-key-required" + return {"id": f"openai:chat:{model}", "config": config} + + +def _tests(cases: Sequence[EvalTestCase], asserts: list[dict[str, Any]]) -> list[dict[str, Any]]: + """One promptfoo test per case, all sharing the same assertions. + + ``expected_file_ids`` is deliberately absent from ``vars``: no assertion + reads it. The ranking metrics are computed from the retrieved ids in + ``metrics.summarize``, not by promptfoo. + """ + return [ + { + "vars": {"query": case.query, "expected_answer": case.expected_answer}, + # A fresh copy per test: a shared list would serialise as a YAML + # anchor plus aliases. + "assert": [dict(assertion) for assertion in asserts], + } + for case in cases + ] + + +def build_retrieval_config( + *, + cases: Sequence[EvalTestCase], + api_base_url: str, + partition: str, + token: str, + grader_model: str, + grader_base_url: str, + grader_api_key: str | None = None, + top_k: int = 5, + relevance_threshold: float = 0.0, +) -> dict[str, Any]: + """Config that measures what the retriever returns for each question. + + ``relevance_threshold`` defaults to 0 so ``context-relevance`` records a + score without failing the run; the deterministic ranking metrics are + computed from the same responses afterwards. + """ + url = f"{api_base_url.rstrip('/')}/search/partition/{partition}?text={_QUERY_TEMPLATE}&top_k={top_k}" + return { + "description": f"OpenRAG retrieval eval ({partition})", + "prompts": ["{{query}}"], + "providers": [ + { + "id": "https", + "label": "openrag-retrieval", + "config": { + "url": url, + "method": "GET", + "headers": {"Authorization": f"Bearer {token}"}, + "transformResponse": _SEARCH_TRANSFORM, + }, + } + ], + "defaultTest": {"options": {"provider": _grader(grader_model, grader_base_url, grader_api_key)}}, + "tests": _tests( + cases, + [ + { + "type": "context-relevance", + "contextTransform": "output.map(d => d.content).join('\\n\\n')", + "threshold": relevance_threshold, + } + ], + ), + } + + +def build_answer_config( + *, + cases: Sequence[EvalTestCase], + api_base_url: str, + partition: str, + token: str, + grader_model: str, + grader_base_url: str, + grader_api_key: str | None = None, +) -> dict[str, Any]: + """Config that grades the generated answer against the expected one.""" + return { + "description": f"OpenRAG answer eval ({partition})", + "prompts": ["{{query}}"], + "providers": [ + { + "id": "https", + "label": "openrag-chat", + "config": { + "url": f"{api_base_url.rstrip('/')}/v1/chat/completions", + "method": "POST", + "headers": { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + "body": { + "model": f"openrag-{partition}", + "messages": [{"role": "user", "content": "{{query}}"}], + "stream": False, + }, + "transformResponse": _CHAT_TRANSFORM, + }, + } + ], + "defaultTest": {"options": {"provider": _grader(grader_model, grader_base_url, grader_api_key)}}, + "tests": _tests( + cases, + [ + {"type": "factuality", "value": "{{expected_answer}}"}, + {"type": "llm-rubric", "value": _RUBRIC}, + ], + ), + } + + +__all__ = ["build_answer_config", "build_retrieval_config"] diff --git a/openrag/core/evaluation/runner.py b/openrag/core/evaluation/runner.py new file mode 100644 index 000000000..e1ec48886 --- /dev/null +++ b/openrag/core/evaluation/runner.py @@ -0,0 +1,67 @@ +"""Port for handing an evaluation run to the worker layer. + +``EvaluationService`` owns the orchestration around a run — provisioning the +throwaway partition and the service-user token, enforcing one-run-at-a-time, +recording the queued row. The run *itself* (upload the corpus over HTTP, shell +out to promptfoo, fold the outputs into metrics) executes inside the +``EvalRunner`` Ray actor. + +Defining the three operations the orchestrator needs on a dedicated port keeps +it Ray-free — the Phase 9 rule that all Ray code lives under +``services/workers/`` (``docs/refactoring/REFACTORING_STRATEGY_v1.md``). The +adapter in ``services/workers/eval_dispatcher.py`` binds this to the actor; +tests bind it to a fake. + +No Ray types cross this boundary — only plain strings, mappings and bools. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + +class EvaluationRunner(ABC): + """Operations the evaluation orchestrator needs from the worker layer.""" + + @abstractmethod + async def is_busy(self) -> bool: + """Whether a run currently occupies the runner. + + Doubles as the orchestrator's liveness probe, so an implementation + must raise rather than hang when the worker cannot be reached. + """ + ... + + @abstractmethod + async def dispatch( + self, + *, + run_id: str, + partition: str, + token: str, + api_base_url: str, + corpus_dir: str, + cases: Sequence[Mapping[str, Any]], + ) -> None: + """Hand a queued run to the worker. + + Fire-and-forget: the worker owns the run from here and records its own + terminal status, so this returns as soon as the work is accepted. + """ + ... + + @abstractmethod + async def cancel(self, run_id: str) -> bool: + """Ask the worker to abandon a run. + + Returns ``False`` when no worker owns ``run_id`` — the orchestrator + reaps the orphaned row itself in that case. + """ + ... + + +__all__ = ["EvaluationRunner"] diff --git a/openrag/core/evaluation/testset.py b/openrag/core/evaluation/testset.py new file mode 100644 index 000000000..3dbd700eb --- /dev/null +++ b/openrag/core/evaluation/testset.py @@ -0,0 +1,149 @@ +"""Parsing and validation of the uploaded test-set CSV. + +The admin uploads a plain CSV; everything promptfoo needs is derived from it +so operators never have to learn promptfoo's YAML. Validation is strict and +reports the offending 1-based row numbers, because a malformed test set must +fail at upload time rather than half-way through a run that has already +indexed a corpus. +""" + +from __future__ import annotations + +import csv +import io + +from core.models.evaluation import EvalTestCase +from core.utils.exceptions import ValidationError + +QUERY_COLUMN = "question" +ANSWER_COLUMN = "expected_answer" +FILE_IDS_COLUMN = "expected_file_ids" + +REQUIRED_COLUMNS = (QUERY_COLUMN, ANSWER_COLUMN) +OPTIONAL_COLUMNS = (FILE_IDS_COLUMN,) + +#: ``expected_file_ids`` holds several ids in one cell, separated by this. +FILE_ID_SEPARATOR = ";" + +#: Row numbers are reported to the user, so cap how many we list at once. +_MAX_REPORTED_ERRORS = 10 + + +def _decode(raw: bytes) -> str: + """Decode the upload, tolerating a UTF-8 BOM from Excel exports.""" + try: + return raw.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise ValidationError( + "Test set must be UTF-8 encoded CSV.", + code="EVAL_TESTSET_ENCODING", + status_code=400, + ) from exc + + +def _split_file_ids(cell: str) -> tuple[str, ...]: + return tuple(part.strip() for part in cell.split(FILE_ID_SEPARATOR) if part.strip()) + + +def parse_testset(raw: bytes | str, *, max_rows: int) -> list[EvalTestCase]: + """Parse the CSV upload into test cases. + + Args: + raw: Raw upload bytes, or already-decoded text. + max_rows: Reject test sets longer than this (``EVAL_MAX_TESTSET_ROWS``). + Every row costs a retrieval call plus a graded generation per run, + so the cap is a deployment concern rather than a fixed limit. + + Returns: + One :class:`EvalTestCase` per data row, in file order. + + Raises: + ValidationError: On a missing/duplicated header, an empty file, a row + with a blank required cell, or more than ``max_rows`` rows. + """ + text = _decode(raw) if isinstance(raw, bytes) else raw + reader = csv.DictReader(io.StringIO(text)) + + if reader.fieldnames is None: + raise ValidationError( + "Test set is empty — expected a CSV header row.", + code="EVAL_TESTSET_EMPTY", + status_code=400, + ) + + headers = [(name or "").strip().lower() for name in reader.fieldnames] + missing = [column for column in REQUIRED_COLUMNS if column not in headers] + if missing: + raise ValidationError( + f"Test set is missing required column(s): {', '.join(missing)}. " + f"Expected header: {','.join((*REQUIRED_COLUMNS, *OPTIONAL_COLUMNS))}", + code="EVAL_TESTSET_COLUMNS", + status_code=400, + ) + if len(set(headers)) != len(headers): + raise ValidationError( + "Test set has duplicate column names.", + code="EVAL_TESTSET_COLUMNS", + status_code=400, + ) + + # DictReader keys off the raw header spelling; normalise so " Question " + # and "question" both resolve. + key_for = {column: reader.fieldnames[headers.index(column)] for column in headers if column} + + cases: list[EvalTestCase] = [] + errors: list[str] = [] + error_count = 0 + + for offset, row in enumerate(reader): + # +2: one for the header line, one to make it 1-based like a spreadsheet. + line = offset + 2 + query = (row.get(key_for[QUERY_COLUMN]) or "").strip() + expected = (row.get(key_for[ANSWER_COLUMN]) or "").strip() + + if not query and not expected: + continue # blank trailing line + if not query or not expected: + column = QUERY_COLUMN if not query else ANSWER_COLUMN + error_count += 1 + # Only the reported ones are retained; the rest are just counted. + if len(errors) < _MAX_REPORTED_ERRORS: + errors.append(f"row {line}: '{column}' is empty") + continue + + file_ids_key = key_for.get(FILE_IDS_COLUMN) + # Reject on the row that would exceed the cap, rather than + # materialising every remaining row only to count them afterwards. + if len(cases) >= max_rows: + raise ValidationError( + f"Test set has more than {max_rows} rows.", + code="EVAL_TESTSET_TOO_LARGE", + status_code=400, + ) + file_ids = _split_file_ids(row.get(file_ids_key) or "") if file_ids_key else () + cases.append(EvalTestCase(query=query, expected_answer=expected, expected_file_ids=file_ids)) + + if errors: + suffix = f" (+{error_count - len(errors)} more)" if error_count > len(errors) else "" + raise ValidationError( + "Test set has invalid rows — " + "; ".join(errors) + suffix, + code="EVAL_TESTSET_ROWS", + status_code=400, + ) + if not cases: + raise ValidationError( + "Test set contains no usable rows.", + code="EVAL_TESTSET_EMPTY", + status_code=400, + ) + return cases + + +__all__ = [ + "ANSWER_COLUMN", + "FILE_IDS_COLUMN", + "FILE_ID_SEPARATOR", + "QUERY_COLUMN", + "REQUIRED_COLUMNS", + "parse_testset", +] diff --git a/openrag/core/models/evaluation.py b/openrag/core/models/evaluation.py new file mode 100644 index 000000000..1c7382273 --- /dev/null +++ b/openrag/core/models/evaluation.py @@ -0,0 +1,172 @@ +"""Domain models for the evaluation feature. + +An *evaluation dataset* pairs a corpus (the files to index) with a test set +(the questions to ask). An *evaluation run* indexes that corpus into a +throwaway partition, replays the test set against the live API through +promptfoo, and records three families of metrics: indexing speed, retrieval +quality, and answer quality. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum + +#: Runs index their corpus into a throwaway partition named from the run id. +#: These are an implementation detail of an eval and are filtered out of the +#: partition listings so they never show up as user-facing collections. +EVAL_PARTITION_PREFIX = "__eval_" + + +def is_eval_partition(partition: str) -> bool: + """True for the throwaway partition a run creates.""" + return partition.startswith(EVAL_PARTITION_PREFIX) + + +class EvalRunStatus(str, Enum): + """Lifecycle of a single evaluation run. + + Mirrors the indexing task vocabulary (``services.workers.task_state``) so + the admin UI can reuse the same status styling. + """ + + QUEUED = "QUEUED" + INDEXING = "INDEXING" + EVALUATING = "EVALUATING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + CANCELLED = "CANCELLED" + + @property + def is_terminal(self) -> bool: + return self in (EvalRunStatus.COMPLETED, EvalRunStatus.FAILED, EvalRunStatus.CANCELLED) + + +@dataclass(frozen=True) +class EvalTestCase: + """One row of the uploaded test set CSV. + + ``expected_file_ids`` is optional: rows without it still contribute to the + answer-quality metrics, but are excluded from hit rate / MRR / recall + rather than being counted as misses. + """ + + query: str + expected_answer: str + expected_file_ids: tuple[str, ...] = () + + @property + def has_ground_truth_sources(self) -> bool: + return bool(self.expected_file_ids) + + +@dataclass +class EvalDataset: + """A stored corpus + test set pair.""" + + id: str + name: str + corpus_file_count: int + testset_row_count: int + created_at: datetime | None = None + created_by: int | None = None + + +@dataclass +class FileIndexingSample: + """Wall-clock cost of indexing one corpus file.""" + + filename: str + size_bytes: int + duration_seconds: float + failed: bool = False + + +@dataclass +class IndexingMetrics: + """Aggregate indexing speed over a run's corpus.""" + + files_total: int = 0 + files_failed: int = 0 + bytes_total: int = 0 + wall_seconds: float = 0.0 + files_per_minute: float = 0.0 + megabytes_per_second: float = 0.0 + p50_seconds: float = 0.0 + p95_seconds: float = 0.0 + by_extension: dict[str, dict[str, float]] = field(default_factory=dict) + samples: list[FileIndexingSample] = field(default_factory=list) + + +@dataclass +class RetrievalMetrics: + """Ranking quality of the retrieved chunks. + + ``scored_cases`` counts the test rows that carried ``expected_file_ids``; + ``skipped_cases`` counts those that did not. Both are reported so a + near-empty ground truth can never masquerade as a perfect score. + """ + + scored_cases: int = 0 + skipped_cases: int = 0 + hit_rate: float = 0.0 + mrr: float = 0.0 + recall: float = 0.0 + context_relevance: float | None = None + + +@dataclass +class AnswerMetrics: + """LLM-graded quality of the generated answers.""" + + scored_cases: int = 0 + pass_rate: float = 0.0 + factuality: float | None = None + rubric_score: float | None = None + + +@dataclass +class EvalCaseResult: + """Per-question detail surfaced in the run detail table.""" + + query: str + retrieved_file_ids: list[str] = field(default_factory=list) + expected_file_ids: list[str] = field(default_factory=list) + hit: bool | None = None + reciprocal_rank: float | None = None + answer: str | None = None + answer_passed: bool | None = None + grader_reason: str | None = None + + +@dataclass +class EvalRun: + """One evaluation execution against a dataset.""" + + id: str + dataset_id: str + status: EvalRunStatus = EvalRunStatus.QUEUED + started_at: datetime | None = None + finished_at: datetime | None = None + indexing: IndexingMetrics | None = None + retrieval: RetrievalMetrics | None = None + answer: AnswerMetrics | None = None + cases: list[EvalCaseResult] = field(default_factory=list) + error: str | None = None + created_by: int | None = None + + +__all__ = [ + "EVAL_PARTITION_PREFIX", + "AnswerMetrics", + "EvalCaseResult", + "EvalDataset", + "EvalRun", + "EvalRunStatus", + "EvalTestCase", + "FileIndexingSample", + "IndexingMetrics", + "RetrievalMetrics", + "is_eval_partition", +] diff --git a/openrag/core/ports/catalog_store.py b/openrag/core/ports/catalog_store.py index cf0ff6f26..d5ad778fd 100644 --- a/openrag/core/ports/catalog_store.py +++ b/openrag/core/ports/catalog_store.py @@ -13,6 +13,7 @@ from .conversation_repo import ConversationRepository from .document_repo import DocumentRepository from .entity_repo import EntityRepository +from .evaluation_repo import EvaluationRepository from .idempotency_repo import IdempotencyRepository from .job_repo import JobRepository from .model_endpoint_repo import ModelEndpointRepository @@ -70,6 +71,10 @@ def model_endpoint_repo(self) -> ModelEndpointRepository: ... @abstractmethod def preset_repo(self) -> PresetRepository: ... + @property + @abstractmethod + def evaluation_repo(self) -> EvaluationRepository: ... + @property @abstractmethod def chunk_repo(self) -> ChunkRepository: ... diff --git a/openrag/core/ports/evaluation_repo.py b/openrag/core/ports/evaluation_repo.py new file mode 100644 index 000000000..66b321592 --- /dev/null +++ b/openrag/core/ports/evaluation_repo.py @@ -0,0 +1,63 @@ +"""Port for evaluation dataset and run persistence.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from core.models.evaluation import EvalDataset, EvalRun, EvalRunStatus + + +class EvaluationRepository(ABC): + """Storage contract for evaluation datasets and runs.""" + + @abstractmethod + async def create_dataset(self, dataset: EvalDataset) -> EvalDataset: + """Persist a new dataset row.""" + + @abstractmethod + async def list_datasets(self) -> list[EvalDataset]: + """All datasets, newest first.""" + + @abstractmethod + async def get_dataset(self, dataset_id: str) -> EvalDataset | None: + """One dataset, or ``None`` when it does not exist.""" + + @abstractmethod + async def delete_dataset(self, dataset_id: str) -> bool: + """Delete a dataset. Returns ``False`` when nothing was deleted.""" + + @abstractmethod + async def create_run(self, run: EvalRun) -> EvalRun: + """Persist a queued run. + + Raises: + ConflictError: A run is already active. At most one may be, and the + store is what enforces it. + """ + + @abstractmethod + async def list_runs(self, limit: int = 50) -> list[EvalRun]: + """Recent runs, newest first.""" + + @abstractmethod + async def get_run(self, run_id: str) -> EvalRun | None: + """One run with its metrics, or ``None``.""" + + @abstractmethod + async def active_run(self) -> EvalRun | None: + """The run currently occupying the runner, if any. + + Advisory only — mutual exclusion between starts is enforced by the + store's single-active-run index, not by reading this. + """ + + @abstractmethod + async def update_run_status(self, run_id: str, status: EvalRunStatus, *, error: str | None = None) -> None: + """Move a run to a new status, stamping ``finished_at`` when terminal.""" + + @abstractmethod + async def save_run_results(self, run: EvalRun) -> None: + """Write the metric payloads and terminal status of a finished run.""" + + +__all__ = ["EvaluationRepository"] diff --git a/openrag/di/container.py b/openrag/di/container.py index 2510b5638..d5763867d 100644 --- a/openrag/di/container.py +++ b/openrag/di/container.py @@ -45,6 +45,7 @@ from core.ports.conversation_repo import ConversationRepository from core.ports.document_repo import DocumentRepository from core.ports.entity_repo import EntityRepository + from core.ports.evaluation_repo import EvaluationRepository from core.ports.idempotency_repo import IdempotencyRepository from core.ports.job_repo import JobRepository from core.ports.model_endpoint_repo import ModelEndpointRepository @@ -59,6 +60,7 @@ from core.vector_stores import VectorStore from services.orchestrators.auth_service import AuthService from services.orchestrators.conversion_service import ConversionService + from services.orchestrators.evaluation_service import EvaluationService from services.orchestrators.indexing_service import IndexingService from services.orchestrators.job_service import JobService from services.orchestrators.mcp_service import MCPService @@ -116,6 +118,7 @@ def __init__(self, settings: Settings | None = None) -> None: self._partition_service: PartitionService | None = None self._model_endpoint_service: ModelEndpointService | None = None self._preset_service: PresetService | None = None + self._evaluation_service: EvaluationService | None = None self._workspace_service: WorkspaceService | None = None self._retrieval_service: RetrievalService | None = None self._query_service: QueryService | None = None @@ -360,6 +363,10 @@ def model_endpoint_repo(self) -> ModelEndpointRepository: def preset_repo(self) -> PresetRepository: return self.catalog_store.preset_repo + @property + def evaluation_repo(self) -> EvaluationRepository: + return self.catalog_store.evaluation_repo + # ------------------------------------------------------------------ # Orchestrators (Phase 8) # ------------------------------------------------------------------ @@ -459,6 +466,25 @@ def preset_service(self) -> PresetService: ) return self._preset_service + @property + def evaluation_service(self) -> EvaluationService: + """EvaluationService — dataset storage and run dispatch.""" + if self._evaluation_service is None: + from services.orchestrators.evaluation_service import EvaluationService + from services.workers.eval_dispatcher import from_ray_namespace + + self._evaluation_service = EvaluationService( + repo=self.evaluation_repo, + # The adapter resolves its detached actor on first use, so + # building the service here does not spawn a worker. + runner=from_ray_namespace(), + user_service=self.user_service, + user_repo=self.user_repo, + partition_service=self.partition_service, + config=self._require_settings(), + ) + return self._evaluation_service + @property def workspace_service(self) -> WorkspaceService: """WorkspaceService — lazily built, cached for the container's lifetime.""" diff --git a/openrag/di/providers.py b/openrag/di/providers.py index 866cccba7..d6a54455d 100644 --- a/openrag/di/providers.py +++ b/openrag/di/providers.py @@ -149,6 +149,11 @@ def get_preset_service(request: Request = None) -> Any: return _get_optional_service(_require_initialized(request), "preset_service") +def get_evaluation_service(request: Request = None) -> Any: + """Resolve the evaluation orchestrator from the active container.""" + return _get_optional_service(_require_initialized(request), "evaluation_service") + + def get_config(request: Request = None): """Resolve application configuration from the active container.""" return _require_initialized(request).config @@ -159,6 +164,7 @@ def get_config(request: Request = None): "get_config", "get_container", "get_conversion_service", + "get_evaluation_service", "get_indexing_service", "get_job_service", "get_mcp_service", diff --git a/openrag/di/repositories.py b/openrag/di/repositories.py index c39c4b090..c53ceb819 100644 --- a/openrag/di/repositories.py +++ b/openrag/di/repositories.py @@ -36,13 +36,7 @@ def create_catalog_store( ``settings.vectordb.collection_name`` so the new adapter targets the same Postgres database the legacy actor has always used. """ - rdb = settings.rdb - if rdb.database is None: - rdb = rdb.model_copy( - update={ - "database": f"partitions_for_collection_{settings.vectordb.collection_name}", - }, - ) + rdb = settings.resolved_rdb() if run_migrations is None: run_migrations = rdb.run_migrations return PostgresStore(rdb, run_migrations=run_migrations) diff --git a/openrag/services/orchestrators/evaluation_service.py b/openrag/services/orchestrators/evaluation_service.py new file mode 100644 index 000000000..92a1c053a --- /dev/null +++ b/openrag/services/orchestrators/evaluation_service.py @@ -0,0 +1,395 @@ +"""EvaluationService — datasets on disk, runs dispatched to the worker layer. + +Setup and teardown of a run's *identity* live here rather than in the worker, +because creating users and partitions is orchestration the API layer already +owns. The worker receives a partition it may write to and a token it may use, +and nothing else about the system. + +Dispatch goes through the :class:`~core.evaluation.runner.EvaluationRunner` +port, so this orchestrator stays Ray-free; the Ray actor lives behind the +adapter in ``services/workers/eval_dispatcher.py``. + +The bearer token handed to the worker belongs to a single long-lived service +user (``__openrag_eval__``) whose token is **regenerated at the start of every +run**. That keeps exactly one non-admin service account in the database while +ensuring no usable plaintext token is ever stored at rest — the previous one +stops working the moment a new run starts. +""" + +from __future__ import annotations + +import asyncio +import shutil +import uuid +from pathlib import Path +from typing import TYPE_CHECKING + +from core.evaluation import parse_testset +from core.models.evaluation import ( + EVAL_PARTITION_PREFIX, + EvalDataset, + EvalRun, + EvalRunStatus, + EvalTestCase, + is_eval_partition, +) +from core.models.user import UserCreate +from core.utils.exceptions import ConflictError, NotFoundError, OpenRAGError, ValidationError +from core.utils.logging import get_logger + +if TYPE_CHECKING: + from collections.abc import Sequence + from typing import IO + + from core.config.root import Settings + from core.evaluation.runner import EvaluationRunner + from core.ports.evaluation_repo import EvaluationRepository + from core.ports.user_repo import UserRepository + from services.orchestrators.partition_service import PartitionService + from services.orchestrators.user_service import UserService + +logger = get_logger() + +#: Stable identity of the service account runs authenticate as. +EVAL_USER_EXTERNAL_ID = "__openrag_eval__" +EVAL_USER_DISPLAY_NAME = "OpenRAG Evaluation" + +TESTSET_FILENAME = "testset.csv" +CORPUS_DIRNAME = "corpus" + +#: Block size for streaming an upload to disk. +_COPY_CHUNK_BYTES = 1024 * 1024 + + +def eval_partition_name(run_id: str) -> str: + return f"{EVAL_PARTITION_PREFIX}{run_id}" + + +class EvaluationRunnerUnavailableError(OpenRAGError): + """The runner actor could not be reached. Maps to HTTP 503.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="EVAL_RUNNER_UNAVAILABLE", status_code=503) + + +class EvaluationService: + """Dataset storage plus run dispatch for the admin evaluation page.""" + + def __init__( + self, + *, + repo: EvaluationRepository, + runner: EvaluationRunner, + user_service: UserService, + user_repo: UserRepository, + partition_service: PartitionService, + config: Settings, + ) -> None: + self._repo = repo + self._runner = runner + self._user_service = user_service + self._user_repo = user_repo + self._partition_service = partition_service + self._config = config + self._settings = config.evaluation + self._root = Path(config.paths.data_dir) / "eval" + + # ── datasets ───────────────────────────────────────────────────── + + def _dataset_dir(self, dataset_id: str) -> Path: + return self._root / dataset_id + + async def create_dataset( + self, + *, + name: str, + corpus: Sequence[tuple[str, IO[bytes]]], + testset: IO[bytes], + user_id: int | None, + ) -> EvalDataset: + """Validate and store a corpus + test set. + + The CSV is parsed here so a malformed test set is rejected at upload + rather than after a run has already spent minutes indexing. + + Uploads arrive as open binary streams rather than ``bytes``: each is + read under a size cap and copied to disk in fixed-size blocks, so a + large corpus is never held in memory. The blocking file I/O runs on a + worker thread so it cannot stall the event loop. + """ + if not name.strip(): + raise ValidationError("Dataset name is required.", status_code=400) + if not corpus: + raise ValidationError("At least one corpus file is required.", status_code=400) + + testset_csv = await asyncio.to_thread( + self._read_capped, + testset, + self._settings.max_testset_bytes, + f"Test set exceeds the {self._settings.max_testset_mb} MB limit.", + ) + cases = parse_testset(testset_csv, max_rows=self._settings.max_testset_rows) + + dataset_id = uuid.uuid4().hex + directory = self._dataset_dir(dataset_id) + try: + written = await asyncio.to_thread(self._store_upload, directory, corpus, testset_csv) + return await self._repo.create_dataset( + EvalDataset( + id=dataset_id, + name=name.strip(), + corpus_file_count=written, + testset_row_count=len(cases), + created_by=user_id, + ) + ) + except Exception: + await asyncio.to_thread(shutil.rmtree, directory, True) + raise + + @staticmethod + def _read_capped(stream: IO[bytes], limit: int, message: str) -> bytes: + """Read a stream, refusing anything past ``limit``. + + Reads one byte beyond the cap rather than trusting a client-supplied + length, so an inflated ``Content-Length`` cannot get past it. + """ + stream.seek(0) + payload = stream.read(limit + 1) + if len(payload) > limit: + raise ValidationError(message, status_code=413) + return payload + + def _store_upload( + self, + directory: Path, + corpus: Sequence[tuple[str, IO[bytes]]], + testset_csv: bytes, + ) -> int: + """Write the corpus and test set to disk. Blocking; call in a thread.""" + corpus_dir = directory / CORPUS_DIRNAME + corpus_dir.mkdir(parents=True, exist_ok=True) + + written = 0 + budget = self._settings.max_corpus_bytes + for filename, stream in corpus: + # Flatten any path components a browser may have sent. + target = corpus_dir / Path(filename).name + if target.exists(): + raise ValidationError( + f"Corpus contains more than one file named '{target.name}'.", + status_code=400, + ) + budget -= self._copy_within_budget(stream, target, budget) + written += 1 + + (directory / TESTSET_FILENAME).write_bytes(testset_csv) + return written + + def _copy_within_budget(self, stream: IO[bytes], target: Path, budget: int) -> int: + """Copy ``stream`` into ``target``, refusing to exceed ``budget``.""" + stream.seek(0) + written = 0 + with target.open("wb") as handle: + while chunk := stream.read(_COPY_CHUNK_BYTES): + written += len(chunk) + if written > budget: + raise ValidationError( + f"Corpus exceeds the {self._settings.max_corpus_mb} MB limit.", + status_code=413, + ) + handle.write(chunk) + return written + + async def list_datasets(self) -> list[EvalDataset]: + return await self._repo.list_datasets() + + async def delete_dataset(self, dataset_id: str) -> None: + """Delete a dataset and its stored files. + + Refused while a run is using it: the runner reads the corpus from disk + for the whole indexing phase, so removing it mid-run would surface as a + confusing FileNotFoundError instead of a clear conflict. + """ + active = await self._repo.active_run() + if active is not None and active.dataset_id == dataset_id: + raise ConflictError(f"Evaluation run '{active.id}' is still using this dataset.") + + if not await self._repo.delete_dataset(dataset_id): + raise NotFoundError(f"Evaluation dataset '{dataset_id}' not found") + await asyncio.to_thread(shutil.rmtree, self._dataset_dir(dataset_id), True) + + # ── runs ───────────────────────────────────────────────────────── + + async def list_runs(self, limit: int = 50) -> list[EvalRun]: + return await self._repo.list_runs(limit) + + async def get_run(self, run_id: str) -> EvalRun: + run = await self._repo.get_run(run_id) + if run is None: + raise NotFoundError(f"Evaluation run '{run_id}' not found") + return run + + async def start_run(self, dataset_id: str, user_id: int | None) -> EvalRun: + """Provision a run's partition and token, then dispatch it. + + The run row is inserted before anything is provisioned: the partial + unique index ``ux_eval_runs_single_active`` makes that insert the mutual + exclusion between concurrent starts. A read-then-insert would let two + racing requests both regenerate the shared eval user's token, the second + revoking the credentials the first is still indexing with. + + Raises: + NotFoundError: The dataset does not exist. + ConflictError: Another run is already in flight — the runner + executes one at a time so timings stay comparable. + """ + dataset = await self._repo.get_dataset(dataset_id) + if dataset is None: + raise NotFoundError(f"Evaluation dataset '{dataset_id}' not found") + + directory = self._dataset_dir(dataset_id) + testset_path = directory / TESTSET_FILENAME + if not testset_path.exists(): + raise NotFoundError(f"Test set for dataset '{dataset_id}' is missing on disk") + cases = parse_testset(testset_path.read_bytes(), max_rows=self._settings.max_testset_rows) + + # Reach the runner before claiming the slot: dispatch is + # fire-and-forget, so an unreachable worker would otherwise strand the + # run in QUEUED with a partition and token provisioned for nobody. + await self._ping_runner() + + run_id = uuid.uuid4().hex + partition = eval_partition_name(run_id) + run = await self._repo.create_run( + EvalRun( + id=run_id, + dataset_id=dataset_id, + status=EvalRunStatus.QUEUED, + created_by=user_id, + ) + ) + + try: + eval_user_id = await self._ensure_eval_user() + token = (await self._user_service.regenerate_token(eval_user_id))["token"] + await self._partition_service.create_partition(partition, user_id=eval_user_id) + await self._dispatch(run_id, partition, token, directory, cases) + except Exception as exc: + # The run row is the lock; leaving it active would block every + # later run. + logger.exception(f"Could not start evaluation run {run_id}: {exc}") + await self._repo.update_run_status( + run_id, + EvalRunStatus.FAILED, + error=f"Could not start the run: {exc}", + ) + await self._drop_orphaned_partition(run_id) + raise + + logger.bind(run_id=run_id, dataset_id=dataset_id).info("Dispatched evaluation run") + return run + + async def _dispatch( + self, + run_id: str, + partition: str, + token: str, + directory: Path, + cases: Sequence[EvalTestCase], + ) -> None: + """Hand the run to the worker. + + Fire and forget: the worker owns the run from here and records its own + outcome. + """ + await self._runner.dispatch( + run_id=run_id, + partition=partition, + token=token, + api_base_url=self._config.server.internal_url, + corpus_dir=str(directory / CORPUS_DIRNAME), + cases=[ + { + "query": case.query, + "expected_answer": case.expected_answer, + "expected_file_ids": list(case.expected_file_ids), + } + for case in cases + ], + ) + + async def cancel_run(self, run_id: str) -> EvalRun: + """Ask the worker to stop, or reap the run if no worker owns it. + + The worker writes the terminal status for a run it is executing. When + it disowns the run — it restarted, or died before picking the run up — + nothing else would ever move that row out of an active status, and it + would block every subsequent run. Cancelling reaps it instead. + """ + run = await self.get_run(run_id) + if run.status.is_terminal: + raise ConflictError(f"Evaluation run '{run_id}' has already finished.") + + owned = False + try: + owned = await self._runner.cancel(run_id) + except Exception as exc: # noqa: BLE001 — an unreachable runner still has to be reaped + logger.warning(f"Evaluation runner unreachable while cancelling {run_id}: {exc}") + + if not owned: + await self._repo.update_run_status( + run_id, + EvalRunStatus.CANCELLED, + error="No runner owns this run — it was orphaned and has been reaped.", + ) + await self._drop_orphaned_partition(run_id) + return await self.get_run(run_id) + + async def _drop_orphaned_partition(self, run_id: str) -> None: + """Best-effort cleanup of the throwaway partition of a reaped run.""" + try: + await self._partition_service.delete_partition(eval_partition_name(run_id)) + except Exception as exc: # noqa: BLE001 — it may never have been created + logger.debug(f"No eval partition to drop for run {run_id}: {exc}") + + # ── internals ──────────────────────────────────────────────────── + + async def _ping_runner(self) -> None: + """Fail fast when the runner cannot be reached. + + Raises: + OpenRAGError: The worker is unreachable — surfaced to the caller + instead of being discovered as a run that never leaves QUEUED. + """ + try: + await self._runner.is_busy() + except Exception as exc: + logger.exception(f"Evaluation runner is unavailable: {exc}") + raise EvaluationRunnerUnavailableError(f"The evaluation runner could not be reached: {exc}") from exc + + async def _ensure_eval_user(self) -> int: + """Get-or-create the non-admin service user runs authenticate as.""" + existing = await self._user_repo.get_user_by_external_id(EVAL_USER_EXTERNAL_ID) + if existing is not None: + return int(existing.id) + created = await self._user_service.create_user( + UserCreate( + display_name=EVAL_USER_DISPLAY_NAME, + external_user_id=EVAL_USER_EXTERNAL_ID, + is_admin=False, + # A corpus is uploaded on every run, so a quota would fail the + # second one for reasons unrelated to the eval. + file_quota=-1, + ) + ) + return int(created["id"]) + + +__all__ = [ + "EVAL_PARTITION_PREFIX", + "EVAL_USER_EXTERNAL_ID", + "EvaluationService", + "eval_partition_name", + "is_eval_partition", +] diff --git a/openrag/services/orchestrators/partition_service.py b/openrag/services/orchestrators/partition_service.py index b0204a9e4..2a9776beb 100644 --- a/openrag/services/orchestrators/partition_service.py +++ b/openrag/services/orchestrators/partition_service.py @@ -34,6 +34,7 @@ from core.config.indexation_pipeline import IndexationPipelineConfig from core.config.retrieval_pipeline import RetrievalPipelineConfig from core.indexing.validators import validate_partition_name +from core.models.evaluation import is_eval_partition from core.models.preset import PartitionConfig from core.utils.conts import is_internal_metadata_key from core.utils.exceptions import ( @@ -242,7 +243,8 @@ async def _ensure_partition_for_operation(self, partition: str, *, operation: An raise PartitionNotFoundError(f"Partition '{partition}' does not exist.") async def list_partitions(self) -> list[dict]: - return await self._partition_repo.list_partitions() + rows = await self._partition_repo.list_partitions() + return [row for row in rows if not is_eval_partition(str(row.get("partition", "")))] async def file_counts_by_partition(self) -> dict[str, int]: """Return a ``{partition: document_count}`` map for all partitions (one query).""" @@ -262,6 +264,10 @@ async def list_partition_summaries(self) -> dict[str, dict]: summaries: dict[str, dict] = {} for r in rows: name = r["partition"] + # Throwaway eval partitions are hidden here as well as in + # list_partitions: this is what GET /partition/ responds from. + if is_eval_partition(str(name)): + continue created = r.get("created_at") summaries[name] = { "partition": name, diff --git a/openrag/services/persistence/evaluation_repo.py b/openrag/services/persistence/evaluation_repo.py new file mode 100644 index 000000000..0ebd4dfc2 --- /dev/null +++ b/openrag/services/persistence/evaluation_repo.py @@ -0,0 +1,210 @@ +"""asyncpg-backed :class:`EvaluationRepository`. + +Backs the ``eval_datasets`` and ``eval_runs`` tables. Metric payloads round-trip +as JSONB; the dataclasses in ``core.models.evaluation`` define their shape. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from dataclasses import asdict +from typing import TYPE_CHECKING, Any + +from core.models.evaluation import ( + AnswerMetrics, + EvalCaseResult, + EvalDataset, + EvalRun, + EvalRunStatus, + FileIndexingSample, + IndexingMetrics, + RetrievalMetrics, +) +from core.ports.evaluation_repo import EvaluationRepository +from core.utils.exceptions import ConflictError + +if TYPE_CHECKING: + import asyncpg + +_ACTIVE_STATUSES = ("QUEUED", "INDEXING", "EVALUATING") + + +def _dump(payload: Any) -> str | None: + """Serialise a metrics dataclass — or a list of them — for a JSONB column.""" + if payload is None: + return None + if isinstance(payload, list): + return json.dumps([asdict(item) for item in payload]) + return json.dumps(asdict(payload) if hasattr(payload, "__dataclass_fields__") else payload) + + +def _load(raw: Any) -> Any: + """asyncpg returns JSONB as ``str`` unless a codec is registered.""" + if raw is None: + return None + return json.loads(raw) if isinstance(raw, str | bytes) else raw + + +class PgEvaluationRepository(EvaluationRepository): + """asyncpg-backed implementation of :class:`EvaluationRepository`.""" + + def __init__(self, pool_getter: Callable[[], asyncpg.Pool]) -> None: + self._pool_getter = pool_getter + + @property + def pool(self) -> asyncpg.Pool: + return self._pool_getter() + + # ── datasets ───────────────────────────────────────────────────── + + async def create_dataset(self, dataset: EvalDataset) -> EvalDataset: + row = await self.pool.fetchrow( + """ + INSERT INTO eval_datasets (id, name, corpus_file_count, + testset_row_count, created_by) + VALUES ($1, $2, $3, $4, $5) + RETURNING * + """, + dataset.id, + dataset.name, + dataset.corpus_file_count, + dataset.testset_row_count, + dataset.created_by, + ) + return self._row_to_dataset(row) + + async def list_datasets(self) -> list[EvalDataset]: + rows = await self.pool.fetch("SELECT * FROM eval_datasets ORDER BY created_at DESC") + return [self._row_to_dataset(row) for row in rows] + + async def get_dataset(self, dataset_id: str) -> EvalDataset | None: + row = await self.pool.fetchrow("SELECT * FROM eval_datasets WHERE id = $1", dataset_id) + return self._row_to_dataset(row) if row else None + + async def delete_dataset(self, dataset_id: str) -> bool: + result = await self.pool.execute("DELETE FROM eval_datasets WHERE id = $1", dataset_id) + return result.endswith(" 1") + + # ── runs ───────────────────────────────────────────────────────── + + async def create_run(self, run: EvalRun) -> EvalRun: + import asyncpg + + try: + row = await self.pool.fetchrow( + """ + INSERT INTO eval_runs (id, dataset_id, status, created_by) + VALUES ($1, $2, $3, $4) + RETURNING * + """, + run.id, + run.dataset_id, + run.status.value, + run.created_by, + ) + except asyncpg.UniqueViolationError as exc: + # ux_eval_runs_single_active: a run is already in flight. + raise ConflictError("An evaluation run is already in progress.") from exc + return self._row_to_run(row) + + async def list_runs(self, limit: int = 50) -> list[EvalRun]: + rows = await self.pool.fetch("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT $1", limit) + return [self._row_to_run(row) for row in rows] + + async def get_run(self, run_id: str) -> EvalRun | None: + row = await self.pool.fetchrow("SELECT * FROM eval_runs WHERE id = $1", run_id) + return self._row_to_run(row) if row else None + + async def active_run(self) -> EvalRun | None: + row = await self.pool.fetchrow( + """ + SELECT * FROM eval_runs + WHERE status = ANY($1::text[]) + ORDER BY started_at DESC + LIMIT 1 + """, + list(_ACTIVE_STATUSES), + ) + return self._row_to_run(row) if row else None + + async def update_run_status(self, run_id: str, status: EvalRunStatus, *, error: str | None = None) -> None: + await self.pool.execute( + """ + UPDATE eval_runs + SET status = $2, + error = COALESCE($3, error), + finished_at = CASE WHEN $4 THEN now() ELSE finished_at END + WHERE id = $1 + """, + run_id, + status.value, + error, + status.is_terminal, + ) + + async def save_run_results(self, run: EvalRun) -> None: + await self.pool.execute( + """ + UPDATE eval_runs + SET status = $2, + indexing = $3::jsonb, + retrieval = $4::jsonb, + answer = $5::jsonb, + cases = $6::jsonb, + error = $7, + finished_at = now() + WHERE id = $1 + """, + run.id, + run.status.value, + _dump(run.indexing), + _dump(run.retrieval), + _dump(run.answer), + _dump(run.cases), + run.error, + ) + + # ── row mapping ────────────────────────────────────────────────── + + @staticmethod + def _row_to_dataset(row: Any) -> EvalDataset: + return EvalDataset( + id=row["id"], + name=row["name"], + corpus_file_count=row["corpus_file_count"], + testset_row_count=row["testset_row_count"], + created_at=row["created_at"], + created_by=row["created_by"], + ) + + @staticmethod + def _row_to_run(row: Any) -> EvalRun: + indexing = _load(row["indexing"]) + retrieval = _load(row["retrieval"]) + answer = _load(row["answer"]) + cases = _load(row["cases"]) or [] + samples = indexing.pop("samples", []) if indexing else [] + return EvalRun( + id=row["id"], + dataset_id=row["dataset_id"], + status=EvalRunStatus(row["status"]), + started_at=row["started_at"], + finished_at=row["finished_at"], + indexing=( + IndexingMetrics( + **indexing, + samples=[FileIndexingSample(**sample) for sample in samples], + ) + if indexing + else None + ), + retrieval=RetrievalMetrics(**retrieval) if retrieval else None, + answer=AnswerMetrics(**answer) if answer else None, + cases=[EvalCaseResult(**case) for case in cases], + error=row["error"], + created_by=row["created_by"], + ) + + +__all__ = ["PgEvaluationRepository"] diff --git a/openrag/services/persistence/migrations/alembic/versions/a7c9e1f2b3d4_add_evaluation_tables.py b/openrag/services/persistence/migrations/alembic/versions/a7c9e1f2b3d4_add_evaluation_tables.py new file mode 100644 index 000000000..8e344bf03 --- /dev/null +++ b/openrag/services/persistence/migrations/alembic/versions/a7c9e1f2b3d4_add_evaluation_tables.py @@ -0,0 +1,104 @@ +"""add evaluation datasets and runs + +Revision ID: a7c9e1f2b3d4 +Revises: d4e5f6a7b8c9 +Create Date: 2026-07-27 12:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from schema_helpers import index_exists, table_exists +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "a7c9e1f2b3d4" +down_revision: str | Sequence[str] | None = "d4e5f6a7b8c9" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema. + + Idempotent: ``Base.metadata.create_all()`` runs at startup, so a freshly + bootstrapped database already has these tables before alembic sees them. + """ + if not table_exists("eval_datasets"): + op.create_table( + "eval_datasets", + sa.Column("id", sa.String, primary_key=True), + sa.Column("name", sa.String, nullable=False), + sa.Column("corpus_file_count", sa.Integer, server_default="0", nullable=False), + sa.Column("testset_row_count", sa.Integer, server_default="0", nullable=False), + sa.Column( + "created_by", + sa.Integer, + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + if not table_exists("eval_runs"): + op.create_table( + "eval_runs", + sa.Column("id", sa.String, primary_key=True), + sa.Column( + "dataset_id", + sa.String, + sa.ForeignKey("eval_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("status", sa.String, server_default="QUEUED", nullable=False), + sa.Column("indexing", postgresql.JSONB, nullable=True), + sa.Column("retrieval", postgresql.JSONB, nullable=True), + sa.Column("answer", postgresql.JSONB, nullable=True), + sa.Column("cases", postgresql.JSONB, nullable=True), + sa.Column("error", sa.String, nullable=True), + sa.Column( + "created_by", + sa.Integer, + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + index=True, + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint( + "status IN ('QUEUED','INDEXING','EVALUATING','COMPLETED','FAILED','CANCELLED')", + name="ck_eval_run_status", + ), + ) + # At most one active run. Created separately from the table so a database + # already bootstrapped by create_all() picks it up too. + if not index_exists("eval_runs", "ux_eval_runs_single_active"): + op.create_index( + "ux_eval_runs_single_active", + "eval_runs", + [sa.text("(status IS NOT NULL)")], + unique=True, + postgresql_where=sa.text("status IN ('QUEUED','INDEXING','EVALUATING')"), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + if index_exists("eval_runs", "ux_eval_runs_single_active"): + op.drop_index("ux_eval_runs_single_active", table_name="eval_runs") + if table_exists("eval_runs"): + op.drop_table("eval_runs") + if table_exists("eval_datasets"): + op.drop_table("eval_datasets") diff --git a/openrag/services/persistence/schema.py b/openrag/services/persistence/schema.py index 5bd37eb9b..0afdecf5f 100644 --- a/openrag/services/persistence/schema.py +++ b/openrag/services/persistence/schema.py @@ -331,6 +331,77 @@ ) +eval_datasets = Table( + "eval_datasets", + metadata, + Column("id", String, primary_key=True), + Column("name", String, nullable=False), + Column("corpus_file_count", Integer, server_default="0", nullable=False), + Column("testset_row_count", Integer, server_default="0", nullable=False), + Column( + "created_by", + Integer, + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + Column( + "created_at", + DateTime(timezone=True), + server_default=text("now()"), + nullable=False, + ), +) + + +eval_runs = Table( + "eval_runs", + metadata, + Column("id", String, primary_key=True), + Column( + "dataset_id", + String, + ForeignKey("eval_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + Column("status", String, server_default=text("'QUEUED'"), nullable=False), + # Metric payloads, shaped by core.models.evaluation. Stored as JSONB rather + # than columns because the metric set is expected to grow. + Column("indexing", JSONB, nullable=True), + Column("retrieval", JSONB, nullable=True), + Column("answer", JSONB, nullable=True), + Column("cases", JSONB, nullable=True), + Column("error", String, nullable=True), + Column( + "created_by", + Integer, + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + Column( + "started_at", + DateTime(timezone=True), + server_default=text("now()"), + nullable=False, + index=True, + ), + Column("finished_at", DateTime(timezone=True), nullable=True), + CheckConstraint( + "status IN ('QUEUED','INDEXING','EVALUATING','COMPLETED','FAILED','CANCELLED')", + name="ck_eval_run_status", + ), + # At most one active run: starting one regenerates the shared eval user's + # token, so concurrent runs would revoke each other's credentials. Indexing + # an always-true expression lets the partial index admit a single row. + Index( + "ux_eval_runs_single_active", + text("(status IS NOT NULL)"), + unique=True, + postgresql_where=text("status IN ('QUEUED','INDEXING','EVALUATING')"), + ), +) + + __all__ = [ "metadata", "model_endpoints", @@ -343,4 +414,6 @@ "partition_memberships", "workspaces", "workspace_files", + "eval_datasets", + "eval_runs", ] diff --git a/openrag/services/storage/postgres_store.py b/openrag/services/storage/postgres_store.py index e6b0b3f79..4c22a4920 100644 --- a/openrag/services/storage/postgres_store.py +++ b/openrag/services/storage/postgres_store.py @@ -31,6 +31,7 @@ from services.persistence.conversation_repo import PgConversationRepository from services.persistence.document_repo import PgDocumentRepository from services.persistence.entity_repo import PgEntityRepository +from services.persistence.evaluation_repo import PgEvaluationRepository from services.persistence.idempotency_repo import PgIdempotencyRepository from services.persistence.job_repo import PgJobRepository from services.persistence.model_endpoint_repo import PgModelEndpointRepository @@ -51,6 +52,7 @@ from core.ports.conversation_repo import ConversationRepository from core.ports.document_repo import DocumentRepository from core.ports.entity_repo import EntityRepository + from core.ports.evaluation_repo import EvaluationRepository from core.ports.idempotency_repo import IdempotencyRepository from core.ports.job_repo import JobRepository from core.ports.model_endpoint_repo import ModelEndpointRepository @@ -96,6 +98,7 @@ def __init__(self, config: RDBConfig, *, run_migrations: bool = True) -> None: self._topic_tag_repo = PgTopicTagRepository(pool_getter) self._model_endpoint_repo = PgModelEndpointRepository(pool_getter) self._preset_repo = PgPresetRepository(pool_getter) + self._evaluation_repo = PgEvaluationRepository(pool_getter) # ------------------------------------------------------------------ # Lifecycle @@ -214,6 +217,10 @@ def model_endpoint_repo(self) -> ModelEndpointRepository: def preset_repo(self) -> PresetRepository: return self._preset_repo + @property + def evaluation_repo(self) -> EvaluationRepository: + return self._evaluation_repo + # ------------------------------------------------------------------ # Helpers # ------------------------------------------------------------------ diff --git a/openrag/services/workers/eval_dispatcher.py b/openrag/services/workers/eval_dispatcher.py new file mode 100644 index 000000000..4b2859a2d --- /dev/null +++ b/openrag/services/workers/eval_dispatcher.py @@ -0,0 +1,84 @@ +"""Ray adapter for :class:`~core.evaluation.runner.EvaluationRunner`. + +Binds the port to the ``EvalRunner`` actor and keeps every Ray concern — +actor lookup, ``.remote()`` calls, timeout and cancellation handling — on this +side of the boundary, so ``EvaluationService`` never imports Ray. + +The actor handle is resolved on first use rather than in ``__init__``: +``EvalRunner`` is a *detached* actor, so merely building this adapter must not +be what spawns it. Listing datasets should not start a worker process. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from core.evaluation.runner import EvaluationRunner +from services.workers.ray_utils import call_ray_actor_with_timeout + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + +#: Bound on the calls that are awaited (liveness probe, cancellation). +#: ``dispatch`` is fire-and-forget and so has nothing to time out. +DEFAULT_TIMEOUT = 60.0 + + +class RayEvaluationRunner(EvaluationRunner): + """``EvaluationRunner`` backed by the ``EvalRunner`` Ray actor.""" + + def __init__(self, namespace: str = "openrag", timeout: float = DEFAULT_TIMEOUT) -> None: + self._namespace = namespace + self._timeout = timeout + self._actor: Any = None + + def _handle(self) -> Any: + """Get-or-create the detached actor, memoised for the process.""" + if self._actor is None: + from services.workers.eval_runner import build_eval_runner + + self._actor = build_eval_runner(namespace=self._namespace) + return self._actor + + async def is_busy(self) -> bool: + return await call_ray_actor_with_timeout( + future=self._handle().is_busy.remote(), + timeout=self._timeout, + task_description="reaching the evaluation runner", + ) + + async def dispatch( + self, + *, + run_id: str, + partition: str, + token: str, + api_base_url: str, + corpus_dir: str, + cases: Sequence[Mapping[str, Any]], + ) -> None: + # Deliberately not awaited: the worker owns the run from here and + # records its own outcome, so the ObjectRef is dropped. + self._handle().run.remote( + run_id=run_id, + partition=partition, + token=token, + api_base_url=api_base_url, + corpus_dir=corpus_dir, + cases=[dict(case) for case in cases], + ) + + async def cancel(self, run_id: str) -> bool: + return await call_ray_actor_with_timeout( + future=self._handle().cancel.remote(run_id), + timeout=self._timeout, + task_description=f"cancelling evaluation run {run_id}", + ) + + +def from_ray_namespace(namespace: str = "openrag", timeout: float = DEFAULT_TIMEOUT) -> RayEvaluationRunner: + """Build the adapter bound to the detached ``EvalRunner`` actor.""" + return RayEvaluationRunner(namespace=namespace, timeout=timeout) + + +__all__ = ["DEFAULT_TIMEOUT", "RayEvaluationRunner", "from_ray_namespace"] diff --git a/openrag/services/workers/eval_runner.py b/openrag/services/workers/eval_runner.py new file mode 100644 index 000000000..4541e6012 --- /dev/null +++ b/openrag/services/workers/eval_runner.py @@ -0,0 +1,367 @@ +"""``EvalRunner`` — the Ray actor that executes one evaluation run. + +The runner drives OpenRAG through its own HTTP API rather than through +in-process calls, for two reasons: it is the path a real user's documents take +(so the indexing timings mean something), and it is the same surface promptfoo +itself talks to, so an eval can never pass against a code path the API does not +expose. + +Everything it needs is handed to it at dispatch time — partition, bearer token, +corpus directory, parsed test cases. It owns only the mechanical work: upload +and time each file, shell out to promptfoo twice, fold the outputs into +metrics, persist, and drop the throwaway partition on the way out. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import tempfile +import time +from pathlib import Path +from typing import Any +from urllib.parse import quote + +import ray + +#: Terminal states of an indexing task (``services.workers.task_state``). +_TERMINAL_TASK_STATES = frozenset({"COMPLETED", "FAILED", "CANCELLED"}) + +#: Only the tail of promptfoo's stderr is kept for the failure message. +_ERROR_TAIL_CHARS = 2000 + + +class EvalRunError(RuntimeError): + """A run failed in a way worth surfacing verbatim to the admin.""" + + +@ray.remote +class EvalRunner: + """Serialises evaluation runs — one at a time, by construction.""" + + def __init__(self) -> None: + from core.config import load_config + from core.utils.logging import get_logger + from services.persistence.connection import ConnectionManager + from services.persistence.evaluation_repo import PgEvaluationRepository + + self._logger = get_logger() + self._config = load_config() + self._settings = self._config.evaluation + self._connection = ConnectionManager(self._config.resolved_rdb()) + self._connection_ready = False + self._repo = PgEvaluationRepository(lambda: self._connection.pool) + self._cancelled = False + self._active_run_id: str | None = None + self._process: asyncio.subprocess.Process | None = None + + # ── lifecycle ──────────────────────────────────────────────────── + + async def _ensure_connection(self) -> None: + if not self._connection_ready: + await self._connection.initialize() + self._connection_ready = True + + def _api_client(self, api_base_url: str, token: str) -> Any: + import httpx + + return httpx.AsyncClient( + base_url=api_base_url.rstrip("/"), + headers={"Authorization": f"Bearer {token}"}, + timeout=self._settings.http_timeout_seconds, + follow_redirects=True, + ) + + async def is_busy(self) -> bool: + """Liveness probe, pinged before a run is dispatched.""" + return self._active_run_id is not None + + async def cancel(self, run_id: str) -> bool: + """Ask the in-flight run to stop at its next checkpoint.""" + if self._active_run_id != run_id: + return False + self._cancelled = True + if self._process is not None and self._process.returncode is None: + self._process.kill() + return True + + def _check_cancelled(self) -> None: + if self._cancelled: + raise asyncio.CancelledError + + # ── the run ────────────────────────────────────────────────────── + + async def run( + self, + *, + run_id: str, + partition: str, + token: str, + api_base_url: str, + corpus_dir: str, + cases: list[dict[str, Any]], + ) -> None: + """Execute a full run, persisting its outcome. + + Never raises: every failure is recorded on the run row, because the + caller dispatched this fire-and-forget and has nobody to catch for. + """ + from core.models.evaluation import EvalRun, EvalRunStatus, EvalTestCase + + await self._ensure_connection() + self._cancelled = False + self._active_run_id = run_id + log = self._logger.bind(run_id=run_id, partition=partition) + + test_cases = [ + EvalTestCase( + query=case["query"], + expected_answer=case["expected_answer"], + expected_file_ids=tuple(case.get("expected_file_ids") or ()), + ) + for case in cases + ] + run = EvalRun(id=run_id, dataset_id="", status=EvalRunStatus.QUEUED) + + try: + async with self._api_client(api_base_url, token) as client: + await self._repo.update_run_status(run_id, EvalRunStatus.INDEXING) + run.indexing = await self._index_corpus(client, partition, Path(corpus_dir)) + log.info( + f"Indexed {run.indexing.files_total} file(s) in " + f"{run.indexing.wall_seconds}s ({run.indexing.files_per_minute}/min)" + ) + + self._check_cancelled() + await self._repo.update_run_status(run_id, EvalRunStatus.EVALUATING) + retrieval_payload, answer_payload = await self._run_promptfoo( + cases=test_cases, + partition=partition, + token=token, + api_base_url=api_base_url, + ) + + from core.evaluation import summarize + + run.retrieval, run.answer, run.cases = summarize( + cases=test_cases, + retrieval_payload=retrieval_payload, + answer_payload=answer_payload, + ) + run.status = EvalRunStatus.COMPLETED + await self._repo.save_run_results(run) + log.info("Evaluation run completed") + + except asyncio.CancelledError: + run.status = EvalRunStatus.CANCELLED + run.error = "Run cancelled." + await self._repo.save_run_results(run) + log.info("Evaluation run cancelled") + except Exception as exc: # noqa: BLE001 — recorded, not swallowed + run.status = EvalRunStatus.FAILED + run.error = str(exc)[:_ERROR_TAIL_CHARS] + await self._repo.save_run_results(run) + log.exception(f"Evaluation run failed: {exc}") + finally: + self._active_run_id = None + self._process = None + await self._drop_partition(api_base_url, token, partition) + + # ── indexing phase ─────────────────────────────────────────────── + + async def _index_corpus(self, client: Any, partition: str, corpus_dir: Path) -> Any: + """Upload every corpus file, timing each one end to end.""" + from core.evaluation import indexing_metrics, sanitize_file_id + from core.models.evaluation import FileIndexingSample + + files = sorted(path for path in corpus_dir.iterdir() if path.is_file()) + if not files: + raise EvalRunError("Dataset corpus is empty — nothing to index.") + + samples: list[FileIndexingSample] = [] + started = time.perf_counter() + + for path in files: + self._check_cancelled() + file_started = time.perf_counter() + failed = False + try: + await self._index_one(client, partition, sanitize_file_id(path.name), path) + except Exception as exc: # noqa: BLE001 — one bad file must not void the run + failed = True + self._logger.warning(f"Eval corpus file '{path.name}' failed to index: {exc}") + samples.append( + FileIndexingSample( + filename=path.name, + size_bytes=path.stat().st_size, + duration_seconds=round(time.perf_counter() - file_started, 3), + failed=failed, + ) + ) + + metrics = indexing_metrics(samples, time.perf_counter() - started) + if metrics.files_failed == metrics.files_total: + raise EvalRunError("Every corpus file failed to index — check the indexer logs.") + return metrics + + async def _index_one(self, client: Any, partition: str, file_id: str, path: Path) -> None: + """Upload one file and wait for its indexing task to settle.""" + with path.open("rb") as handle: + response = await client.post( + f"/indexer/partition/{partition}/file/{quote(file_id, safe='')}", + files={"file": (path.name, handle)}, + ) + if response.status_code >= 400: + raise EvalRunError(f"Upload of '{path.name}' failed: {response.status_code} {response.text[:200]}") + + status_url = response.json().get("task_status_url") + if not status_url: + raise EvalRunError(f"Upload of '{path.name}' returned no task URL.") + await self._await_task(client, status_url, path.name) + + async def _await_task(self, client: Any, status_url: str, label: str) -> None: + deadline = time.monotonic() + self._settings.task_timeout_seconds + while True: + self._check_cancelled() + response = await client.get(status_url) + state = response.json().get("task_state") if response.status_code < 400 else None + if state in _TERMINAL_TASK_STATES: + if state != "COMPLETED": + raise EvalRunError(f"Indexing of '{label}' ended as {state}.") + return + if time.monotonic() > deadline: + raise EvalRunError(f"Indexing of '{label}' timed out.") + await asyncio.sleep(self._settings.task_poll_seconds) + + # ── promptfoo phase ────────────────────────────────────────────── + + async def _run_promptfoo( + self, + *, + cases: list[Any], + partition: str, + token: str, + api_base_url: str, + ) -> tuple[Any, Any]: + """Render both configs, run them, and return the parsed outputs.""" + import yaml + from core.evaluation import build_answer_config, build_retrieval_config + + grader = self._config.llm + shared = { + "cases": cases, + "api_base_url": api_base_url, + "partition": partition, + "token": token, + "grader_model": grader.model, + "grader_base_url": grader.base_url, + "grader_api_key": grader.api_key, + } + configs = { + "retrieval": build_retrieval_config(**shared, top_k=self._settings.top_k), + "answer": build_answer_config(**shared), + } + + outputs: dict[str, Any] = {} + with tempfile.TemporaryDirectory(prefix="openrag-eval-") as workdir: + root = Path(workdir) + # promptfoo keeps a SQLite eval history under its config dir, + # defaulting to $HOME/.promptfoo. A per-run directory guarantees it + # is writable and never contended. + config_dir = root / "promptfoo-home" + config_dir.mkdir() + for name, config in configs.items(): + self._check_cancelled() + config_path = root / f"{name}.yaml" + output_path = root / f"{name}-results.json" + config_path.write_text(yaml.safe_dump(config, sort_keys=False, allow_unicode=True), encoding="utf-8") + await self._exec_promptfoo(config_path, output_path, config_dir) + outputs[name] = json.loads(output_path.read_text(encoding="utf-8")) + + return outputs["retrieval"], outputs["answer"] + + async def _exec_promptfoo(self, config_path: Path, output_path: Path, config_dir: Path) -> None: + binary = self._settings.promptfoo_bin + env = { + **os.environ, + "PROMPTFOO_DISABLE_TELEMETRY": "1", + "PROMPTFOO_DISABLE_UPDATE": "1", + # Results are persisted on the run row; the local history would + # only grow unbounded. + "PROMPTFOO_DISABLE_SHARING": "1", + "PROMPTFOO_CONFIG_DIR": str(config_dir), + # WAL mode is unsupported on some filesystems. + "PROMPTFOO_DISABLE_WAL_MODE": "true", + } + try: + self._process = await asyncio.create_subprocess_exec( + binary, + "eval", + "--config", + str(config_path), + "--output", + str(output_path), + "--no-progress-bar", + "--no-cache", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + except FileNotFoundError as exc: + raise EvalRunError( + f"promptfoo executable '{binary}' not found. It ships in the Ray image; " + "set PROMPTFOO_BIN if it lives elsewhere." + ) from exc + + try: + stdout, stderr = await asyncio.wait_for( + self._process.communicate(), timeout=self._settings.promptfoo_timeout_seconds + ) + except TimeoutError as exc: + self._process.kill() + raise EvalRunError("promptfoo timed out.") from exc + + returncode = self._process.returncode + self._process = None + self._check_cancelled() + + # promptfoo exits non-zero when assertions fail, which is a result, not + # an error — the output file is what decides. + if not output_path.exists(): + # Both streams: promptfoo reports config errors on stdout. + detail = "\n".join( + part + for part in ( + (stdout or b"").decode("utf-8", "replace").strip(), + (stderr or b"").decode("utf-8", "replace").strip(), + ) + if part + )[-_ERROR_TAIL_CHARS:] + raise EvalRunError(f"promptfoo produced no output (exit {returncode}): {detail or '(no output)'}") + + # ── teardown ───────────────────────────────────────────────────── + + async def _drop_partition(self, api_base_url: str, token: str, partition: str) -> None: + """Delete the throwaway partition, logging rather than raising.""" + try: + async with self._api_client(api_base_url, token) as client: + response = await client.delete(f"/partition/{partition}") + if response.status_code >= 400: + self._logger.warning(f"Could not drop eval partition '{partition}': {response.status_code}") + except Exception as exc: # noqa: BLE001 — teardown must not mask the run's outcome + self._logger.warning(f"Could not drop eval partition '{partition}': {exc}") + + +def build_eval_runner(namespace: str = "openrag") -> Any: + """Get-or-create the detached, single-instance runner actor.""" + return EvalRunner.options( # type: ignore[attr-defined] + name="EvalRunner", + namespace=namespace, + get_if_exists=True, + lifetime="detached", + max_concurrency=4, # run() holds a slot; cancel()/is_busy() must still land + ).remote() + + +__all__ = ["EvalRunError", "EvalRunner", "build_eval_runner"] diff --git a/tests/evaluation/README.md b/tests/evaluation/README.md new file mode 100644 index 000000000..e59598df4 --- /dev/null +++ b/tests/evaluation/README.md @@ -0,0 +1,49 @@ +# Sample evaluation dataset + +`rag_dataset_sample.csv` is a ready-made test set for the admin **System → +Evaluation** tab: 11 questions over 6 documents, every answer checked against +the source PDF rather than generated. + +The corpus itself is not committed — the documents are third-party PDFs +totalling ~36 MB. `corpus.txt` lists the 24 filenames, drawn from the internal +`rag_dataset` collection (French public-sector, agricultural, medical and AI +documents). + +## Why 24 documents for 11 questions + +Only 6 documents are the subject of a question. The other 18 are deliberate +distractors, each topically adjacent to a question's source — other AI-policy +papers, other gut/neuro medical papers, other agricultural press releases. A +corpus where every document is on a different subject makes retrieval look +better than it is: any half-working retriever scores a perfect hit rate when +there is only one candidate per topic. + +## Assembling it + +```bash +mkdir -p /tmp/eval-corpus +while IFS= read -r f; do cp "/rag_dataset/$f" /tmp/eval-corpus/; done \ + < tests/evaluation/corpus.txt +``` + +Then upload it, either through the Evaluation tab or the API: + +```bash +args=(); for f in /tmp/eval-corpus/*; do args+=(-F "corpus=@${f}"); done +curl -X POST "$OPENRAG_URL/evaluation/datasets" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -F "name=rag_dataset sample" \ + -F "testset=@tests/evaluation/rag_dataset_sample.csv" \ + "${args[@]}" +``` + +## Test set format + +`question,expected_answer,expected_file_ids` — `expected_file_ids` is optional +and semicolon-separated. Name the files as they appear on disk; the indexer +sanitises the id it stores (spaces are not valid in a `file_id`), and the +ranking metrics match against the original filename in the chunk metadata. + +Rows without `expected_file_ids` still count toward answer quality, but are +reported as `skipped_cases` in hit rate / MRR / recall rather than scored as +misses. diff --git a/tests/evaluation/corpus.txt b/tests/evaluation/corpus.txt new file mode 100644 index 000000000..e8dc27e33 --- /dev/null +++ b/tests/evaluation/corpus.txt @@ -0,0 +1,24 @@ +2017-10-30-DP-sauvons-le-colza-francais.pdf +2018-01-18-CP-red-2-energie-renouvelable-la-contribution-agricole-largement-reconnue.pdf +2019-12-03-DP-la-semence-certifiee-de-soja.pdf +2024-04-17-upcycling.pdf +202407_charte-de-deontologie-AFCL.pdf +20240930_Note_LINAGORA_IA_OpenSource_Universelle.pdf +20241121_CP_ OSPX24_LUCIE-V2.pdf +579_Urban-immunization-toolkit_final-1563547313.pdf +9789240049130-eng.pdf +Antiinflammatoire SCFA acetate propionate.pdf +Competition_in_cloud_sector.pdf +GAVI_use_case_Sample_draft.pdf +IA Ethique.pdf +Intelligence-artificielle-01-2024.pdf +Kynurenine pathway ALS.pdf +Leaky gut in systemic infammation.pdf +Lignes-directrices-nouvelle-version-2024-10.pdf +Make_France_AI_Powerhouse.pdf +Manifeste-CannabiSante-PrincipesActifs.pdf +Note Syndrome de Prader-Willi V2 Jan 22.pdf +Note de positionnement Pensons Patients.pdf +Note_aux_Operateurs_432_08_12_23.pdf +Nouveau_Bigster_Dacia_en_grand.pdf +ReST meets ReAct.pdf diff --git a/tests/evaluation/rag_dataset_sample.csv b/tests/evaluation/rag_dataset_sample.csv new file mode 100644 index 000000000..6f1ed3b7b --- /dev/null +++ b/tests/evaluation/rag_dataset_sample.csv @@ -0,0 +1,12 @@ +question,expected_answer,expected_file_ids +Quand la communauté OpenLLM France a-t-elle été créée et à l'initiative de quelle entreprise ?,En juin 2023 à l'impulsion de LINAGORA.,20241121_CP_ OSPX24_LUCIE-V2.pdf +Quand le pré-entraînement du modèle LUCIE a-t-il démarré ?,Dès décembre 2023.,20241121_CP_ OSPX24_LUCIE-V2.pdf +À quelle date se tient le Paris Open Source AI Summit ?,Le 22 janvier 2025.,20241121_CP_ OSPX24_LUCIE-V2.pdf +Quelle est la fréquence à la naissance du syndrome de Prader-Willi ?,Environ une naissance sur 21 000.,Note Syndrome de Prader-Willi V2 Jan 22.pdf +Sur quel chromosome se situe l'anomalie génétique à l'origine du syndrome de Prader-Willi ?,Le chromosome 15.,Note Syndrome de Prader-Willi V2 Jan 22.pdf +Qui a décrit le syndrome de Prader-Willi et en quelle année ?,"En 1956, par trois médecins suisses : Andrea Prader, Alexis Labhart et Heinrich Willi.",Note Syndrome de Prader-Willi V2 Jan 22.pdf +Quel pourcentage de biocarburants dans les transports le secteur agricole européen a-t-il permis d'atteindre selon le communiqué sur la directive RED II ?,"7,7 % de biocarburants dans les transports.",2018-01-18-CP-red-2-energie-renouvelable-la-contribution-agricole-largement-reconnue.pdf +Quelle obligation d'énergie renouvelable dans les transports le Parlement européen a-t-il retenue dans la directive RED II ?,Une obligation de 12 % d'énergie renouvelable dans les transports.,2018-01-18-CP-red-2-energie-renouvelable-la-contribution-agricole-largement-reconnue.pdf +Combien de morts par an les maladies cardioneurovasculaires causent-elles en France ?,Environ 140 000 morts par an.,Note de positionnement Pensons Patients.pdf +À partir de quelle taille de population le président d'un établissement public de coopération intercommunale est-il visé par les incompatibilités de la charte de déontologie du conseil en lobbying ?,Plus de 100 000 habitants.,202407_charte-de-deontologie-AFCL.pdf +Who is Tony in the GAVI use case scenario and what is his role?,"Tony is a 35-year-old community health worker in a remote rural village in Africa, managing a local health facility that is the first point of contact for village residents.",GAVI_use_case_Sample_draft.pdf diff --git a/tests/unit/core/config/test_resolved_rdb.py b/tests/unit/core/config/test_resolved_rdb.py new file mode 100644 index 000000000..5ef4ad220 --- /dev/null +++ b/tests/unit/core/config/test_resolved_rdb.py @@ -0,0 +1,42 @@ +"""Tests for Settings.resolved_rdb(). + +``rdb.database`` is optional in config; every process that opens its own +Postgres connection has to derive the same name. A Ray worker that skipped +this derivation died in its constructor with "RDBConfig.database is required", +which surfaced only as a run stuck in QUEUED — hence the coverage. +""" + +from __future__ import annotations + +from core.config.infrastructure import RDBConfig, VectorDBConfig +from core.config.root import Settings + + +def _settings(**rdb_fields) -> Settings: + return Settings( + rdb=RDBConfig(**rdb_fields), + vectordb=VectorDBConfig(collection_name="my_collection"), + ) + + +def test_derives_the_database_name_from_the_collection_when_unset(): + assert _settings(database=None).resolved_rdb().database == "partitions_for_collection_my_collection" + + +def test_keeps_an_explicit_database_name(): + assert _settings(database="explicit_db").resolved_rdb().database == "explicit_db" + + +def test_does_not_mutate_the_original_config(): + settings = _settings(database=None) + + settings.resolved_rdb() + + assert settings.rdb.database is None + + +def test_preserves_the_other_connection_fields(): + resolved = _settings(database=None, host="db.internal", port=6543).resolved_rdb() + + assert resolved.host == "db.internal" + assert resolved.port == 6543 diff --git a/tests/unit/core/evaluation/test_metrics.py b/tests/unit/core/evaluation/test_metrics.py new file mode 100644 index 000000000..5a960c0ec --- /dev/null +++ b/tests/unit/core/evaluation/test_metrics.py @@ -0,0 +1,293 @@ +"""Tests for indexing aggregation and promptfoo result folding.""" + +from __future__ import annotations + +from core.evaluation.metrics import extract_results, indexing_metrics, summarize +from core.models.evaluation import EvalTestCase, FileIndexingSample + + +def _sample(name: str, seconds: float, size: int = 1024, failed: bool = False): + return FileIndexingSample(filename=name, size_bytes=size, duration_seconds=seconds, failed=failed) + + +def _retrieval_row(query: str, file_ids: list[str], score: float | None = None): + row = { + "vars": {"query": query}, + "response": { + "output": [ + {"content": f"chunk from {fid}", "metadata": {"file_id": fid, "source": fid}} for fid in file_ids + ] + }, + } + if score is not None: + row["gradingResult"] = { + "score": score, + "componentResults": [ + {"score": score, "assertion": {"type": "context-relevance"}}, + ], + } + return row + + +def _answer_row(query: str, answer: str, success: bool, factuality: float = 1.0): + return { + "vars": {"query": query}, + "response": {"output": {"answer": answer, "sources": []}}, + "success": success, + "gradingResult": { + "pass": success, + "reason": "graded", + "componentResults": [ + {"score": factuality, "assertion": {"type": "factuality"}}, + {"score": 0.5, "assertion": {"type": "llm-rubric"}}, + ], + }, + } + + +# ── indexing ───────────────────────────────────────────────────────── + + +def test_throughput_uses_wall_clock_not_summed_durations(): + """Files may be indexed concurrently, so summing per-file durations would + overstate throughput.""" + metrics = indexing_metrics([_sample("a.pdf", 4.0), _sample("b.pdf", 4.0)], wall_seconds=4.0) + assert metrics.files_per_minute == 30.0 + + +def test_failed_files_are_counted_but_excluded_from_throughput(): + metrics = indexing_metrics([_sample("a.pdf", 2.0), _sample("b.pdf", 0.0, failed=True)], wall_seconds=2.0) + assert metrics.files_total == 2 + assert metrics.files_failed == 1 + assert metrics.files_per_minute == 30.0 + + +def test_percentiles_on_a_single_file_return_that_file(): + metrics = indexing_metrics([_sample("a.pdf", 3.0)], wall_seconds=3.0) + assert metrics.p50_seconds == 3.0 + assert metrics.p95_seconds == 3.0 + + +def test_p50_of_an_even_sample_takes_the_lower_middle(): + """Nearest-rank p50 is ceil(n/2); rounding half-to-even would report the + slower file for even n whose half is odd.""" + metrics = indexing_metrics([_sample("a.pdf", 1.0), _sample("b.pdf", 10.0)], wall_seconds=11.0) + assert metrics.p50_seconds == 1.0 + assert metrics.p95_seconds == 10.0 + + +def test_p50_ignores_files_that_failed_to_index(): + metrics = indexing_metrics( + [_sample("a.pdf", 5.0), _sample("b.pdf", 0.0, failed=True)], + wall_seconds=5.0, + ) + assert metrics.p50_seconds == 5.0 + + +def test_zero_wall_time_does_not_divide_by_zero(): + metrics = indexing_metrics([_sample("a.pdf", 0.0)], wall_seconds=0.0) + assert metrics.files_per_minute == 0.0 + assert metrics.megabytes_per_second == 0.0 + + +def test_breakdown_is_grouped_by_lowercased_extension(): + metrics = indexing_metrics( + [_sample("a.PDF", 2.0), _sample("b.pdf", 4.0), _sample("c.txt", 1.0)], + wall_seconds=7.0, + ) + assert metrics.by_extension[".pdf"]["files"] == 2 + assert metrics.by_extension[".pdf"]["mean_seconds"] == 3.0 + assert metrics.by_extension[".txt"]["files"] == 1 + + +# ── promptfoo envelope ─────────────────────────────────────────────── + + +def test_extract_results_accepts_the_nested_v3_envelope(): + rows = extract_results({"results": {"version": 3, "results": [{"vars": {}}]}}) + assert len(rows) == 1 + + +def test_extract_results_accepts_a_bare_list(): + assert len(extract_results([{"vars": {}}, {"vars": {}}])) == 2 + + +def test_extract_results_tolerates_an_unexpected_shape(): + assert extract_results({"unexpected": True}) == [] + assert extract_results(None) == [] + + +# ── summarize ──────────────────────────────────────────────────────── + + +def test_ranking_metrics_use_the_first_matching_rank(): + cases = [EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("gold.pdf",))] + retrieval, _, details = summarize( + cases=cases, + retrieval_payload=[_retrieval_row("q1", ["noise.pdf", "gold.pdf"])], + answer_payload=[], + ) + assert retrieval.hit_rate == 1.0 + assert retrieval.mrr == 0.5 + assert retrieval.recall == 1.0 + assert details[0].reciprocal_rank == 0.5 + + +def test_a_miss_scores_zero_across_the_ranking_metrics(): + cases = [EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("gold.pdf",))] + retrieval, _, details = summarize( + cases=cases, + retrieval_payload=[_retrieval_row("q1", ["noise.pdf"])], + answer_payload=[], + ) + assert retrieval.hit_rate == 0.0 + assert retrieval.mrr == 0.0 + assert details[0].hit is False + + +def test_cases_without_ground_truth_sources_are_skipped_not_failed(): + """A sparsely-annotated test set must not read as a broken retriever.""" + cases = [ + EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("gold.pdf",)), + EvalTestCase(query="q2", expected_answer="b"), + ] + retrieval, _, _ = summarize( + cases=cases, + retrieval_payload=[ + _retrieval_row("q1", ["gold.pdf"]), + _retrieval_row("q2", ["whatever.pdf"]), + ], + answer_payload=[], + ) + assert retrieval.scored_cases == 1 + assert retrieval.skipped_cases == 1 + assert retrieval.hit_rate == 1.0 + + +def test_recall_is_the_fraction_of_expected_sources_found(): + cases = [EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("a.pdf", "b.pdf"))] + retrieval, _, _ = summarize( + cases=cases, + retrieval_payload=[_retrieval_row("q1", ["a.pdf", "z.pdf"])], + answer_payload=[], + ) + assert retrieval.recall == 0.5 + + +def test_context_relevance_is_averaged_when_present(): + cases = [ + EvalTestCase(query="q1", expected_answer="a"), + EvalTestCase(query="q2", expected_answer="b"), + ] + retrieval, _, _ = summarize( + cases=cases, + retrieval_payload=[ + _retrieval_row("q1", ["a.pdf"], score=1.0), + _retrieval_row("q2", ["b.pdf"], score=0.0), + ], + answer_payload=[], + ) + assert retrieval.context_relevance == 0.5 + + +def test_context_relevance_is_none_when_no_row_carried_a_grade(): + cases = [EvalTestCase(query="q1", expected_answer="a")] + retrieval, _, _ = summarize(cases=cases, retrieval_payload=[_retrieval_row("q1", ["a.pdf"])], answer_payload=[]) + assert retrieval.context_relevance is None + + +def test_answer_metrics_average_pass_rate_and_component_scores(): + cases = [ + EvalTestCase(query="q1", expected_answer="a"), + EvalTestCase(query="q2", expected_answer="b"), + ] + _, answer, details = summarize( + cases=cases, + retrieval_payload=[], + answer_payload=[ + _answer_row("q1", "correct", True, factuality=1.0), + _answer_row("q2", "wrong", False, factuality=0.0), + ], + ) + assert answer.scored_cases == 2 + assert answer.pass_rate == 0.5 + assert answer.factuality == 0.5 + assert answer.rubric_score == 0.5 + assert details[0].answer == "correct" + assert details[1].answer_passed is False + assert details[0].grader_reason == "graded" + + +def test_missing_rows_leave_the_case_unscored_rather_than_crashing(): + """promptfoo can drop a row on a provider error; the run still reports.""" + cases = [EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("gold.pdf",))] + retrieval, answer, details = summarize(cases=cases, retrieval_payload=[], answer_payload=[]) + assert retrieval.scored_cases == 1 + assert retrieval.hit_rate == 0.0 + assert answer.scored_cases == 0 + assert details[0].answer is None + + +def test_ground_truth_matches_the_original_filename_when_the_file_id_was_sanitised(): + """The indexer rewrites 'A B.pdf' to 'A_B.pdf' because the API rejects + spaces in a file_id, while a test set names the real file.""" + cases = [EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("A B.pdf",))] + row = { + "vars": {"query": "q1"}, + "response": {"output": [{"content": "chunk", "metadata": {"file_id": "A_B.pdf", "source": "/data/A B.pdf"}}]}, + } + retrieval, _, details = summarize(cases=cases, retrieval_payload=[row], answer_payload=[]) + + assert retrieval.hit_rate == 1.0 + assert retrieval.recall == 1.0 + # Display uses the file_id — `source` is a server-side storage path. + assert details[0].retrieved_file_ids == ["A_B.pdf"] + + +def test_ground_truth_still_matches_a_sanitised_file_id_directly(): + """Authors who wrote the sanitised id are not punished for it.""" + cases = [EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("A_B.pdf",))] + row = { + "vars": {"query": "q1"}, + "response": {"output": [{"content": "chunk", "metadata": {"file_id": "A_B.pdf", "source": "/data/A B.pdf"}}]}, + } + retrieval, _, _ = summarize(cases=cases, retrieval_payload=[row], answer_payload=[]) + + assert retrieval.hit_rate == 1.0 + + +def test_matching_survives_file_id_sanitisation_on_either_side(): + """Whichever form the metadata carries, and whichever the author wrote, + must match: otherwise the ranking metrics silently read as zero.""" + cases = [EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("A B.pdf",))] + for metadata in ( + {"file_id": "A_B.pdf", "source": "A_B.pdf"}, + {"file_id": "A_B.pdf", "source": "/data/A B.pdf"}, + {"file_id": "A_B.pdf"}, + ): + row = {"vars": {"query": "q1"}, "response": {"output": [{"content": "c", "metadata": metadata}]}} + retrieval, _, _ = summarize(cases=cases, retrieval_payload=[row], answer_payload=[]) + assert retrieval.hit_rate == 1.0, metadata + + +def test_retrieved_documents_are_named_by_file_id_not_the_storage_path(): + """metadata.source is a server-side storage path, which is meaningless in + the per-question table.""" + cases = [EvalTestCase(query="q1", expected_answer="a", expected_file_ids=("report.pdf",))] + row = { + "vars": {"query": "q1"}, + "response": { + "output": [ + { + "content": "c", + "metadata": { + "file_id": "report.pdf", + "source": "/data/1700000000000_ab12_report.pdf", + }, + } + ] + }, + } + _, _, details = summarize(cases=cases, retrieval_payload=[row], answer_payload=[]) + + assert details[0].retrieved_file_ids == ["report.pdf"] diff --git a/tests/unit/core/evaluation/test_promptfoo_config.py b/tests/unit/core/evaluation/test_promptfoo_config.py new file mode 100644 index 000000000..8fabf4480 --- /dev/null +++ b/tests/unit/core/evaluation/test_promptfoo_config.py @@ -0,0 +1,99 @@ +"""Tests for the generated promptfoo configs.""" + +from __future__ import annotations + +from core.evaluation.promptfoo_config import build_answer_config, build_retrieval_config +from core.models.evaluation import EvalTestCase + +CASES = [ + EvalTestCase(query="What is the refund window?", expected_answer="30 days", expected_file_ids=("p.pdf",)), + EvalTestCase(query="Who approves?", expected_answer="The CFO"), +] + +COMMON = { + "api_base_url": "http://openrag:8080/", + "partition": "__eval_abc", + "token": "or-secret", + "grader_model": "qwen", + "grader_base_url": "http://vllm:8000/v1", +} + + +def test_retrieval_provider_targets_the_single_partition_search_route(): + config = build_retrieval_config(cases=CASES, **COMMON, top_k=7) + url = config["providers"][0]["config"]["url"] + assert url.startswith("http://openrag:8080/search/partition/__eval_abc") + assert "top_k=7" in url + + +def test_retrieval_query_is_url_encoded(): + """A question containing '&' would otherwise truncate the query string.""" + config = build_retrieval_config(cases=CASES, **COMMON) + assert "{{ query | urlencode }}" in config["providers"][0]["config"]["url"] + + +def test_retrieval_asserts_on_the_chunk_text(): + config = build_retrieval_config(cases=CASES, **COMMON) + assertion = config["tests"][0]["assert"][0] + assert assertion["type"] == "context-relevance" + assert "d.content" in assertion["contextTransform"] + + +def test_answer_provider_posts_to_the_partition_scoped_model(): + config = build_answer_config(cases=CASES, **COMMON) + body = config["providers"][0]["config"]["body"] + assert config["providers"][0]["config"]["url"] == "http://openrag:8080/v1/chat/completions" + assert body["model"] == "openrag-__eval_abc" + assert body["stream"] is False + + +def test_answer_transform_is_a_single_expression(): + """promptfoo evaluates transformResponse as an expression — a statement or + an IIFE fails at runtime with a transform error, which manifests as every + answer scoring zero.""" + transform = build_answer_config(cases=CASES, **COMMON)["providers"][0]["config"]["transformResponse"] + assert transform == "json.choices[0].message.content" + assert "return" not in transform + assert ";" not in transform + + +def test_answer_grades_against_the_expected_answer(): + config = build_answer_config(cases=CASES, **COMMON) + types = [assertion["type"] for assertion in config["tests"][0]["assert"]] + assert types == ["factuality", "llm-rubric"] + assert config["tests"][0]["assert"][0]["value"] == "{{expected_answer}}" + + +def test_both_configs_send_the_bearer_token(): + for config in ( + build_retrieval_config(cases=CASES, **COMMON), + build_answer_config(cases=CASES, **COMMON), + ): + headers = config["providers"][0]["config"]["headers"] + assert headers["Authorization"] == "Bearer or-secret" + + +def test_grader_points_at_the_configured_openrag_llm(): + """Model-graded assertions must not silently fall back to OpenAI.""" + config = build_answer_config(cases=CASES, **COMMON) + grader = config["defaultTest"]["options"]["provider"] + assert grader["id"] == "openai:chat:qwen" + assert grader["config"]["apiBaseUrl"] == "http://vllm:8000/v1" + assert grader["config"]["apiKey"] + + +def test_every_case_becomes_a_test_with_its_vars(): + """Only the vars an assertion actually templates are emitted — the ranking + metrics read expected_file_ids from the test set, not from promptfoo.""" + config = build_retrieval_config(cases=CASES, **COMMON) + assert len(config["tests"]) == 2 + assert config["tests"][0]["vars"] == { + "query": "What is the refund window?", + "expected_answer": "30 days", + } + + +def test_assertions_are_not_shared_between_tests(): + """A shared list would serialise as a YAML anchor plus aliases.""" + tests = build_answer_config(cases=CASES, **COMMON)["tests"] + assert tests[0]["assert"][0] is not tests[1]["assert"][0] diff --git a/tests/unit/core/evaluation/test_testset.py b/tests/unit/core/evaluation/test_testset.py new file mode 100644 index 000000000..5caeedad9 --- /dev/null +++ b/tests/unit/core/evaluation/test_testset.py @@ -0,0 +1,93 @@ +"""Tests for the evaluation test-set CSV parser.""" + +from __future__ import annotations + +import pytest +from core.evaluation.testset import parse_testset +from core.utils.exceptions import ValidationError + +#: The cap is deployment config (EVAL_MAX_TESTSET_ROWS); these tests pin +#: their own so they stay independent of the shipped default. +MAX_ROWS = 500 + +VALID = ( + "question,expected_answer,expected_file_ids\n" + "What is the refund window?,30 days,policy.pdf\n" + "Who approves large spend?,The CFO,finance.pdf;approvals.pdf\n" +) + + +def test_parses_rows_and_splits_file_ids(): + cases = parse_testset(VALID, max_rows=MAX_ROWS) + assert [case.query for case in cases] == [ + "What is the refund window?", + "Who approves large spend?", + ] + assert cases[0].expected_file_ids == ("policy.pdf",) + assert cases[1].expected_file_ids == ("finance.pdf", "approvals.pdf") + + +def test_expected_file_ids_column_is_optional(): + """Answer-quality-only test sets are legitimate — the ranking metrics + just report them as skipped.""" + cases = parse_testset("question,expected_answer\nWhy?,Because\n", max_rows=MAX_ROWS) + assert cases[0].expected_file_ids == () + assert cases[0].has_ground_truth_sources is False + + +def test_accepts_bytes_with_utf8_bom(): + """Excel writes a BOM; decoding with plain utf-8 would corrupt the first + header and make the required-column check fail.""" + cases = parse_testset(VALID.encode("utf-8-sig"), max_rows=MAX_ROWS) + assert len(cases) == 2 + + +def test_header_case_and_whitespace_are_normalised(): + cases = parse_testset(" Question , Expected_Answer \nWhy?,Because\n", max_rows=MAX_ROWS) + assert cases[0].query == "Why?" + assert cases[0].expected_answer == "Because" + + +def test_blank_trailing_lines_are_ignored(): + cases = parse_testset(VALID + ",\n\n", max_rows=MAX_ROWS) + assert len(cases) == 2 + + +def test_missing_required_column_is_rejected(): + with pytest.raises(ValidationError) as excinfo: + parse_testset("question,answer\nWhy?,Because\n", max_rows=MAX_ROWS) + assert "expected_answer" in str(excinfo.value) + + +def test_empty_required_cell_reports_the_spreadsheet_row_number(): + """Row 3 = second data row, counting the header as row 1.""" + with pytest.raises(ValidationError) as excinfo: + parse_testset("question,expected_answer\nWhy?,Because\n,Orphan answer\n", max_rows=MAX_ROWS) + assert "row 3" in str(excinfo.value) + + +def test_empty_file_is_rejected(): + with pytest.raises(ValidationError): + parse_testset("", max_rows=MAX_ROWS) + + +def test_header_only_file_is_rejected(): + with pytest.raises(ValidationError): + parse_testset("question,expected_answer\n", max_rows=MAX_ROWS) + + +def test_duplicate_columns_are_rejected(): + with pytest.raises(ValidationError): + parse_testset("question,question,expected_answer\na,b,c\n", max_rows=MAX_ROWS) + + +def test_row_cap_is_enforced(): + rows = "".join(f"q{i},a{i}\n" for i in range(MAX_ROWS + 1)) + with pytest.raises(ValidationError) as excinfo: + parse_testset("question,expected_answer\n" + rows, max_rows=MAX_ROWS) + assert str(MAX_ROWS) in str(excinfo.value) + + +def test_invalid_encoding_is_rejected(): + with pytest.raises(ValidationError): + parse_testset(b"\xff\xfe\x00question", max_rows=MAX_ROWS) diff --git a/tests/unit/di/test_container.py b/tests/unit/di/test_container.py index 3ae184bcf..805eaaf1f 100644 --- a/tests/unit/di/test_container.py +++ b/tests/unit/di/test_container.py @@ -347,6 +347,7 @@ def test_does_not_mutate_input_settings(self): ("job_service", "get_job_service"), ("conversion_service", "get_conversion_service"), ("mcp_service", "get_mcp_service"), + ("evaluation_service", "get_evaluation_service"), ] _OPTIONAL_PHASE_PROVIDERS = {"get_model_endpoint_service", "get_preset_service"} diff --git a/tests/unit/services/orchestrators/test_evaluation_service.py b/tests/unit/services/orchestrators/test_evaluation_service.py new file mode 100644 index 000000000..e25c6d416 --- /dev/null +++ b/tests/unit/services/orchestrators/test_evaluation_service.py @@ -0,0 +1,336 @@ +"""Tests for EvaluationService run dispatch and cancellation. + +Both behaviours here were written after a real deployment produced a run that +sat in QUEUED forever: the runner actor had died in its constructor, dispatch +is fire-and-forget so nothing noticed, and cancelling could not clear the row +because no actor claimed it — which blocked every later run. +""" + +from __future__ import annotations + +import pytest +from core.evaluation.runner import EvaluationRunner +from core.models.evaluation import EvalDataset, EvalRun, EvalRunStatus +from core.utils.exceptions import ConflictError +from services.orchestrators.evaluation_service import ( + EvaluationRunnerUnavailableError, + EvaluationService, +) + +DATASET_ID = "ds1" + + +class FakeRepo: + def __init__(self, run: EvalRun | None = None) -> None: + self.deleted_datasets: list[str] = [] + self.dataset = EvalDataset(id=DATASET_ID, name="d", corpus_file_count=1, testset_row_count=1) + self.run = run + self.status_updates: list[tuple[str, EvalRunStatus, str | None]] = [] + + async def get_dataset(self, dataset_id): + return self.dataset if dataset_id == DATASET_ID else None + + async def create_run(self, run): + self.run = run + return run + + async def get_run(self, run_id): + return self.run + + async def active_run(self): + if self.run is not None and not self.run.status.is_terminal: + return self.run + return None + + async def delete_dataset(self, dataset_id): + self.deleted_datasets.append(dataset_id) + return True + + async def update_run_status(self, run_id, status, *, error=None): + self.status_updates.append((run_id, status, error)) + if self.run is not None: + self.run.status = status + self.run.error = error + + +class FakeRunner(EvaluationRunner): + """In-memory ``EvaluationRunner`` — no Ray, no actor, no worker process.""" + + def __init__(self, *, busy_error: Exception | None = None, owns: bool = True) -> None: + self._busy_error = busy_error + self._owns = owns + self.dispatched: dict | None = None + + async def is_busy(self) -> bool: + if self._busy_error: + raise self._busy_error + return False + + async def dispatch(self, **kwargs) -> None: + self.dispatched = kwargs + + async def cancel(self, run_id: str) -> bool: + return self._owns + + +class FakePartitionService: + def __init__(self, create_error: Exception | None = None) -> None: + self.deleted: list[str] = [] + self.created: list[str] = [] + self._create_error = create_error + + async def delete_partition(self, partition): + self.deleted.append(partition) + + async def create_partition(self, partition, user_id=None): + if self._create_error: + raise self._create_error + self.created.append(partition) + + +class FakeUserRepo: + """Serves only what the ``UserRepository`` port declares.""" + + def __init__(self, user=None) -> None: + self.user = user + + async def get_user_by_external_id(self, external_id): + return self.user + + +class FakeUserService: + """Counts token regeneration — the side effect the run lock protects.""" + + def __init__(self) -> None: + self.regenerated = 0 + + async def regenerate_token(self, user_id): + self.regenerated += 1 + return {"token": "or-testtoken"} + + +def _service( + repo, + runner, + partition_service=None, + tmp_path=None, + settings=None, + user_repo=None, + user_service=None, +): + from core.config.root import Settings + + settings = settings or Settings() + if tmp_path is not None: + settings = settings.model_copy(update={"paths": settings.paths.model_copy(update={"data_dir": str(tmp_path)})}) + return EvaluationService( + repo=repo, + runner=runner, + user_service=user_service or FakeUserService(), + user_repo=user_repo if user_repo is not None else FakeUserRepo(), + partition_service=partition_service or FakePartitionService(), + config=settings, + ) + + +@pytest.mark.asyncio +async def test_start_run_refuses_when_the_runner_cannot_be_reached(tmp_path): + """A dead actor must surface as an error, not as a run stuck in QUEUED.""" + dataset_dir = tmp_path / "eval" / DATASET_ID + (dataset_dir / "corpus").mkdir(parents=True) + (dataset_dir / "testset.csv").write_text("question,expected_answer\nq,a\n", encoding="utf-8") + + repo = FakeRepo() + runner = FakeRunner(busy_error=RuntimeError("actor died in __init__")) + service = _service(repo, runner, tmp_path=tmp_path) + + with pytest.raises(EvaluationRunnerUnavailableError): + await service.start_run(DATASET_ID, user_id=1) + + assert runner.dispatched is None + # Nothing was provisioned and no run row was left behind. + assert repo.run is None + + +@pytest.mark.asyncio +async def test_dispatch_uses_the_configured_internal_url(tmp_path): + """The worker's API base URL comes from Settings, not from the environment.""" + from core.config.root import Settings + from core.models.user import User + + dataset_dir = tmp_path / "eval" / DATASET_ID + (dataset_dir / "corpus").mkdir(parents=True) + (dataset_dir / "testset.csv").write_text("question,expected_answer\nq,a\n", encoding="utf-8") + + settings = Settings() + settings = settings.model_copy( + update={"server": settings.server.model_copy(update={"internal_url": "http://api.internal:9000"})} + ) + runner = FakeRunner() + service = _service( + FakeRepo(), + runner, + tmp_path=tmp_path, + settings=settings, + # The eval service user already exists, so it is resolved through the + # port's ``get_user_by_external_id`` rather than being created. + user_repo=FakeUserRepo(User(id=7, external_user_id="__openrag_eval__")), + ) + + await service.start_run(DATASET_ID, user_id=1) + + assert runner.dispatched is not None + assert runner.dispatched["api_base_url"] == "http://api.internal:9000" + assert runner.dispatched["cases"] == [{"query": "q", "expected_answer": "a", "expected_file_ids": []}] + + +@pytest.mark.asyncio +async def test_cancel_reaps_a_run_no_runner_owns(): + """Otherwise the orphaned row blocks every subsequent run forever.""" + run = EvalRun(id="r1", dataset_id=DATASET_ID, status=EvalRunStatus.QUEUED) + repo = FakeRepo(run) + partitions = FakePartitionService() + service = _service(repo, FakeRunner(owns=False), partition_service=partitions) + + result = await service.cancel_run("r1") + + assert result.status is EvalRunStatus.CANCELLED + assert "orphaned" in (result.error or "") + assert partitions.deleted == ["__eval_r1"] + + +@pytest.mark.asyncio +async def test_cancel_leaves_an_owned_run_for_the_worker_to_finalise(): + """The worker writes its own terminal status, including the metrics.""" + run = EvalRun(id="r1", dataset_id=DATASET_ID, status=EvalRunStatus.EVALUATING) + repo = FakeRepo(run) + service = _service(repo, FakeRunner(owns=True)) + + await service.cancel_run("r1") + + assert repo.status_updates == [] + + +@pytest.mark.asyncio +async def test_cancel_rejects_an_already_finished_run(): + run = EvalRun(id="r1", dataset_id=DATASET_ID, status=EvalRunStatus.COMPLETED) + service = _service(FakeRepo(run), FakeRunner()) + + with pytest.raises(ConflictError): + await service.cancel_run("r1") + + +def _dataset_on_disk(tmp_path): + dataset_dir = tmp_path / "eval" / DATASET_ID + (dataset_dir / "corpus").mkdir(parents=True) + (dataset_dir / "testset.csv").write_text("question,expected_answer\nq,a\n", encoding="utf-8") + + +@pytest.mark.asyncio +async def test_a_second_start_is_refused_before_the_token_is_regenerated(tmp_path): + """The run row is the lock, so the 409 has to land before provisioning. + + Regenerating the shared eval user's token is what makes a lost race + destructive: it revokes the credentials the in-flight run is indexing with. + """ + from core.utils.exceptions import ConflictError + + _dataset_on_disk(tmp_path) + + class BusyRepo(FakeRepo): + async def create_run(self, run): + raise ConflictError("An evaluation run is already in progress.") + + repo = BusyRepo() + runner = FakeRunner() + users = FakeUserService() + partitions = FakePartitionService() + service = _service(repo, runner, partition_service=partitions, tmp_path=tmp_path, user_service=users) + + with pytest.raises(ConflictError): + await service.start_run(DATASET_ID, user_id=1) + + assert users.regenerated == 0, "the loser must not touch the in-flight run's token" + assert partitions.created == [] + assert runner.dispatched is None + + +@pytest.mark.asyncio +async def test_a_failed_provision_releases_the_run_lock(tmp_path): + """A run left in an active status would block every later run.""" + _dataset_on_disk(tmp_path) + + from core.models.user import User + + repo = FakeRepo() + runner = FakeRunner() + partitions = FakePartitionService(create_error=RuntimeError("milvus unreachable")) + service = _service( + repo, + runner, + partition_service=partitions, + tmp_path=tmp_path, + user_repo=FakeUserRepo(User(id=7, external_user_id="__openrag_eval__")), + ) + + with pytest.raises(RuntimeError): + await service.start_run(DATASET_ID, user_id=1) + + assert runner.dispatched is None + statuses = [status for _, status, _ in repo.status_updates] + assert EvalRunStatus.FAILED in statuses, "the lock must be released" + assert partitions.deleted, "the throwaway partition must not leak" + + +@pytest.mark.asyncio +async def test_deleting_a_dataset_in_use_is_refused(tmp_path): + """The runner reads the corpus off disk for the whole indexing phase, so + removing it mid-run would surface as a FileNotFoundError.""" + from core.utils.exceptions import ConflictError + + _dataset_on_disk(tmp_path) + run = EvalRun(id="run-1", dataset_id=DATASET_ID, status=EvalRunStatus.INDEXING) + repo = FakeRepo(run=run) + service = _service(repo, FakeRunner(), tmp_path=tmp_path) + + with pytest.raises(ConflictError): + await service.delete_dataset(DATASET_ID) + + assert repo.deleted_datasets == [] + assert (tmp_path / "eval" / DATASET_ID).exists(), "files must survive a refused delete" + + +@pytest.mark.asyncio +async def test_deleting_a_dataset_an_idle_run_used_is_allowed(tmp_path): + """Only an *active* run blocks deletion; history keeps its results.""" + _dataset_on_disk(tmp_path) + run = EvalRun(id="run-1", dataset_id=DATASET_ID, status=EvalRunStatus.COMPLETED) + repo = FakeRepo(run=run) + service = _service(repo, FakeRunner(), tmp_path=tmp_path) + + await service.delete_dataset(DATASET_ID) + + assert repo.deleted_datasets == [DATASET_ID] + assert not (tmp_path / "eval" / DATASET_ID).exists() + + +@pytest.mark.asyncio +async def test_an_oversized_test_set_is_rejected_without_buffering_it_all(tmp_path): + """The stream is read to one byte past the cap, not to its end.""" + import io + + from core.utils.exceptions import ValidationError + + service = _service(FakeRepo(), FakeRunner(), tmp_path=tmp_path) + cap = service._settings.max_testset_bytes + oversized = io.BytesIO(b"x" * (cap + 5000)) + + with pytest.raises(ValidationError) as excinfo: + await service.create_dataset( + name="d", + corpus=[("a.txt", io.BytesIO(b"hello"))], + testset=oversized, + user_id=1, + ) + assert excinfo.value.status_code == 413 + assert oversized.tell() <= cap + 1, "must stop reading once the cap is exceeded" diff --git a/tests/unit/services/orchestrators/test_partition_preset_resolution.py b/tests/unit/services/orchestrators/test_partition_preset_resolution.py index 8f156292e..1c0e68427 100644 --- a/tests/unit/services/orchestrators/test_partition_preset_resolution.py +++ b/tests/unit/services/orchestrators/test_partition_preset_resolution.py @@ -53,6 +53,9 @@ async def get_partition_row(self, name: str) -> dict | None: async def list_partition_rows(self) -> list[dict]: return list(self._store.values()) + async def list_partitions(self) -> list[dict]: + return list(self._store.values()) + async def update_partition(self, name: str, **fields) -> dict | None: self.calls.append(("update_partition", (name,))) row = self._store.get(name) @@ -474,6 +477,28 @@ async def test_list_partition_summaries_has_counts_and_no_pipelines(): assert "retrieval_pipeline" not in summaries["p1"] +@pytest.mark.asyncio +async def test_list_partition_summaries_hides_throwaway_eval_partitions(): + """GET /partition/ responds from here, so an orphaned __eval_ would + otherwise surface as a user-facing collection.""" + repo = _FakePartitionRepo(rows=[_full_row("p1"), _full_row("__eval_deadbeef")]) + svc = _make_service(repo) + + summaries = await svc.list_partition_summaries() + + assert set(summaries) == {"p1"} + + +@pytest.mark.asyncio +async def test_list_partitions_hides_throwaway_eval_partitions(): + repo = _FakePartitionRepo(rows=[_full_row("p1"), _full_row("__eval_deadbeef")]) + svc = _make_service(repo) + + names = [row["partition"] for row in await svc.list_partitions()] + + assert names == ["p1"] + + @pytest.mark.asyncio async def test_get_partition_config_missing_raises_404(): from core.utils.exceptions import PartitionNotFoundError diff --git a/ui/src/components/shared/status-badge.tsx b/ui/src/components/shared/status-badge.tsx index a50446737..20f9bc274 100644 --- a/ui/src/components/shared/status-badge.tsx +++ b/ui/src/components/shared/status-badge.tsx @@ -5,6 +5,7 @@ const statusStyles: Record = { PROCESSING: "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200", RUNNING: "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200", INDEXING: "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200", + EVALUATING: "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200", // OpenRag indexing-task states (TaskStateManager) SERIALIZING: "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200", CANCELLED: "bg-gray-100 text-gray-800 dark:bg-gray-800 dark:text-gray-200", diff --git a/ui/src/lib/api/evaluation.ts b/ui/src/lib/api/evaluation.ts new file mode 100644 index 000000000..adebf7259 --- /dev/null +++ b/ui/src/lib/api/evaluation.ts @@ -0,0 +1,146 @@ +import { request } from "./client"; + +// Admin evaluation endpoints: +// GET /evaluation/datasets → EvalDataset[] +// POST /evaluation/datasets → EvalDataset (multipart) +// DELETE /evaluation/datasets/{id} → 204 +// GET /evaluation/runs → EvalRunSummary[] +// POST /evaluation/runs → EvalRun (202, 409 when one is in flight) +// GET /evaluation/runs/{id} → EvalRun +// POST /evaluation/runs/{id}/cancel → EvalRun + +export interface EvalDataset { + id: string; + name: string; + corpus_file_count: number; + testset_row_count: number; + created_at: string | null; + created_by: number | null; +} + +export type EvalRunStatus = + | "QUEUED" + | "INDEXING" + | "EVALUATING" + | "COMPLETED" + | "FAILED" + | "CANCELLED"; + +export const ACTIVE_RUN_STATUSES: EvalRunStatus[] = ["QUEUED", "INDEXING", "EVALUATING"]; + +export function isActiveStatus(status: EvalRunStatus): boolean { + return ACTIVE_RUN_STATUSES.includes(status); +} + +/** Refetch cadence while a run can still change, shared by both eval views. */ +export const EVAL_POLL_MS = 3000; + +export interface FileIndexingSample { + filename: string; + size_bytes: number; + duration_seconds: number; + failed: boolean; +} + +export interface IndexingMetrics { + files_total: number; + files_failed: number; + bytes_total: number; + wall_seconds: number; + files_per_minute: number; + megabytes_per_second: number; + p50_seconds: number; + p95_seconds: number; + by_extension: Record>; + samples: FileIndexingSample[]; +} + +export interface RetrievalMetrics { + scored_cases: number; + skipped_cases: number; + hit_rate: number; + mrr: number; + recall: number; + context_relevance: number | null; +} + +export interface AnswerMetrics { + scored_cases: number; + pass_rate: number; + factuality: number | null; + rubric_score: number | null; +} + +export interface EvalCase { + query: string; + retrieved_file_ids: string[]; + expected_file_ids: string[]; + hit: boolean | null; + reciprocal_rank: number | null; + answer: string | null; + answer_passed: boolean | null; + grader_reason: string | null; +} + +export interface EvalRun { + id: string; + dataset_id: string; + status: EvalRunStatus; + started_at: string | null; + finished_at: string | null; + indexing: IndexingMetrics | null; + retrieval: RetrievalMetrics | null; + answer: AnswerMetrics | null; + cases: EvalCase[]; + error: string | null; + created_by: number | null; +} + +export interface EvalRunSummary { + id: string; + dataset_id: string; + status: EvalRunStatus; + started_at: string | null; + finished_at: string | null; + hit_rate: number | null; + mrr: number | null; + answer_pass_rate: number | null; + files_per_minute: number | null; + error: string | null; +} + +export function listEvalDatasets() { + return request("/evaluation/datasets"); +} + +export function createEvalDataset(name: string, testset: File, corpus: File[]) { + const body = new FormData(); + body.append("name", name); + body.append("testset", testset); + // FastAPI reads repeated parts as `list[UploadFile]`. + corpus.forEach((file) => body.append("corpus", file)); + return request("/evaluation/datasets", { method: "POST", body }); +} + +export function deleteEvalDataset(id: string) { + return request(`/evaluation/datasets/${encodeURIComponent(id)}`, { method: "DELETE" }); +} + +export function listEvalRuns(limit = 50) { + return request(`/evaluation/runs?limit=${limit}`); +} + +export function startEvalRun(datasetId: string) { + return request("/evaluation/runs", { + method: "POST", + body: JSON.stringify({ dataset_id: datasetId }), + }); +} + +export function getEvalRun(id: string) { + return request(`/evaluation/runs/${encodeURIComponent(id)}`); +} + +export function cancelEvalRun(id: string) { + return request(`/evaluation/runs/${encodeURIComponent(id)}/cancel`, { method: "POST" }); +} diff --git a/ui/src/pages/admin/evaluation/dataset-card.tsx b/ui/src/pages/admin/evaluation/dataset-card.tsx new file mode 100644 index 000000000..852a6d551 --- /dev/null +++ b/ui/src/pages/admin/evaluation/dataset-card.tsx @@ -0,0 +1,222 @@ +import { useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { Play, Trash2, Upload } from "lucide-react"; +import { + createEvalDataset, + deleteEvalDataset, + listEvalDatasets, + startEvalRun, +} from "@/lib/api/evaluation"; +import { ConfirmDialog } from "@/components/shared/confirm-dialog"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; + +const CSV_HINT = "question,expected_answer,expected_file_ids"; + +export function DatasetCard({ runActive }: { runActive: boolean }) { + const queryClient = useQueryClient(); + const [uploadOpen, setUploadOpen] = useState(false); + + const { data, isLoading } = useQuery({ queryKey: ["eval-datasets"], queryFn: listEvalDatasets }); + + const startMut = useMutation({ + mutationFn: (datasetId: string) => startEvalRun(datasetId), + onSuccess: () => { + toast.success("Evaluation run queued"); + queryClient.invalidateQueries({ queryKey: ["eval-runs"] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const deleteMut = useMutation({ + mutationFn: (id: string) => deleteEvalDataset(id), + onSuccess: () => { + toast.success("Dataset deleted"); + queryClient.invalidateQueries({ queryKey: ["eval-datasets"] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const datasets = data ?? []; + + return ( + + +
+ Datasets + + A corpus to index plus a CSV test set ({CSV_HINT}). + +
+ + + + {isLoading ? ( + + ) : datasets.length === 0 ? ( +

+ No datasets yet. Upload a corpus and a test set to run an evaluation. +

+ ) : ( + + + + Name + Files + Questions + + + + + {datasets.map((dataset) => ( + + {dataset.name} + {dataset.corpus_file_count} + {dataset.testset_row_count} + + + deleteMut.mutate(dataset.id)} + > + + + + + ))} + +
+ )} + + + + + ); +} + +function UploadDialog({ + open, + onOpenChange, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const queryClient = useQueryClient(); + const [name, setName] = useState(""); + const [testset, setTestset] = useState(null); + const [corpus, setCorpus] = useState([]); + + const reset = () => { + setName(""); + setTestset(null); + setCorpus([]); + }; + + const createMut = useMutation({ + mutationFn: () => createEvalDataset(name, testset as File, corpus), + onSuccess: (dataset) => { + toast.success(`Dataset "${dataset.name}" created (${dataset.testset_row_count} questions)`); + queryClient.invalidateQueries({ queryKey: ["eval-datasets"] }); + reset(); + onOpenChange(false); + }, + // The API validates the CSV row by row; surface its message verbatim. + onError: (e) => toast.error((e as Error).message), + }); + + const canSubmit = name.trim() !== "" && testset !== null && corpus.length > 0; + + // Every dismissal path — backdrop, Escape, Cancel — clears the form, so a + // reopened dialog never silently resubmits the previous selection. + const close = () => { + reset(); + onOpenChange(false); + }; + + return ( + (next ? onOpenChange(true) : close())}> + + + New evaluation dataset + + The corpus is re-indexed on every run, which is what the indexing-speed numbers measure. + + + +
+
+ + setName(e.target.value)} + placeholder="Support docs — July" + /> +
+
+ + setTestset(e.target.files?.[0] ?? null)} + /> +

+ Header: {CSV_HINT}. expected_file_ids is optional and + semicolon-separated; rows without it are excluded from hit rate, MRR and recall. +

+
+
+ + setCorpus(Array.from(e.target.files ?? []))} + /> + {corpus.length > 0 && ( +

{corpus.length} file(s) selected

+ )} +
+
+ + + + + + + + ); +} diff --git a/ui/src/pages/admin/evaluation/index.test.tsx b/ui/src/pages/admin/evaluation/index.test.tsx new file mode 100644 index 000000000..cee1299c7 --- /dev/null +++ b/ui/src/pages/admin/evaluation/index.test.tsx @@ -0,0 +1,212 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + cancelEvalRun, + getEvalRun, + listEvalDatasets, + listEvalRuns, + startEvalRun, + type EvalRun, + type EvalRunSummary, +} from "@/lib/api/evaluation"; +import { EvaluationTab } from "./index"; + +vi.mock("sonner", () => ({ + toast: { success: vi.fn(), error: vi.fn() }, +})); + +vi.mock("@/lib/api/evaluation", async () => { + const actual = await vi.importActual("@/lib/api/evaluation"); + return { + ...actual, + listEvalDatasets: vi.fn(), + listEvalRuns: vi.fn(), + getEvalRun: vi.fn(), + startEvalRun: vi.fn(), + cancelEvalRun: vi.fn(), + deleteEvalDataset: vi.fn(), + createEvalDataset: vi.fn(), + }; +}); + +const listDatasetsMock = vi.mocked(listEvalDatasets); +const listRunsMock = vi.mocked(listEvalRuns); +const getRunMock = vi.mocked(getEvalRun); +const startRunMock = vi.mocked(startEvalRun); +const cancelRunMock = vi.mocked(cancelEvalRun); + +const DATASET = { + id: "ds1", + name: "Support docs", + corpus_file_count: 3, + testset_row_count: 12, + created_at: null, + created_by: 1, +}; + +const COMPLETED_RUN: EvalRunSummary = { + id: "run-completed", + dataset_id: "ds1", + status: "COMPLETED", + started_at: "2026-07-27T10:00:00Z", + finished_at: "2026-07-27T10:05:00Z", + hit_rate: 0.75, + mrr: 0.5, + answer_pass_rate: 1, + files_per_minute: 12.5, + error: null, +}; + +const RUN_DETAIL: EvalRun = { + id: "run-completed", + dataset_id: "ds1", + status: "COMPLETED", + started_at: "2026-07-27T10:00:00Z", + finished_at: "2026-07-27T10:05:00Z", + indexing: { + files_total: 3, + files_failed: 0, + bytes_total: 3 * 1024 * 1024, + wall_seconds: 14.4, + files_per_minute: 12.5, + megabytes_per_second: 0.21, + p50_seconds: 4.5, + p95_seconds: 6.1, + by_extension: {}, + samples: [], + }, + retrieval: { + scored_cases: 8, + skipped_cases: 4, + hit_rate: 0.75, + mrr: 0.5, + recall: 0.6, + context_relevance: 0.82, + }, + answer: { scored_cases: 12, pass_rate: 1, factuality: 0.9, rubric_score: 0.85 }, + cases: [ + { + query: "What is the refund window?", + retrieved_file_ids: ["policy.pdf"], + expected_file_ids: ["policy.pdf"], + hit: true, + reciprocal_rank: 1, + answer: "30 days", + answer_passed: true, + grader_reason: "Matches the reference answer", + }, + ], + error: null, + created_by: 1, +}; + +function renderTab() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + , + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + listDatasetsMock.mockResolvedValue([DATASET]); + listRunsMock.mockResolvedValue([COMPLETED_RUN]); + getRunMock.mockResolvedValue(RUN_DETAIL); +}); + +describe("EvaluationTab", () => { + it("lists datasets with their corpus and question counts", async () => { + renderTab(); + expect(await screen.findByText("Support docs")).toBeTruthy(); + expect(screen.getByText("3")).toBeTruthy(); + expect(screen.getByText("12")).toBeTruthy(); + }); + + it("starts a run for the chosen dataset", async () => { + startRunMock.mockResolvedValue(RUN_DETAIL); + renderTab(); + + await userEvent.click(await screen.findByRole("button", { name: /^run$/i })); + + await waitFor(() => expect(startRunMock).toHaveBeenCalledWith("ds1")); + }); + + it("disables starting a run while one is in flight", async () => { + listRunsMock.mockResolvedValue([{ ...COMPLETED_RUN, id: "run-active", status: "INDEXING" }]); + getRunMock.mockResolvedValue({ ...RUN_DETAIL, id: "run-active", status: "INDEXING" }); + renderTab(); + + await waitFor(() => + expect(screen.getByRole("button", { name: /^run$/i }).hasAttribute("disabled")).toBe(true), + ); + }); + + it("offers cancel only while a run is active", async () => { + renderTab(); + await screen.findByText("Support docs"); + expect(screen.queryByRole("button", { name: /cancel run/i })).toBeNull(); + + listRunsMock.mockResolvedValue([{ ...COMPLETED_RUN, id: "run-active", status: "EVALUATING" }]); + getRunMock.mockResolvedValue({ ...RUN_DETAIL, id: "run-active", status: "EVALUATING" }); + cancelRunMock.mockResolvedValue({ ...RUN_DETAIL, status: "CANCELLED" }); + + const { unmount } = renderTab(); + const cancelButton = await screen.findAllByRole("button", { name: /cancel run/i }); + await userEvent.click(cancelButton[0]); + await waitFor(() => expect(cancelRunMock).toHaveBeenCalledWith("run-active")); + unmount(); + }); + + it("shows the three metric families for the selected run", async () => { + renderTab(); + + expect(await screen.findByText("Indexing speed")).toBeTruthy(); + expect(screen.getByText("Retrieval quality")).toBeTruthy(); + expect(screen.getByText("Answer quality")).toBeTruthy(); + // hit_rate 0.75 rendered as a percentage in the detail panel + expect(screen.getByText("75.0%")).toBeTruthy(); + // Throughput appears twice: once in the run row, once as a detail stat. + expect(screen.getAllByText("12.5").length).toBeGreaterThan(0); + }); + + it("reports how many questions were skipped for lacking ground-truth sources", async () => { + renderTab(); + expect( + await screen.findByText(/8 question\(s\) scored, 4 skipped \(no expected_file_ids\)/), + ).toBeTruthy(); + }); + + it("renders the per-question table with the grader's reasoning", async () => { + renderTab(); + expect(await screen.findByText("What is the refund window?")).toBeTruthy(); + expect(screen.getByText("Matches the reference answer")).toBeTruthy(); + }); + + it("surfaces a failed run's error message", async () => { + listRunsMock.mockResolvedValue([ + { ...COMPLETED_RUN, id: "run-failed", status: "FAILED", error: "promptfoo timed out." }, + ]); + getRunMock.mockResolvedValue({ + ...RUN_DETAIL, + id: "run-failed", + status: "FAILED", + error: "promptfoo timed out.", + }); + renderTab(); + + expect(await screen.findByText("promptfoo timed out.")).toBeTruthy(); + }); + + it("tells the admin what to do when there are no datasets yet", async () => { + listDatasetsMock.mockResolvedValue([]); + listRunsMock.mockResolvedValue([]); + renderTab(); + + expect(await screen.findByText(/No datasets yet/)).toBeTruthy(); + expect(screen.getByText("No runs yet.")).toBeTruthy(); + }); +}); diff --git a/ui/src/pages/admin/evaluation/index.tsx b/ui/src/pages/admin/evaluation/index.tsx new file mode 100644 index 000000000..ae2c0af0b --- /dev/null +++ b/ui/src/pages/admin/evaluation/index.tsx @@ -0,0 +1,133 @@ +import { useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { Ban } from "lucide-react"; +import { EVAL_POLL_MS, cancelEvalRun, isActiveStatus, listEvalRuns } from "@/lib/api/evaluation"; +import { StatusBadge } from "@/components/shared/status-badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { DatasetCard } from "./dataset-card"; +import { RunDetail } from "./run-detail"; + +function percent(value: number | null): string { + return value === null ? "—" : `${(value * 100).toFixed(0)}%`; +} + +/** + * Evaluation tab: upload a dataset, run it, read the numbers. + * + * Runs are serialised server-side (one at a time), so the run list drives + * both the polling cadence and whether a new run can be started. + */ +export function EvaluationTab() { + const queryClient = useQueryClient(); + const [selectedRunId, setSelectedRunId] = useState(null); + + const { data, isLoading } = useQuery({ + queryKey: ["eval-runs"], + queryFn: () => listEvalRuns(), + refetchInterval: (query) => + (query.state.data ?? []).some((run) => isActiveStatus(run.status)) ? EVAL_POLL_MS : false, + }); + + const cancelMut = useMutation({ + mutationFn: (id: string) => cancelEvalRun(id), + onSuccess: () => { + toast.success("Cancellation requested"); + queryClient.invalidateQueries({ queryKey: ["eval-runs"] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const runs = data ?? []; + const activeRun = runs.find((run) => isActiveStatus(run.status)) ?? null; + // Default to the newest run so the tab is not empty after a run finishes. + const shownRunId = selectedRunId ?? activeRun?.id ?? runs[0]?.id ?? null; + + return ( +
+ + + + +
+ Runs + + One run at a time, so indexing timings stay comparable between them. + +
+ {activeRun && ( + + )} + + + {isLoading ? ( + + ) : runs.length === 0 ? ( +

No runs yet.

+ ) : ( + + + + Started + Status + Files/min + Hit rate + MRR + Answers + + + + {runs.map((run) => ( + setSelectedRunId(run.id)} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + setSelectedRunId(run.id); + } + }} + > + + {run.started_at ? new Date(run.started_at).toLocaleString() : "—"} + + + + + + {run.files_per_minute?.toFixed(1) ?? "—"} + + {percent(run.hit_rate)} + + {run.mrr?.toFixed(2) ?? "—"} + + + {percent(run.answer_pass_rate)} + + + ))} + +
+ )} + + + + {shownRunId && } +
+ ); +} diff --git a/ui/src/pages/admin/evaluation/run-detail.tsx b/ui/src/pages/admin/evaluation/run-detail.tsx new file mode 100644 index 000000000..7e9f7c968 --- /dev/null +++ b/ui/src/pages/admin/evaluation/run-detail.tsx @@ -0,0 +1,240 @@ +import { useQuery } from "@tanstack/react-query"; +import { EVAL_POLL_MS, getEvalRun, isActiveStatus, type EvalRun } from "@/lib/api/evaluation"; +import { StatusBadge } from "@/components/shared/status-badge"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; + +function percent(value: number | null | undefined): string { + return value === null || value === undefined ? "—" : `${(value * 100).toFixed(1)}%`; +} + +function score(value: number | null | undefined): string { + return value === null || value === undefined ? "—" : value.toFixed(3); +} + +function Stat({ label, value, hint }: { label: string; value: string; hint?: string }) { + return ( +
+
{label}
+
{value}
+ {hint &&

{hint}

} +
+ ); +} + +export function RunDetail({ runId }: { runId: string }) { + const { data: run, isLoading } = useQuery({ + queryKey: ["eval-run", runId], + queryFn: () => getEvalRun(runId), + // Poll only while the run can still change. + refetchInterval: (query) => { + const status = query.state.data?.status; + return status && isActiveStatus(status) ? EVAL_POLL_MS : false; + }, + }); + + if (isLoading) return ; + if (!run) return null; + + return ( +
+ + {run.error && ( + + {run.error} + + )} + + + +
+ ); +} + +function RunHeader({ run }: { run: EvalRun }) { + const elapsed = + run.started_at && run.finished_at + ? `${Math.round( + (new Date(run.finished_at).getTime() - new Date(run.started_at).getTime()) / 1000, + )}s` + : "—"; + + return ( + + +
+ {run.id.slice(0, 12)} + + Started {run.started_at ? new Date(run.started_at).toLocaleString() : "—"} · took {elapsed} + +
+ + + + ); +} + +function IndexingPanel({ run }: { run: EvalRun }) { + const metrics = run.indexing; + return ( + + + Indexing speed + End-to-end ingestion of the corpus into a throwaway partition. + + + {!metrics ? ( +

Not measured yet.

+ ) : ( + <> +
+ + + + + + 0 ? `${metrics.files_failed} failed` : undefined} + /> + +
+ {Object.keys(metrics.by_extension).length > 1 && ( +
+ {Object.entries(metrics.by_extension).map(([extension, bucket]) => ( + + {extension} · {bucket.files} file(s) ·{" "} + {bucket.mean_seconds}s avg + + ))} +
+ )} + + )} + + + ); +} + +function QualityPanel({ run }: { run: EvalRun }) { + const retrieval = run.retrieval; + const answer = run.answer; + + return ( +
+ + + Retrieval quality + + {retrieval + ? `${retrieval.scored_cases} question(s) scored${ + retrieval.skipped_cases > 0 + ? `, ${retrieval.skipped_cases} skipped (no expected_file_ids)` + : "" + }` + : "Not measured yet."} + + + + {retrieval && ( +
+ + + + +
+ )} + + + + + + Answer quality + + {answer ? `${answer.scored_cases} answer(s) graded by the LLM` : "Not measured yet."} + + + + {answer && ( +
+ + + +
+ )} + + +
+ ); +} + +function CasesTable({ run }: { run: EvalRun }) { + if (run.cases.length === 0) return null; + + return ( + + + Questions + + +
+ + + + Question + Hit + RR + Answer + Grader + + + + {run.cases.map((testCase, index) => ( + + +

+ {testCase.query} +

+ {testCase.retrieved_file_ids.length > 0 && ( +

+ {testCase.retrieved_file_ids.join(", ")} +

+ )} + + + {testCase.hit === null ? ( + — + ) : ( + + )} + + + {testCase.reciprocal_rank === null ? "—" : testCase.reciprocal_rank.toFixed(2)} + + + {testCase.answer_passed === null ? ( + — + ) : ( + + )} + + +

+ {testCase.grader_reason ?? "—"} +

+ + + ))} + +
+
+ + + ); +} diff --git a/ui/src/pages/admin/system.tsx b/ui/src/pages/admin/system.tsx index 67de38263..fb768f834 100644 --- a/ui/src/pages/admin/system.tsx +++ b/ui/src/pages/admin/system.tsx @@ -15,6 +15,7 @@ import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Badge } from "@/components/ui/badge"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { EvaluationTab } from "./evaluation"; import { Skeleton } from "@/components/ui/skeleton"; const GRAFANA_URL = import.meta.env.VITE_GRAFANA_URL || ""; @@ -31,7 +32,7 @@ export default function SystemPage() {
@@ -50,6 +51,7 @@ export default function SystemPage() { Actors Metrics Config + Evaluation @@ -64,6 +66,9 @@ export default function SystemPage() { + + +
);