diff --git a/packages/nemo_evaluator_sdk/examples/codex_docker/example.py b/packages/nemo_evaluator_sdk/examples/codex_docker/example.py deleted file mode 100644 index e8640adf00..0000000000 --- a/packages/nemo_evaluator_sdk/examples/codex_docker/example.py +++ /dev/null @@ -1,167 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Run an agent evaluation through the Docker Codex CLI evidence path. - -Prerequisites: - -* Docker is installed and the daemon is running. -* ``codex login`` has created ``~/.codex/auth.json``. Set ``CODEX_AUTH_PATH`` - when the auth file lives elsewhere. - -Run from the repository root with:: - - uv run python packages/nemo_evaluator_sdk/examples/codex_docker/example.py - -Run bundles are stored under ``temp/codex-docker-eval-output`` by default. -Set ``CODEX_DOCKER_EVAL_OUTPUT_ROOT`` to use a different location. - -The example deliberately reads a nested file through the trial's ``workspace`` -evidence descriptor. A successful score therefore exercises the Docker bind mount, -post-run ownership/permission normalization, private-tree validation, final-output -publication, and metric-side filesystem evidence access. -""" - -from __future__ import annotations - -import asyncio -import logging -import os -from datetime import UTC, datetime -from pathlib import Path - -from nemo_evaluator_sdk import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator -from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, BundleLocation -from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexDockerCliAgentRuntime -from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentTaskRunner - -EXPECTED_TEXT = "codex docker evidence works" -ARTIFACT_PATH = "sanity/result.txt" -CODEX_AUTH_PATH_ENV_NAME = "CODEX_AUTH_PATH" -CODEX_MODEL_ENV_NAME = "CODEX_MODEL" -OUTPUT_ROOT_ENV_NAME = "CODEX_DOCKER_EVAL_OUTPUT_ROOT" -REPO_ROOT = Path(__file__).resolve().parents[4] - - -class WorkspaceArtifactMetric: - """Score the final response and a file read through workspace evidence.""" - - @property - def type(self) -> str: - return "workspace_artifact" - - def output_spec(self) -> list[MetricOutputSpec]: - return [ - MetricOutputSpec.boolean("output_matches"), - MetricOutputSpec.boolean("artifact_matches"), - ] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - reference = input.row.data.get("reference", {}) - expected = reference.get("expected") if isinstance(reference, dict) else None - artifact_path = reference.get("artifact_path") if isinstance(reference, dict) else None - - output_matches = ( - isinstance(expected, str) - and input.candidate.output_text is not None - and input.candidate.output_text.strip() == expected - ) - - artifact_matches = False - evidence = input.candidate.evidence - if evidence is not None and isinstance(expected, str) and isinstance(artifact_path, str): - try: - workspace = await evidence.filesystem("workspace") - artifact_matches = (await workspace.read_text(artifact_path)).strip() == expected - except (KeyError, OSError, ValueError): - artifact_matches = False - - return MetricResult( - outputs=[ - MetricOutput(name="output_matches", value=output_matches), - MetricOutput(name="artifact_matches", value=artifact_matches), - ] - ) - - -def _new_output_dir() -> Path: - default_root = REPO_ROOT / "temp" / "codex-docker-eval-output" - output_root = Path(os.getenv(OUTPUT_ROOT_ENV_NAME, str(default_root))).expanduser() - timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") - return output_root / timestamp - - -def _docker_runtime(output_dir: Path) -> CodexDockerCliAgentRuntime: - auth_path = os.getenv(CODEX_AUTH_PATH_ENV_NAME) - model = os.getenv(CODEX_MODEL_ENV_NAME) - return CodexDockerCliAgentRuntime( - model=model or None, - work_root=output_dir / "evidence" / "codex-docker", - auth_path=auth_path or None, - ) - - -async def evaluate( - *, - output_dir: str | Path | None = None, - runtime: AgentTaskRunner | None = None, - write_dashboard: bool = True, -) -> tuple[AgentEvalResult, BundleLocation]: - """Run one Docker Codex task, score its workspace evidence, and store the run. - - Returns the result and where it was written: ``run`` itself no longer persists, so storing is an - explicit step here. - """ - resolved_output_dir = Path(output_dir).expanduser() if output_dir is not None else _new_output_dir() - target = runtime or _docker_runtime(resolved_output_dir) - - task = AgentEvalTask( - id="codex-docker-evidence", - intent="Create a nested artifact that remains private and host-readable after Docker exits.", - inputs={ - "instruction": ( - f"Create the directory {Path(ARTIFACT_PATH).parent.as_posix()} in the workspace. " - f"Write exactly '{EXPECTED_TEXT}' followed by a newline to {ARTIFACT_PATH}. " - f"Then reply with exactly: {EXPECTED_TEXT}" - ) - }, - reference={"artifact_path": ARTIFACT_PATH, "expected": EXPECTED_TEXT}, - metrics=[WorkspaceArtifactMetric()], - ) - - result = await AgentEvaluator().run( - tasks=[task], - target=target, - config=AgentEvalRunConfig( - work_dir=resolved_output_dir, - parallelism=1, - labels={"scenario": "codex-docker-evidence-sanity"}, - ), - ) - return result, result.persist(write_dashboard=write_dashboard) - - -async def main() -> None: - logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s") - result, location = await evaluate() - - trial = result.trials[0] - if trial.output is None or trial.evidence is None: - raise RuntimeError(f"Docker Codex trial failed: {trial.metadata}") - - workspace = await trial.evidence.filesystem("workspace") - artifact = workspace.path(ARTIFACT_PATH) - scores = {f"{score.metric_type}.{output.name}": output.value for score in result.scores for output in score.outputs} - - print(f"response: {trial.output.output_text}") - print(f"artifact: {artifact}") - print(f"artifact contents: {artifact.read_text(encoding='utf-8').strip()}") - print(f"workspace_artifact.output_matches: {scores['workspace_artifact.output_matches']}") - print(f"workspace_artifact.artifact_matches: {scores['workspace_artifact.artifact_matches']}") - print(f"run bundle: {location.output_dir}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/packages/nemo_evaluator_sdk/examples/profbench/README.md b/packages/nemo_evaluator_sdk/examples/profbench/README.md index b328166925..cc573437a0 100644 --- a/packages/nemo_evaluator_sdk/examples/profbench/README.md +++ b/packages/nemo_evaluator_sdk/examples/profbench/README.md @@ -9,32 +9,15 @@ Complete the following before running any command below: - **Python & uv**: Python 3.12+ with [`uv`](https://docs.astral.sh/uv/) and the repo synced (`uv sync`). - **Inference access**: API access for the configured chat-completions model. The default `live-judge` and `live-candidate` paths call this model, so set the relevant key (for example `OPENAI_API_KEY` or `NVIDIA_API_KEY`) in your environment. -- **Codex agent (only for `--agent codex`)**: install the `nemo-evaluator-sdk[agent-runtimes]` extra. Then either: - - `--runtime local`: the Codex CLI (`codex`) on `PATH` and authenticated (`codex login`). - - `--runtime docker`: a running Docker daemon. Codex uses the SDK Docker sandbox when `OPENAI_API_KEY` is an OpenAI Platform key (`sk-…`); otherwise it runs the Codex CLI in a container with your local `~/.codex/auth.json` mounted read-only. +- **Docker sandbox candidate (only for the SDK-level runtime below)**: install the `nemo-evaluator-sdk[agent-runtimes]` extra and have a running Docker daemon. +- **Coding-agent candidate (only for `--agent fabric-codex`)**: the Fabric harness adapters (`uv sync --frozen --package nemo-evaluator-sdk --extra fabric`, or `script/dev-install-fabric.sh`) and the Codex CLI (`codex`) on `PATH` and authenticated (`codex login`). No `nemo-relay` gateway is needed — this arm does not capture trajectories. Run the example from the repository root: -- Local - ```bash -python -m packages.nemo_evaluator_sdk.examples.profbench.runner \ +uv run --frozen --package nemo-evaluator-sdk python -m packages.nemo_evaluator_sdk.examples.profbench.runner \ --output-dir env/profbench-results \ - --limit=1 \ - --agent codex \ - --runtime local \ - --agent-model gpt-5.5 -``` - -- Docker - -```bash -python -m packages.nemo_evaluator_sdk.examples.profbench.runner \ - --output-dir env/profbench-results \ - --limit=1 \ - --agent codex \ - --runtime docker \ - --agent-model gpt-5.5 + --limit=1 ``` Each invocation creates one run directory under `--output-dir`, then writes each enabled runner mode under that run: @@ -63,31 +46,31 @@ env/profbench-results/ Live paths are enabled by default and require API access through the configured model settings. Pass `--no-run-live-judge` or `--no-run-live-candidate` to skip either live path. -## Code Sandbox Invocation +## Coding-agent candidate -The CLI runner above uses the configured chat-completions model directly for `live-candidate`. To run ProfBench with a Codex-style candidate agent, pass `--agent codex`. Docker is the default Codex runtime preference: it uses SDK Docker when `OPENAI_API_KEY` is an OpenAI Platform secret key, and otherwise starts a lightweight Docker container that runs Codex CLI with your local Codex auth mounted read-only. +`live-candidate` calls the configured chat-completions model directly by default. Pass `--agent fabric-codex` to generate the candidate answers with the Codex CLI instead, driven through [NeMo Fabric](https://github.com/nvidia/nemo-fabric): ```bash -python -m packages.nemo_evaluator_sdk.examples.profbench.runner \ +uv run --frozen --package nemo-evaluator-sdk --extra fabric python -m packages.nemo_evaluator_sdk.examples.profbench.runner \ --output-dir env/profbench-results \ --limit=1 \ - --agent codex \ + --agent fabric-codex \ --agent-model gpt-5.4 ``` -List locally visible Codex model slugs before choosing `--agent-model`: +Notes on this arm: -```bash -python -m packages.nemo_evaluator_sdk.examples.profbench.runner --agent codex --list-agent-models -``` +- Fabric selects the harness by `harness.adapter_id` (`nvidia.fabric.codex` here), never from the model. `--agent-model` supplies the config's default model; the Codex adapter refuses to start without one, so the example ships `openai/gpt-5.4` as its default. +- The sandbox is `read-only`: ProfBench grades a block of answer text, so the agent has no reason to write files. +- The agent arm prefixes each task's instruction with an answer-only preamble, because Fabric sends `inputs["instruction"]` verbatim and a coding agent otherwise returns tool logs and markdown fences that a rubric judge reads as a worse answer. The baseline and live-judge arms score recorded responses and are left unframed, so the comparison stays honest. +- Trajectory capture is off, so the `nemo-relay` gateway is not required. The rubric judge scores the answer, not the agent's steps. +- Runs are labelled `score_source=fabric_codex_candidate_and_live_judge`, against `fresh_candidate_and_live_judge` for the model arm. + +Credentials: the Codex CLI uses your own Codex login (or `OPENAI_API_KEY`). The live rubric judge is unaffected — it still goes through the regular SDK model path and needs `NVIDIA_API_KEY` unless you override the judge model and secret settings. -Runtime and credential behavior: +## Docker sandbox candidate (SDK level) -- `--agent codex --runtime docker` prefers `DockerSandboxAgentRuntime` when `OPENAI_API_KEY` looks like an OpenAI Platform secret key (`sk-...`). Model calls are made by the OpenAI Agents SDK from the host process; the Docker container is the execution workspace and `~/.codex/auth.json` is not mounted. -- `--agent codex --runtime docker` falls back to Dockerized Codex CLI when `OPENAI_API_KEY` is missing or is a Codex OAuth token. It mounts `~/.codex/auth.json` read-only into a `node:22-alpine` container and runs `npx -y @openai/codex@0.137.0 exec ...`; Codex OAuth tokens are not converted into API keys. Because Docker is the isolation boundary, this path runs Codex with its nested shell-command sandbox disabled to avoid `bwrap` user-namespace failures inside the container. -- `--agent codex --runtime local` always uses host `codex exec` from `PATH`, so it relies on your local Codex login/auth. It passes `--ignore-user-config` so benchmark runs do not inherit `$CODEX_HOME/config.toml` MCP servers, plugins, approval settings, or other user-specific tool configuration. No Docker containers are expected in this mode. -- The live ProfBench judge still uses the regular NeMo Evaluator SDK model settings, so the default judge path still needs `NVIDIA_API_KEY` unless you override the judge model and secret settings. -- If `--agent-model` is omitted, SDK Docker uses the documented default Codex sandbox model; Dockerized/local CLI uses the default model in your Codex config. +`DockerSandboxAgentRuntime` generates candidate answers by running an OpenAI Agents SDK `SandboxAgent` in a Docker container per task. It has no ProfBench CLI flag — drive it directly: Install the optional runtime extra for SDK Docker mode: @@ -173,7 +156,6 @@ Credential flow: - `DockerSandboxAgentRuntime` uses the OpenAI Agents SDK `SandboxAgent`; model calls are made from the host process and use `OPENAI_API_KEY`. - The Docker container is an execution sandbox for task workspace/files. It does not receive your API keys unless you explicitly mount or write them into the task workspace later. - `ProfBenchModelJudge` uses the regular NeMo Evaluator SDK model path. In the example above, `SecretRef(root="NVIDIA_API_KEY")` resolves the judge key from the local `NVIDIA_API_KEY` environment variable. -- For the normal ProfBench CLI, the evaluated and judge models default to NVIDIA NIM. Configure them with `NEMO_EVALUATOR_PROFBENCH_EVALUATED_MODEL_URL`, `NEMO_EVALUATOR_PROFBENCH_EVALUATED_MODEL`, `NEMO_EVALUATOR_PROFBENCH_JUDGE_MODEL_URL`, and `NEMO_EVALUATOR_PROFBENCH_JUDGE_MODEL`. The API-key environment variable name defaults to `NVIDIA_API_KEY` and can be changed with `NMP_EVALUATOR_DEFAULT_API_KEY_SECRET`. ## Domain Model diff --git a/packages/nemo_evaluator_sdk/examples/profbench/runner.py b/packages/nemo_evaluator_sdk/examples/profbench/runner.py index f66b904eb6..77859b9ddd 100644 --- a/packages/nemo_evaluator_sdk/examples/profbench/runner.py +++ b/packages/nemo_evaluator_sdk/examples/profbench/runner.py @@ -10,7 +10,6 @@ import logging import os import uuid -from collections.abc import Mapping from datetime import datetime from enum import StrEnum from pathlib import Path @@ -23,12 +22,7 @@ from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult -from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import ( - EffectiveCodexRuntime, - RuntimeChoice, - print_codex_agent_models, - resolve_codex_runtime, -) +from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTarget, AgentEvalTrial from nemo_evaluator_sdk.values import InferenceParams, Model, RunConfigOnlineModel, SecretRef @@ -46,11 +40,12 @@ DEFAULT_MODEL_URL = "https://integrate.api.nvidia.com/v1/chat/completions" DEFAULT_MODEL_NAME = "nvidia/nemotron-3-nano-30b-a3b" DEFAULT_API_KEY_SECRET = os.getenv("NMP_EVALUATOR_DEFAULT_API_KEY_SECRET", "NVIDIA_API_KEY") +DEFAULT_FABRIC_CODEX_MODEL = "gpt-5.4" class AgentChoice(StrEnum): MODEL = "model" - CODEX = "codex" + FABRIC_CODEX = "fabric-codex" class ProfBenchMode(StrEnum): @@ -74,7 +69,6 @@ async def run_profbench_mode( run_instance_id: str | None = None, agent: AgentChoice = AgentChoice.MODEL, agent_model: str | None = None, - runtime: RuntimeChoice = RuntimeChoice.DOCKER, ) -> None: """Run one ProfBench mode. @@ -101,23 +95,21 @@ async def run_profbench_mode( trials: list[AgentEvalTrial] | None = None params: RunConfigOnlineModel | None = None benchmark_labels = {key: str(value) for key, value in benchmark.metadata.items()} + tasks = benchmark.tasks if mode is ProfBenchMode.LIVE_CANDIDATE: - target, params, score_source, effective_codex_runtime = _live_candidate_target( - agent=agent, - agent_model=agent_model, - runtime=runtime, - output_dir=output_dir, + target, params, score_source = _live_candidate_target( + agent=agent, agent_model=agent_model, output_dir=output_dir ) - if effective_codex_runtime is not None: - print(f"Codex runtime: {effective_codex_runtime}") benchmark_labels["score_source"] = score_source + if agent is AgentChoice.FABRIC_CODEX: + tasks = [_as_candidate_task(task) for task in tasks] else: trials = benchmark.trials if mode is ProfBenchMode.LIVE_JUDGE: benchmark_labels["score_source"] = "live_judge" result = await AgentEvaluator().run( - tasks=benchmark.tasks, + tasks=tasks, trials=trials, target=target, config=AgentEvalRunConfig( @@ -149,7 +141,6 @@ async def run_examples( run_instance_id: str | None = None, agent: AgentChoice = AgentChoice.MODEL, agent_model: str | None = None, - runtime: RuntimeChoice = RuntimeChoice.DOCKER, ) -> None: """Execute the enabled ProfBench agent-eval modes under one shared run folder.""" output_root = _resolve_profbench_output_root(output_root) @@ -185,7 +176,6 @@ async def run_examples( run_instance_id=run_instance_id, agent=agent, agent_model=agent_model, - runtime=runtime, ) else: print("Skipping live ProfBench candidate example. Remove --no-run-live-candidate to run it.") @@ -242,62 +232,63 @@ def _judge_model() -> Model: ) -# ProfBench-specific Codex policy. Runtime *selection* is generic (resolve_codex_runtime, in the SDK); -# ProfBench owns only how a task is framed for the agent and how the resulting score is labeled. -# ProfBench runs Codex as a *candidate* whose single text answer a live judge grades, so the prompt -# forbids tool chatter and the score_source records the candidate+judge topology. -PROFBENCH_SCORE_SOURCE = { - EffectiveCodexRuntime.LOCAL_CLI: "codex_cli_candidate_and_live_judge", - EffectiveCodexRuntime.DOCKER_CLI: "codex_docker_cli_candidate_and_live_judge", - EffectiveCodexRuntime.DOCKER_SANDBOX: "docker_sandbox_candidate_and_live_judge", +#: Fabric config for the coding-agent candidate: the Codex CLI harness, run once per task as a +#: subprocess. ``sandbox: read-only`` because ProfBench grades a text answer and the agent has no +#: reason to write files. A plain mapping (rather than nemo_fabric's typed config) keeps this module +#: importable without the native Fabric stack installed. +PROFBENCH_FABRIC_CODEX_CONFIG = { + "metadata": {"name": "profbench-candidate"}, + "harness": {"adapter_id": "nvidia.fabric.codex", "settings": {"sandbox": "read-only"}}, + # The Codex adapter refuses to start without a model provider, so the config carries a default + # rather than deferring to the CLI's own; `--agent-model` overrides it. + "models": {"default": {"provider": "openai", "model": DEFAULT_FABRIC_CODEX_MODEL}}, + "runtime": {"mode": "oneshot", "transport": "cli"}, } +#: ProfBench grades one block of answer text against a rubric, so tool logs and commentary read as a +#: worse answer. A chat model gives a bare answer already; a coding agent has to be told. +PROFBENCH_CANDIDATE_PREAMBLE = ( + "Answer the task below. Return only the final answer text; do not include analysis, " + "markdown fences, tool logs, or commentary.\n\n" +) -def profbench_codex_prompt(task: AgentEvalTask) -> str: - """Frame a task as a ProfBench candidate: return only the final answer text, no tooling chatter.""" - return ( - "Answer the ProfBench task below. Return only the final answer text; do not include " - "analysis, markdown fences, tool logs, or commentary.\n\n" - f"Task id: {task.id}\n" - f"Intent: {task.intent}\n" - f"Inputs: {task.inputs}\n" - ) + +def _as_candidate_task(task: AgentEvalTask) -> AgentEvalTask: + """Prefix a task's instruction with the answer-only framing, leaving grading untouched. + + ``FabricAgentRuntime`` sends ``inputs['instruction']`` verbatim, so the framing has to live in + the task. Only the agent arm gets it: the baseline and live-judge arms score recorded responses + that were never prompted this way, and re-framing them would change what is being compared. + """ + inputs = dict(task.inputs) + # Read through ``agent_prompt`` rather than ``inputs`` directly: it rejects a task with no + # instruction, and prefixing the preamble onto an empty one would make that value truthy and + # silently run the agent on the preamble alone. + inputs["instruction"] = PROFBENCH_CANDIDATE_PREAMBLE + task.agent_prompt() + return task.model_copy(update={"inputs": inputs}) def _live_candidate_target( - *, - agent: AgentChoice, - agent_model: str | None, - runtime: RuntimeChoice, - output_dir: Path, - env: Mapping[str, str] = os.environ, -) -> tuple[ - AgentEvalTarget, - RunConfigOnlineModel | None, - str, - EffectiveCodexRuntime | None, -]: - if agent == AgentChoice.MODEL: + *, agent: AgentChoice, agent_model: str | None, output_dir: Path +) -> tuple[AgentEvalTarget, RunConfigOnlineModel | None, str]: + if agent is AgentChoice.MODEL: return ( _evaluated_model(agent_model), - RunConfigOnlineModel( - parallelism=2, - inference=InferenceParams(temperature=0.0, max_tokens=32768), - ), + RunConfigOnlineModel(parallelism=2, inference=InferenceParams(temperature=0.0, max_tokens=32768)), "fresh_candidate_and_live_judge", - None, ) - if agent == AgentChoice.CODEX: - # SDK picks the runtime; ProfBench supplies the candidate prompt and the score_source label. - target, effective_runtime = resolve_codex_runtime( - runtime=runtime, + # Trajectory capture is off: it needs the nemo-relay gateway, and the rubric judge scores the + # answer text, not the agent's steps. + return ( + FabricAgentRuntime( + config=PROFBENCH_FABRIC_CODEX_CONFIG, model=agent_model, - output_dir=output_dir, - env=env, - prompt_builder=profbench_codex_prompt, - ) - return target, None, PROFBENCH_SCORE_SOURCE[effective_runtime], effective_runtime - raise ValueError(f"unsupported ProfBench agent {agent!r}") + work_root=output_dir / "evidence" / "fabric", + capture_trajectory=False, + ), + None, + "fabric_codex_candidate_and_live_judge", + ) def _print_example_separator(name: str) -> None: @@ -341,38 +332,21 @@ def _print_example_separator(name: str) -> None: type=AgentChoice, choices=list(AgentChoice), default=AgentChoice.MODEL, - help="Candidate agent for live-candidate mode. Use 'codex' for Codex-backed candidate generation.", - ) - parser.add_argument( - "--runtime", - type=RuntimeChoice, - choices=list(RuntimeChoice), - default=RuntimeChoice.DOCKER, help=( - "Runtime for --agent codex. Default: docker. Docker uses SDK Docker when OPENAI_API_KEY is an " - "OpenAI secret key and runs Codex CLI inside Docker otherwise. Use local to force the host Codex CLI." + "Candidate for live-candidate mode. 'model' calls the chat-completions model directly; " + "'fabric-codex' drives the Codex CLI through NeMo Fabric." ), ) parser.add_argument( "--agent-model", default=None, help=( - "Model name for the selected candidate agent. With --agent codex --runtime local this " - "is passed to `codex exec --model`; with --agent codex --runtime docker this is passed " - "to the effective Codex runtime; with --agent model it overrides the evaluated model name." + "Model for the live candidate. With --agent model it overrides the evaluated model name; " + "with --agent fabric-codex it is a `provider/model` slug applied to the Fabric config " + "(the harness default is used when omitted)." ), ) - parser.add_argument( - "--list-agent-models", - action="store_true", - help="List locally visible Codex model slugs for --agent codex and exit.", - ) args = parser.parse_args() - if args.list_agent_models: - if args.agent != AgentChoice.CODEX: - parser.error("--list-agent-models is only supported with --agent codex") - print_codex_agent_models() - raise SystemExit(0) configure_example_logging() asyncio.run( @@ -383,6 +357,5 @@ def _print_example_separator(name: str) -> None: output_root=args.output_dir, agent=args.agent, agent_model=args.agent_model, - runtime=args.runtime, ) ) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py deleted file mode 100644 index 75b0af038b..0000000000 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py +++ /dev/null @@ -1,641 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Codex-backed agent-eval runtimes.""" - -# ruff: noqa: I001, T201 - the vendored SDK mirror uses different import-order and print settings. - -from __future__ import annotations - -import asyncio -import contextlib -import json -import os -import shlex -import shutil -import stat -import subprocess -import tempfile -from collections.abc import Awaitable, Callable, Mapping, Sequence -from enum import StrEnum -from pathlib import Path -from typing import Any - -from nemo_evaluator_sdk.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime -from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import ( - AgentEvalTrial, - AgentEvalTrialStatus, - AgentOutput, - RunnerInfo, - callable_identity, -) -from nemo_evaluator_sdk.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace -from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor - -#: Wall-clock ceiling for a single task's Codex CLI invocation — one ``process.communicate()`` covering -#: the agent's whole run on that task, not a per-request or per-turn limit. Tasks run independently, so -#: this is not a budget for the evaluation as a whole. On expiry the process is terminated and the task -#: is recorded as a failed trial; it does not abort the run. -DEFAULT_CODEX_TIMEOUT_S = 600 -DEFAULT_CODEX_DOCKER_MODEL = "gpt-5.4" -DEFAULT_CODEX_DOCKER_CLI_IMAGE = "node:22-alpine" -DEFAULT_CODEX_DOCKER_CLI_PACKAGE = "@openai/codex@0.137.0" -ProcessFactory = Callable[..., Awaitable[Any]] - - -class RuntimeChoice(StrEnum): - """Which Codex execution mode the caller wants.""" - - DOCKER = "docker" - LOCAL = "local" - - -class EffectiveCodexRuntime(StrEnum): - """The concrete runtime chosen for a :class:`RuntimeChoice` + environment.""" - - DOCKER_SANDBOX = "docker_sandbox" - DOCKER_CLI = "docker_cli" - LOCAL_CLI = "local_cli" - - -#: Builds the prompt handed to Codex on stdin for a task. Swap it to change how a task is framed -#: (e.g. a benchmark-specific preamble); the default presents the task and invites workspace edits. -CodexPromptBuilder = Callable[[AgentEvalTask], str] - - -class CodexCliAgentRuntime: - """AgentTaskRunner that uses the locally installed Codex CLI credentials.""" - - def __init__( - self, - *, - model: str | None = None, - work_root: str | Path | None = None, - codex_bin: str = "codex", - timeout_s: int = DEFAULT_CODEX_TIMEOUT_S, - prompt_builder: CodexPromptBuilder | None = None, - process_factory: ProcessFactory | None = None, - runtime_name: str = "codex_cli", - ) -> None: - self._model = model - self._work_root = Path(work_root).expanduser() if work_root is not None else None - self._codex_bin = codex_bin - self._timeout_s = timeout_s - self._prompt_builder = prompt_builder or AgentEvalTask.agent_prompt - self._process_factory = process_factory or asyncio.create_subprocess_exec - self._runtime_name = runtime_name - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the Codex CLI settings that shape its results. - - Uses ``runtime_name``, which subclasses already set (the Docker variant reports - ``codex_docker_cli``) and which trials are stamped with, so provenance agrees with them. - """ - return RunnerInfo( - name=self._runtime_name, - kind="runner", - config={ - "model": self._model, - "timeout_s": self._timeout_s, - "codex_bin": self._codex_bin, - "prompt_builder": callable_identity(self._prompt_builder), - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - if shutil.which(self._codex_bin) is None: - raise RuntimeError(f"Codex CLI executable {self._codex_bin!r} was not found on PATH") - - resolved_config = config or AgentEvalRunConfig() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(index, task, resolved_config) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> AgentEvalTrial: - evidence_dir = self._evidence_dir(index, task, config) - workspace_dir = evidence_dir / "workspace" - - try: - # The task directory is mounted into Docker, but its private parent is not. Keeping that - # parent host-owned and 0700 preserves the local confidentiality boundary even when a - # container is interrupted before its recursive cleanup completes. - _ensure_private_directory(evidence_dir.parent) - _ensure_private_directory(evidence_dir) - _ensure_private_directory(workspace_dir) - except Exception as exc: - # The path that failed setup is not safe to use for artifact persistence. In particular, - # writing through a rejected evidence-directory symlink would escape the private tree. - return _failed_codex_trial(task, None, exc, runtime_name=self._runtime_name) - - prompt_path = evidence_dir / "prompt.txt" - task_path = evidence_dir / "task.json" - stdout_path = evidence_dir / "stdout.jsonl" - stderr_path = evidence_dir / "stderr.txt" - final_output_path = evidence_dir / "final_output.txt" - - # Persist the task for debugging, but never the grader-only fields: the docker variant mounts - # this evidence dir into the sandbox (danger-full-access), so serializing `intent` (desired - # behavior) or `reference` (held-out ground truth) here would let the agent read them back out - # of `/evidence/task.json` — the same reward-hacking leak the intent-free prompt closes. - try: - _write_private_text(task_path, task.model_dump_json(indent=2, exclude={"intent", "reference"})) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - - command = self._command(workspace_dir=workspace_dir, final_output_path=final_output_path) - process: Any | None = None - try: - # Seed inside the guarded block so a bad seed (e.g. a path escaping the workspace) fails - # just this task rather than aborting the whole run. Offload to a worker thread: seeding is - # synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this - # runs on the event loop shared by every concurrent task, so a blocking seed would stall them all. - seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - # Build the prompt after seeding and inside the guarded block: an instruction-less task - # raises here, failing just this task instead of aborting the run (and seeding wins if both). - prompt = self._prompt_builder(task) - _write_private_text(prompt_path, prompt) - process = await self._process_factory( - *command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if process is None: - raise RuntimeError("process factory failed to create a process") - stdout, stderr = await asyncio.wait_for( - process.communicate(prompt.encode("utf-8")), - timeout=self._timeout_s, - ) - except TimeoutError as exc: - await _terminate_process(process) - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - - stdout_text = _decode_process_output(stdout) - stderr_text = _decode_process_output(stderr) - artifact_persistence_error: str | None = None - try: - _write_private_text(stdout_path, stdout_text) - _write_private_text(stderr_path, stderr_text) - except Exception as exc: - artifact_persistence_error = f"{exc.__class__.__name__}: {exc}" - - permission_cleanup_error: str | None = None - try: - self._validate_artifact_permissions(evidence_dir) - except Exception as exc: - permission_cleanup_error = f"{exc.__class__.__name__}: {exc}" - - if process.returncode != 0: - return _failed_codex_trial( - task, - evidence_dir, - RuntimeError(f"codex exec exited with status {process.returncode}: {stderr_text.strip()}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - artifact_persistence_error=artifact_persistence_error, - ) - - if artifact_persistence_error is not None: - return _failed_codex_trial( - task, - evidence_dir, - RuntimeError(f"failed to persist Codex evidence: {artifact_persistence_error}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - artifact_persistence_error=artifact_persistence_error, - ) - if permission_cleanup_error is not None: - return _failed_codex_trial( - task, - evidence_dir, - PermissionError(f"Codex evidence permission normalization failed: {permission_cleanup_error}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - ) - - try: - output_text = _read_private_final_output(final_output_path, fallback=stdout_text) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - return AgentEvalTrial( - id=f"{task.id}:codex", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=output_text, - metadata={ - "runtime": self._runtime_name, - "agent": "codex", - "agent_model": self._model, - "evidence_dir": str(evidence_dir), - }, - ), - evidence=CandidateEvidence( - descriptors={ - "workspace": EvidenceDescriptor(kind="filesystem", ref=str(workspace_dir)), - "prompt": EvidenceDescriptor(kind="text", format="txt", ref=str(prompt_path)), - "task": EvidenceDescriptor(kind="json", format="json", ref=str(task_path)), - "stdout": EvidenceDescriptor(kind="codex_stdout", format="jsonl", ref=str(stdout_path)), - "stderr": EvidenceDescriptor(kind="text", format="txt", ref=str(stderr_path)), - "final_output": EvidenceDescriptor(kind="text", format="txt", ref=str(final_output_path)), - }, - metadata={"runtime": self._runtime_name, "agent": "codex"}, - ), - metadata={ - "runtime": self._runtime_name, - "agent": "codex", - "agent_model": self._model, - "agent_ok": True, - "seeded_files": seeded_files, - "generated": True, - }, - ) - - def _command(self, *, workspace_dir: Path, final_output_path: Path) -> list[str]: - command = [ - self._codex_bin, - "exec", - "--skip-git-repo-check", - "--ephemeral", - "--ignore-user-config", - "--sandbox", - "workspace-write", - "--cd", - str(workspace_dir), - "--output-last-message", - str(final_output_path), - "--json", - ] - if self._model is not None: - command.extend(["--model", self._model]) - command.append("-") - return command - - def _validate_artifact_permissions(self, evidence_dir: Path) -> None: - """Validate runtime-specific artifact postconditions after the process exits.""" - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = self._work_root - if root is None: - root = (config.work_dir or Path.cwd()) / "evidence" / "codex" - safe_task_id = _safe_path_name(task.id) - task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return Path(root) / task_dir - - -class CodexDockerCliAgentRuntime(CodexCliAgentRuntime): - """AgentTaskRunner that runs Codex CLI inside a Docker container.""" - - def __init__( - self, - *, - model: str | None = None, - work_root: str | Path | None = None, - docker_bin: str = "docker", - image: str = DEFAULT_CODEX_DOCKER_CLI_IMAGE, - codex_package: str = DEFAULT_CODEX_DOCKER_CLI_PACKAGE, - auth_path: str | Path | None = None, - timeout_s: int = DEFAULT_CODEX_TIMEOUT_S, - prompt_builder: CodexPromptBuilder | None = None, - process_factory: ProcessFactory | None = None, - ) -> None: - super().__init__( - model=model, - work_root=work_root, - timeout_s=timeout_s, - prompt_builder=prompt_builder, - process_factory=process_factory, - runtime_name="codex_docker_cli", - ) - self._docker_bin = docker_bin - self._image = image - self._codex_package = codex_package - self._auth_path = ( - Path(auth_path).expanduser() if auth_path is not None else Path.home() / ".codex" / "auth.json" - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - if shutil.which(self._docker_bin) is None: - raise RuntimeError(f"Docker executable {self._docker_bin!r} was not found on PATH") - if not self._auth_path.exists(): - raise RuntimeError( - f"Codex auth file was not found at {self._auth_path}. Run `codex login` or use OPENAI_API_KEY " - "so --runtime docker can use DockerSandboxAgentRuntime." - ) - - resolved_config = config or AgentEvalRunConfig() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(index, task, resolved_config) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - def _command(self, *, workspace_dir: Path, final_output_path: Path) -> list[str]: - evidence_dir = final_output_path.parent - inner_command = [ - "npx", - "-y", - self._codex_package, - "exec", - "--skip-git-repo-check", - "--ephemeral", - "--sandbox", - "danger-full-access", - "--cd", - "/workspace", - "--output-last-message", - "/evidence/final_output.txt", - "--json", - ] - if self._model is not None: - inner_command.extend(["--model", self._model]) - inner_command.append("-") - # Codex intentionally runs as root: the container mounts its auth under /root and coding tasks - # may need to install tools. Repair the bind-mounted trees before Docker returns so the host can - # score and persist every artifact the agent created without widening access to other host users. - # Capture the bind mount's owner as seen inside this container before Codex runs: raw host UID/GID - # values are not portable across Docker Desktop and rootless user-namespace mappings. Keep Codex - # failures authoritative; only surface the required chmod status when Codex itself succeeded. - shell_command = ( - "host_owner=\"$(stat -c '%u:%g' /evidence 2>/dev/null)\" || true; " - f"{shlex.join(inner_command)}; " - "codex_status=$?; " - 'if [ -n "$host_owner" ]; then ' - 'chown -R "$host_owner" /workspace /evidence 2>/dev/null || true; ' - "fi; " - "chmod -R u+rwX,go-rwx /workspace /evidence; " - "permissions_status=$?; " - 'if [ "$codex_status" -ne 0 ]; then exit "$codex_status"; fi; ' - 'exit "$permissions_status"' - ) - return [ - self._docker_bin, - "run", - "--rm", - "-i", - "-e", - "PYTHONDONTWRITEBYTECODE=1", - "-v", - f"{self._auth_path.resolve()}:/root/.codex/auth.json:ro", - "-v", - f"{workspace_dir.resolve()}:/workspace", - "-v", - f"{evidence_dir.resolve()}:/evidence", - self._image, - "sh", - "-lc", - shell_command, - ] - - def _validate_artifact_permissions(self, evidence_dir: Path) -> None: - _validate_private_tree(evidence_dir) - - -def resolve_codex_runtime( - *, - runtime: RuntimeChoice, - model: str | None, - output_dir: Path, - env: Mapping[str, str] = os.environ, - prompt_builder: CodexPromptBuilder | None = None, -) -> tuple[CodexCliAgentRuntime | CodexDockerCliAgentRuntime | DockerSandboxAgentRuntime, EffectiveCodexRuntime]: - """Pick and construct a Codex runtime for a run-mode + environment. - - ``local`` runs the on-PATH Codex CLI. ``docker`` prefers the OpenAI-Agents ``DockerSandbox`` when - ``OPENAI_API_KEY`` is an OpenAI platform secret (``sk-...``) and otherwise falls back to the - containerized Codex CLI (which mounts ``~/.codex/auth.json``). ``prompt_builder`` is threaded into - the CLI runtimes; the sandbox runtime does its own prompting. Returns the runtime plus the - :class:`EffectiveCodexRuntime` actually chosen so callers can label/report it. - """ - effective_runtime = _resolve_codex_runtime(runtime, env) - if effective_runtime == EffectiveCodexRuntime.LOCAL_CLI: - return ( - CodexCliAgentRuntime( - model=model, - work_root=output_dir / "evidence" / "codex", - prompt_builder=prompt_builder, - ), - effective_runtime, - ) - if effective_runtime == EffectiveCodexRuntime.DOCKER_CLI: - return ( - CodexDockerCliAgentRuntime( - model=model, - work_root=output_dir / "evidence" / "codex-docker", - prompt_builder=prompt_builder, - ), - effective_runtime, - ) - if effective_runtime == EffectiveCodexRuntime.DOCKER_SANDBOX: - return DockerSandboxAgentRuntime(model=model or DEFAULT_CODEX_DOCKER_MODEL), effective_runtime - raise ValueError(f"unsupported Codex runtime {runtime!r}") - - -def _resolve_codex_runtime(runtime: RuntimeChoice, env: Mapping[str, str] = os.environ) -> EffectiveCodexRuntime: - if runtime == RuntimeChoice.LOCAL: - return EffectiveCodexRuntime.LOCAL_CLI - if runtime == RuntimeChoice.DOCKER: - if _openai_sdk_secret_key_is_set(env): - return EffectiveCodexRuntime.DOCKER_SANDBOX - return EffectiveCodexRuntime.DOCKER_CLI - raise ValueError(f"unsupported Codex runtime {runtime!r}") - - -def _openai_sdk_secret_key_is_set(env: Mapping[str, str] = os.environ) -> bool: - return env.get("OPENAI_API_KEY", "").strip().startswith("sk-") - - -def list_codex_agent_models(*, codex_bin: str = "codex") -> list[dict[str, Any]]: - """Return visible Codex model descriptors from the local Codex CLI.""" - if shutil.which(codex_bin) is None: - raise RuntimeError(f"Codex CLI executable {codex_bin!r} was not found on PATH") - result = subprocess.run( - [codex_bin, "debug", "models"], - check=True, - capture_output=True, - text=True, - ) - payload = json.loads(result.stdout) - models = payload.get("models") - if not isinstance(models, list): - raise RuntimeError("Codex model catalog did not contain a models list") - visible = [model for model in models if isinstance(model, dict) and model.get("visibility") == "list"] - return sorted(visible, key=lambda model: int(model.get("priority") or 0), reverse=True) - - -def print_codex_agent_models(*, codex_bin: str = "codex") -> None: - """Print local Codex model slugs and display names.""" - for model in list_codex_agent_models(codex_bin=codex_bin): - slug = model.get("slug") - if not isinstance(slug, str): - continue - display_name = model.get("display_name") - if isinstance(display_name, str) and display_name != slug: - print(f"{slug}\t{display_name}") - else: - print(slug) - - -def _failed_codex_trial( - task: AgentEvalTask, - evidence_dir: Path | None, - exc: Exception, - *, - runtime_name: str = "codex_cli", - permission_cleanup_error: str | None = None, - artifact_persistence_error: str | None = None, -) -> AgentEvalTrial: - evidence: CandidateEvidence | None = None - error_artifact_error: str | None = None - if evidence_dir is not None: - error_path = evidence_dir / "error.json" - try: - _write_private_text( - error_path, json.dumps({"error_type": exc.__class__.__name__, "error": str(exc)}) + "\n" - ) - except Exception as artifact_exc: - error_artifact_error = f"{artifact_exc.__class__.__name__}: {artifact_exc}" - else: - evidence = CandidateEvidence( - descriptors={"error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path))}, - metadata={"runtime": runtime_name, "agent": "codex"}, - ) - - metadata: dict[str, Any] = { - "runtime": runtime_name, - "agent": "codex", - "agent_ok": False, - "error_type": exc.__class__.__name__, - "error": str(exc), - } - if permission_cleanup_error is not None: - metadata["permission_cleanup_error"] = permission_cleanup_error - if artifact_persistence_error is not None: - metadata["artifact_persistence_error"] = artifact_persistence_error - if error_artifact_error is not None: - metadata["error_artifact_error"] = error_artifact_error - return AgentEvalTrial( - id=f"{task.id}:codex", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=evidence, - metadata=metadata, - ) - - -def _ensure_private_directory(path: Path) -> None: - """Create or repair a host-owned directory without following a leaf symlink.""" - path.mkdir(mode=0o700, parents=True, exist_ok=True) - descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - try: - path_stat = os.fstat(descriptor) - if path_stat.st_uid != os.getuid(): - raise PermissionError(f"directory is not owned by the invoking host user: {path}") - os.fchmod(descriptor, 0o700) - finally: - os.close(descriptor) - - -def _write_private_text(path: Path, content: str) -> None: - """Atomically publish a host-created evidence artifact with owner-only access.""" - descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) - temporary_path = Path(temporary_name) - try: - os.fchmod(descriptor, 0o600) - temporary_file = os.fdopen(descriptor, "w", encoding="utf-8") - descriptor = -1 - with temporary_file: - temporary_file.write(content) - os.replace(temporary_path, path) - finally: - if descriptor != -1: - with contextlib.suppress(OSError): - os.close(descriptor) - temporary_path.unlink(missing_ok=True) - - -def _read_private_final_output(path: Path, *, fallback: str) -> str: - """Read a regular agent-created final output without following it, then republish it privately.""" - try: - descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) - except FileNotFoundError: - _write_private_text(path, fallback) - return fallback - - try: - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise PermissionError(f"final output is not a regular file: {path}") - with os.fdopen(descriptor, "r", encoding="utf-8") as output_file: - descriptor = -1 - output_text = output_file.read() - finally: - if descriptor != -1: - os.close(descriptor) - - _write_private_text(path, output_text) - return output_text - - -def _validate_private_tree(root: Path) -> None: - """Require a host-owned, owner-only tree without following agent-created symlinks.""" - expected_uid = os.getuid() - pending = [root] - while pending: - path = pending.pop() - path_stat = path.lstat() - if stat.S_ISLNK(path_stat.st_mode): - continue - if path_stat.st_uid != expected_uid: - raise PermissionError(f"artifact is not owned by the invoking host user: {path}") - - mode = stat.S_IMODE(path_stat.st_mode) - if mode & 0o077: - raise PermissionError(f"artifact grants group or other access: {path} ({mode:o})") - if stat.S_ISDIR(path_stat.st_mode): - if mode & 0o700 != 0o700: - raise PermissionError(f"directory is not owner-readable, writable, and traversable: {path} ({mode:o})") - with os.scandir(path) as entries: - pending.extend(Path(entry.path) for entry in entries) - elif stat.S_ISREG(path_stat.st_mode): - if mode & 0o600 != 0o600: - raise PermissionError(f"file is not owner-readable and writable: {path} ({mode:o})") - else: - raise PermissionError(f"artifact is not a regular file or directory: {path}") - - -async def _terminate_process(process: Any | None) -> None: - if process is None or process.returncode is not None: - return - process.kill() - with contextlib.suppress(Exception): - await process.wait() - - -def _decode_process_output(value: bytes | str | None) -> str: - if value is None: - return "" - if isinstance(value, str): - return value - return value.decode("utf-8", errors="replace") - - -def _safe_path_name(value: str) -> str: - return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/runtime.py index 053011d761..2b295d3c54 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/runtime.py @@ -71,8 +71,8 @@ def __init__(self, *, config: GymRuntimeConfig) -> None: def config(self) -> GymRuntimeConfig: """The settings this runner was constructed with. - Read-only, and the whole config rather than a property per field: unlike the Codex and - Fabric runtimes, everything shaping a Gym run already lives in one validated object. + Read-only, and the whole config rather than a property per field: unlike the Fabric + runtimes, everything shaping a Gym run already lives in one validated object. Exposed so a live runner can be described as the job-spec target that reproduces it, without reaching into a private attribute from another package. ``runner_info()`` cannot serve that diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py deleted file mode 100644 index d7536db36f..0000000000 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Exercise the customer-facing Docker Codex evidence example.""" - -from __future__ import annotations - -import importlib.util -from pathlib import Path - -import pytest -from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo -from nemo_evaluator_sdk.execution.samples import build_metric_input -from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor - -_MODULE_PATH = Path(__file__).resolve().parents[2] / "examples" / "codex_docker" / "example.py" -_spec = importlib.util.spec_from_file_location("codex_docker_example", _MODULE_PATH) -assert _spec is not None and _spec.loader is not None -codex_docker = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(codex_docker) - - -class _FakeCodexRuntime: - def __init__(self, workspace: Path) -> None: - self._workspace = workspace - - def runner_info(self) -> RunnerInfo: - return RunnerInfo(name="fake_codex", kind="runner") - - async def run_tasks( - self, - tasks: list[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> list[AgentEvalTrial]: - task = tasks[0] - artifact = self._workspace / codex_docker.ARTIFACT_PATH - artifact.parent.mkdir(parents=True) - artifact.write_text(f"{codex_docker.EXPECTED_TEXT}\n", encoding="utf-8") - return [ - AgentEvalTrial( - id=f"{task.id}:fake-codex", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=codex_docker.EXPECTED_TEXT, - metadata={"runtime": "fake_codex_docker"}, - ), - evidence=CandidateEvidence( - descriptors={ - "workspace": EvidenceDescriptor(kind="filesystem", ref=str(self._workspace)), - } - ), - metadata={"runtime": "fake_codex_docker", "agent_ok": True}, - ) - ] - - -def test_default_output_dir_is_under_repo_temp(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv(codex_docker.OUTPUT_ROOT_ENV_NAME, raising=False) - - output_dir = codex_docker._new_output_dir() - - assert output_dir.parent == codex_docker.REPO_ROOT / "temp" / "codex-docker-eval-output" - - -@pytest.mark.asyncio -async def test_codex_docker_example_scores_workspace_artifact(tmp_path: Path) -> None: - result, location = await codex_docker.evaluate( - output_dir=tmp_path / "run", - runtime=_FakeCodexRuntime(tmp_path / "workspace"), - write_dashboard=False, - ) - - assert result.trials[0].status is AgentEvalTrialStatus.COMPLETED - assert { - f"{score.metric_type}.{output.name}": output.value for score in result.scores for output in score.outputs - } == { - "workspace_artifact.artifact_matches": True, - "workspace_artifact.output_matches": True, - } - assert (tmp_path / "run" / "run.json").is_file() - assert location.output_dir == tmp_path / "run" - assert location.dashboard_path is None # write_dashboard=False - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("answer", "artifact", "expected_outputs"), - [ - (codex_docker.EXPECTED_TEXT, codex_docker.EXPECTED_TEXT, [True, True]), - ("wrong output", codex_docker.EXPECTED_TEXT, [False, True]), - (codex_docker.EXPECTED_TEXT, "wrong artifact", [True, False]), - ], -) -async def test_workspace_artifact_metric( - tmp_path: Path, - answer: str, - artifact: str, - expected_outputs: list[bool], -) -> None: - workspace = tmp_path / "workspace" - artifact_path = workspace / codex_docker.ARTIFACT_PATH - artifact_path.parent.mkdir(parents=True) - artifact_path.write_text(artifact, encoding="utf-8") - evidence = CandidateEvidence(descriptors={"workspace": EvidenceDescriptor(kind="filesystem", ref=str(workspace))}) - - metric_result = await codex_docker.WorkspaceArtifactMetric().compute_scores( - build_metric_input( - { - "reference": { - "artifact_path": codex_docker.ARTIFACT_PATH, - "expected": codex_docker.EXPECTED_TEXT, - } - }, - {"output_text": answer, "evidence": evidence}, - index=0, - ) - ) - - assert [output.value for output in metric_result.outputs] == expected_outputs diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py deleted file mode 100644 index 711044e693..0000000000 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py +++ /dev/null @@ -1,822 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import asyncio -import json -import os -import stat -import threading -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -import pytest -from nemo_evaluator_sdk.agent_eval import workspace_seeds -from nemo_evaluator_sdk.agent_eval.runtimes.codex import runtime as codex_runtime -from nemo_evaluator_sdk.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime -from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from pydantic import BaseModel - -# The runtime *selection* (local vs docker-cli vs docker-sandbox) is generic and lives here; only the -# ProfBench ``score_source`` labels + candidate prompt live in the example (test_profbench_codex_target). - - -def _prompt_builder(task: AgentEvalTask) -> str: - return f"do: {task.id}\n" - - -def test_resolve_codex_runtime_local_cli_threads_prompt_builder(tmp_path: Path) -> None: - target, effective = codex_runtime.resolve_codex_runtime( - runtime=codex_runtime.RuntimeChoice.LOCAL, - model="gpt-5", - output_dir=tmp_path / "run", - env={"OPENAI_API_KEY": "sk-test-key"}, - prompt_builder=_prompt_builder, - ) - - assert isinstance(target, codex_runtime.CodexCliAgentRuntime) - assert target._model == "gpt-5" - assert target._work_root == tmp_path / "run" / "evidence" / "codex" - assert target._prompt_builder is _prompt_builder - assert effective == codex_runtime.EffectiveCodexRuntime.LOCAL_CLI - - -def test_resolve_codex_runtime_docker_uses_sandbox_for_openai_secret_key(tmp_path: Path) -> None: - target, effective = codex_runtime.resolve_codex_runtime( - runtime=codex_runtime.RuntimeChoice.DOCKER, - model=None, - output_dir=tmp_path / "run", - env={"OPENAI_API_KEY": "sk-test-key"}, - ) - - assert isinstance(target, DockerSandboxAgentRuntime) - assert target._model == codex_runtime.DEFAULT_CODEX_DOCKER_MODEL - assert effective == codex_runtime.EffectiveCodexRuntime.DOCKER_SANDBOX - - -def test_resolve_codex_runtime_docker_falls_back_to_cli_without_sdk_key(tmp_path: Path) -> None: - target, effective = codex_runtime.resolve_codex_runtime( - runtime=codex_runtime.RuntimeChoice.DOCKER, - model="gpt-5.4", - output_dir=tmp_path / "run", - env={"OPENAI_API_KEY": "oauth-token"}, - prompt_builder=_prompt_builder, - ) - - assert isinstance(target, codex_runtime.CodexDockerCliAgentRuntime) - assert target._work_root == tmp_path / "run" / "evidence" / "codex-docker" - assert target._prompt_builder is _prompt_builder - assert effective == codex_runtime.EffectiveCodexRuntime.DOCKER_CLI - - -def test_evidence_dir_prefers_an_explicit_work_root_over_the_run_config(tmp_path: Path) -> None: - # An explicit ``work_root`` is a caller decision about where evidence goes, so it wins over the - # run's ``work_dir``; without one the runtime derives ``/evidence/codex``. Pointing - # ``work_root`` outside ``work_dir`` is legal and keeps working -- persist() leaves such refs - # absolute (test_persist_and_read_keep_external_evidence_refs_absolute) rather than dropping them, - # so the bundle resolves for as long as that directory is there. - task = AgentEvalTask(id="taskA", intent="do a thing", inputs={}) - config = AgentEvalRunConfig(work_dir=tmp_path / "run") - - explicit = codex_runtime.CodexCliAgentRuntime(work_root=tmp_path / "elsewhere") - assert explicit._evidence_dir(0, task, config) == tmp_path / "elsewhere" / "000000-taskA" - - derived = codex_runtime.CodexCliAgentRuntime() - assert derived._evidence_dir(0, task, config) == tmp_path / "run" / "evidence" / "codex" / "000000-taskA" - - -def test_list_codex_agent_models_prints_visible_models( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - class FakeCompletedProcess: - stdout = json.dumps( - { - "models": [ - {"slug": "hidden", "display_name": "Hidden", "visibility": "hidden", "priority": 99}, - {"slug": "gpt-5.4-mini", "display_name": "GPT-5.4 Mini", "visibility": "list", "priority": 3}, - {"slug": "gpt-5.5", "display_name": "GPT-5.5", "visibility": "list", "priority": 9}, - ] - } - ) - - def fake_run(command: list[str], check: bool, capture_output: bool, text: bool) -> FakeCompletedProcess: - assert command == ["codex", "debug", "models"] - assert check is True - assert capture_output is True - assert text is True - return FakeCompletedProcess() - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - monkeypatch.setattr(codex_runtime.subprocess, "run", fake_run) - - codex_runtime.print_codex_agent_models() - - assert capsys.readouterr().out.splitlines() == [ - "gpt-5.5\tGPT-5.5", - "gpt-5.4-mini\tGPT-5.4 Mini", - ] - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_uses_local_codex_command_and_writes_evidence( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - commands: list[tuple[tuple[str, ...], dict[str, Any]]] = [] - - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - # Default prompt is exactly the instruction from inputs — no runtime framing — and never - # leaks the eval-side `intent`. - assert input == b"Question?" - assert b"Answer." not in input # intent stays eval-side - final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) - final_output_path.write_text("codex answer", encoding="utf-8") - return b'{"type":"event"}\n', b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - evidence_dir = Path(command[command.index("--output-last-message") + 1]).parent - assert stat.S_IMODE(evidence_dir.parent.stat().st_mode) == 0o700 - assert stat.S_IMODE(evidence_dir.stat().st_mode) == 0o700 - assert stat.S_IMODE((evidence_dir / "workspace").stat().st_mode) == 0o700 - assert stat.S_IMODE((evidence_dir / "task.json").stat().st_mode) == 0o600 - assert stat.S_IMODE((evidence_dir / "prompt.txt").stat().st_mode) == 0o600 - commands.append((command, kwargs)) - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime( - model="gpt-5", - work_root=tmp_path / "codex", - process_factory=fake_process_factory, - ) - task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Question?"}) - - trials = await runtime.run_tasks([task]) - - command, kwargs = commands[0] - assert command[:2] == ("codex", "exec") - assert "--ephemeral" in command - assert "--ignore-user-config" in command - assert "--skip-git-repo-check" in command - assert command[command.index("--model") + 1] == "gpt-5" - assert command[-1] == "-" - assert kwargs["stdin"] == codex_runtime.subprocess.PIPE - assert trials[0].status == "completed" - assert trials[0].output is not None - assert trials[0].output.output_text == "codex answer" - assert trials[0].evidence is not None - assert trials[0].evidence.require("workspace", kind="filesystem").ref == str( - tmp_path / "codex" / "000000-task-1" / "workspace" - ) - final_output = tmp_path / "codex" / "000000-task-1" / "final_output.txt" - assert final_output.read_text(encoding="utf-8") == "codex answer" - assert final_output.stat().st_mode & 0o777 == 0o600 - assert (tmp_path / "codex" / "000000-task-1" / "stdout.jsonl").read_text(encoding="utf-8") == '{"type":"event"}\n' - assert (tmp_path / "codex" / "000000-task-1" / "stdout.jsonl").stat().st_mode & 0o777 == 0o600 - assert (tmp_path / "codex" / "000000-task-1" / "stderr.txt").stat().st_mode & 0o777 == 0o600 - - -@pytest.mark.asyncio -async def test_codex_task_json_omits_grader_only_fields(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - # The docker variant mounts the evidence dir into the sandbox (danger-full-access), so the persisted - # task.json must never carry grader-only fields — otherwise the agent could read `intent` (desired - # behavior) or the held-out `reference` back out of /evidence and reward-hack. Enforced on the shared - # base runtime so both the local and docker variants persist an agent-safe task.json. - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) - final_output_path.write_text("ok", encoding="utf-8") - return b"", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime(work_root=tmp_path / "codex", process_factory=fake_process_factory) - task = AgentEvalTask( - id="task/1", - intent="SECRET_GRADER_INTENT", - inputs={"instruction": "do the thing"}, - reference={"expected": "HELD_OUT_GROUND_TRUTH"}, - ) - - await runtime.run_tasks([task]) - - task_json = (tmp_path / "codex" / "000000-task-1" / "task.json").read_text(encoding="utf-8") - assert "SECRET_GRADER_INTENT" not in task_json # intent is eval-side desired-behavior metadata - assert "HELD_OUT_GROUND_TRUTH" not in task_json # reference is grader-only ground truth - assert '"intent"' not in task_json and '"reference"' not in task_json # dropped entirely, not just empty - assert "do the thing" in task_json # agent-safe fields (id, inputs) are still persisted - - -@pytest.mark.asyncio -async def test_codex_docker_cli_agent_runtime_runs_codex_in_container_and_writes_evidence( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - auth_path = tmp_path / "auth.json" - auth_path.write_text("{}", encoding="utf-8") - commands: list[tuple[tuple[str, ...], dict[str, Any]]] = [] - - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - assert input == b"Question?" # prompt is the instruction verbatim - evidence_mount = self.command[self.command.index(f"{auth_path.resolve()}:/root/.codex/auth.json:ro") + 4] - evidence_dir = Path(evidence_mount.split(":/evidence", maxsplit=1)[0]) - (evidence_dir / "final_output.txt").write_text("docker codex answer", encoding="utf-8") - for directory, _subdirectories, filenames in os.walk(evidence_dir): - Path(directory).chmod(0o700) - for filename in filenames: - (Path(directory) / filename).chmod(0o600) - return b'{"type":"event"}\n', b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - commands.append((command, kwargs)) - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexDockerCliAgentRuntime( - model="gpt-5.4", - work_root=tmp_path / "codex-docker", - auth_path=auth_path, - process_factory=fake_process_factory, - ) - task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Question?"}) - - trials = await runtime.run_tasks([task]) - - command, kwargs = commands[0] - assert command[:4] == ("docker", "run", "--rm", "-i") - assert command[command.index("-e") + 1] == "PYTHONDONTWRITEBYTECODE=1" - assert f"{auth_path.resolve()}:/root/.codex/auth.json:ro" in command - assert f"{(tmp_path / 'codex-docker' / '000000-task-1' / 'workspace').resolve()}:/workspace" in command - assert f"{(tmp_path / 'codex-docker' / '000000-task-1').resolve()}:/evidence" in command - assert command[-3:] == ("sh", "-lc", command[-1]) - assert f"npx -y {codex_runtime.DEFAULT_CODEX_DOCKER_CLI_PACKAGE} exec" in command[-1] - assert "--sandbox danger-full-access" in command[-1] - assert "--model gpt-5.4" in command[-1] - assert "host_owner=\"$(stat -c '%u:%g' /evidence 2>/dev/null)\" || true" in command[-1] - assert command[-1].index("host_owner=") < command[-1].index("npx -y") - assert "codex_status=$?" in command[-1] - assert 'chown -R "$host_owner" /workspace /evidence 2>/dev/null || true' in command[-1] - assert "chmod -R u+rwX,go-rwx /workspace /evidence" in command[-1] - assert 'if [ "$codex_status" -ne 0 ]; then exit "$codex_status"; fi' in command[-1] - assert 'exit "$permissions_status"' in command[-1] - assert command[-1].index('if [ "$codex_status"') < command[-1].index('exit "$permissions_status"') - assert kwargs["stdin"] == codex_runtime.subprocess.PIPE - assert trials[0].status == "completed" - assert trials[0].output is not None - assert trials[0].output.output_text == "docker codex answer" - assert trials[0].metadata["runtime"] == "codex_docker_cli" - assert trials[0].evidence is not None - assert trials[0].evidence.metadata["runtime"] == "codex_docker_cli" - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_kills_process_on_timeout( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - class FakeProcess: - def __init__(self) -> None: - self.returncode: int | None = None - self.killed = False - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - return b"", b"" - - def kill(self) -> None: - self.killed = True - self.returncode = -9 - - async def wait(self) -> int: - return self.returncode or 0 - - process = FakeProcess() - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return process - - async def fake_wait_for(awaitable: Any, timeout: float) -> Any: - awaitable.close() - raise TimeoutError - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - monkeypatch.setattr(codex_runtime.asyncio, "wait_for", fake_wait_for) - runtime = codex_runtime.CodexCliAgentRuntime( - work_root=tmp_path / "codex", - process_factory=fake_process_factory, - ) - task = AgentEvalTask(id="task-timeout", intent="Answer.", inputs={"instruction": "Q?"}) - - trials = await runtime.run_tasks([task]) - - assert process.killed is True - assert trials[0].status == "failed" - assert trials[0].output is None - assert trials[0].metadata["error_type"] == "TimeoutError" - assert (tmp_path / "codex" / "000000-task-timeout" / "error.json").stat().st_mode & 0o777 == 0o600 - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_falls_back_to_stdout_and_persists_final_output( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - class FakeProcess: - returncode = 0 - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - return b"stdout fallback\n", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess() - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime( - work_root=tmp_path / "codex", - process_factory=fake_process_factory, - ) - task = AgentEvalTask(id="task-2", intent="Answer.", inputs={"instruction": "Q?"}) - - trials = await runtime.run_tasks([task]) - - assert trials[0].status == "completed" - assert trials[0].output is not None - assert trials[0].output.output_text == "stdout fallback\n" - final_output = tmp_path / "codex" / "000000-task-2" / "final_output.txt" - assert final_output.read_text(encoding="utf-8") == "stdout fallback\n" - assert final_output.stat().st_mode & 0o777 == 0o600 - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_rejects_agent_created_final_output_symlink( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - external = tmp_path / "external.txt" - external.write_text("secret", encoding="utf-8") - - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) - final_output_path.symlink_to(external) - return b"stdout fallback\n", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime(work_root=tmp_path / "codex", process_factory=fake_process_factory) - task = AgentEvalTask(id="task-symlink", intent="Answer.", inputs={"instruction": "Q?"}) - - trials = await runtime.run_tasks([task]) - - assert trials[0].status == "failed" - assert trials[0].output is None - assert trials[0].metadata["error_type"] == "OSError" - assert external.read_text(encoding="utf-8") == "secret" - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_rejects_agent_created_final_output_fifo( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) - os.mkfifo(final_output_path, 0o600) - final_output_path.chmod(0o600) - return b"stdout fallback\n", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime(work_root=tmp_path / "codex", process_factory=fake_process_factory) - task = AgentEvalTask(id="task-fifo", intent="Answer.", inputs={"instruction": "Q?"}) - - trials = await asyncio.wait_for(runtime.run_tasks([task]), timeout=5) - - assert trials[0].status == "failed" - assert trials[0].output is None - assert trials[0].metadata["error_type"] == "PermissionError" - - -def test_private_directory_creation_repairs_modes_and_rejects_unsafe_paths( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - private_dir = tmp_path / "private" - private_dir.mkdir(mode=0o755) - - codex_runtime._ensure_private_directory(private_dir) - - assert stat.S_IMODE(private_dir.stat().st_mode) == 0o700 - - symlink = tmp_path / "symlink" - symlink.symlink_to(private_dir, target_is_directory=True) - with pytest.raises(OSError): - codex_runtime._ensure_private_directory(symlink) - - not_a_directory = tmp_path / "file" - not_a_directory.touch() - with pytest.raises(FileExistsError): - codex_runtime._ensure_private_directory(not_a_directory) - - different_uid = os.getuid() + 1 - monkeypatch.setattr(codex_runtime.os, "getuid", lambda: different_uid) - with pytest.raises(PermissionError, match="not owned"): - codex_runtime._ensure_private_directory(private_dir) - - -def test_private_text_write_is_owner_only_atomic_and_replaces_symlinks( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - external = tmp_path / "external.txt" - external.write_text("external", encoding="utf-8") - target = tmp_path / "artifact.txt" - target.symlink_to(external) - original_replace = codex_runtime.os.replace - - def checked_replace(source: str | Path, destination: str | Path) -> None: - assert stat.S_IMODE(Path(source).stat().st_mode) == 0o600 - original_replace(source, destination) - - monkeypatch.setattr(codex_runtime.os, "replace", checked_replace) - - codex_runtime._write_private_text(target, "private") - - assert not target.is_symlink() - assert target.read_text(encoding="utf-8") == "private" - assert external.read_text(encoding="utf-8") == "external" - assert stat.S_IMODE(target.stat().st_mode) == 0o600 - - -def test_private_text_write_cleans_temporary_file_on_replace_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - target = tmp_path / "artifact.txt" - explicitly_closed: list[int] = [] - original_close = codex_runtime.os.close - - def fail_replace(source: str | Path, destination: str | Path) -> None: - raise OSError("replace failed") - - def track_close(descriptor: int) -> None: - explicitly_closed.append(descriptor) - original_close(descriptor) - - monkeypatch.setattr(codex_runtime.os, "replace", fail_replace) - monkeypatch.setattr(codex_runtime.os, "close", track_close) - - with pytest.raises(OSError, match="replace failed"): - codex_runtime._write_private_text(target, "private") - - assert explicitly_closed == [] - assert list(tmp_path.glob(".artifact.txt.*.tmp")) == [] - - -def test_private_text_write_closes_descriptor_when_fdopen_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - target = tmp_path / "artifact.txt" - explicitly_closed: list[int] = [] - original_close = codex_runtime.os.close - - def fail_fdopen(descriptor: int, mode: str, *, encoding: str) -> None: - raise OSError("fdopen failed") - - def track_close(descriptor: int) -> None: - explicitly_closed.append(descriptor) - original_close(descriptor) - - monkeypatch.setattr(codex_runtime.os, "fdopen", fail_fdopen) - monkeypatch.setattr(codex_runtime.os, "close", track_close) - - with pytest.raises(OSError, match="fdopen failed"): - codex_runtime._write_private_text(target, "private") - - assert len(explicitly_closed) == 1 - assert list(tmp_path.glob(".artifact.txt.*.tmp")) == [] - - -@pytest.mark.asyncio -async def test_setup_failure_does_not_write_error_through_untrusted_evidence_symlink( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - work_root = tmp_path / "codex" - work_root.mkdir(mode=0o700) - external = tmp_path / "external" - external.mkdir() - evidence_dir = work_root / "000000-task-symlink" - evidence_dir.symlink_to(external, target_is_directory=True) - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime(work_root=work_root) - task = AgentEvalTask(id="task-symlink", intent="Answer.", inputs={"instruction": "Q?"}) - - trial = (await runtime.run_tasks([task]))[0] - - assert trial.status == "failed" - assert trial.output is None - assert trial.evidence is None - assert not (external / "error.json").exists() - - -def test_private_tree_validation_is_root_inclusive_and_does_not_follow_symlinks( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - root = tmp_path / "evidence" - root.mkdir(mode=0o700) - nested = root / "nested" - nested.mkdir(mode=0o700) - regular = nested / "regular.txt" - regular.write_text("ok", encoding="utf-8") - regular.chmod(0o600) - executable = nested / "script.sh" - executable.write_text("#!/bin/sh\n", encoding="utf-8") - executable.chmod(0o700) - (nested / "host-link").symlink_to("/etc/passwd") - - codex_runtime._validate_private_tree(root) - - regular.chmod(0o640) - with pytest.raises(PermissionError, match="group or other"): - codex_runtime._validate_private_tree(root) - regular.chmod(0o400) - with pytest.raises(PermissionError, match="owner-readable and writable"): - codex_runtime._validate_private_tree(root) - regular.chmod(0o600) - root.chmod(0o750) - with pytest.raises(PermissionError, match="group or other"): - codex_runtime._validate_private_tree(root) - root.chmod(0o700) - different_uid = os.getuid() + 1 - monkeypatch.setattr(codex_runtime.os, "getuid", lambda: different_uid) - with pytest.raises(PermissionError, match="not owned"): - codex_runtime._validate_private_tree(root) - - -def test_private_tree_validation_rejects_special_files(tmp_path: Path) -> None: - root = tmp_path / "evidence" - root.mkdir(mode=0o700) - fifo = root / "agent.fifo" - os.mkfifo(fifo, 0o600) - fifo.chmod(0o600) - - with pytest.raises(PermissionError, match="not a regular file or directory"): - codex_runtime._validate_private_tree(root) - - -@pytest.mark.asyncio -async def test_codex_success_fails_when_permission_postcondition_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - auth_path = tmp_path / "auth.json" - auth_path.write_text("{}", encoding="utf-8") - - class FakeProcess: - returncode = 0 - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - return b"", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess() - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexDockerCliAgentRuntime( - work_root=tmp_path / "codex-docker", - auth_path=auth_path, - process_factory=fake_process_factory, - ) - - def fail_validation(evidence_dir: Path) -> None: - raise PermissionError("unsafe evidence") - - monkeypatch.setattr(runtime, "_validate_artifact_permissions", fail_validation) - task = AgentEvalTask(id="success", intent="Succeed.", inputs={"instruction": "succeed"}) - - (trial,) = await runtime.run_tasks([task]) - - assert trial.status == "failed" - assert trial.metadata["error_type"] == "PermissionError" - assert "unsafe evidence" in trial.metadata["permission_cleanup_error"] - - -@pytest.mark.asyncio -async def test_codex_failure_preserves_status_and_reports_permission_cleanup_error( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - auth_path = tmp_path / "auth.json" - auth_path.write_text("{}", encoding="utf-8") - - class FakeProcess: - returncode = 23 - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - return b"", b"codex failed" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess() - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexDockerCliAgentRuntime( - work_root=tmp_path / "codex-docker", - auth_path=auth_path, - process_factory=fake_process_factory, - ) - - def fail_validation(evidence_dir: Path) -> None: - raise PermissionError("unsafe evidence") - - monkeypatch.setattr(runtime, "_validate_artifact_permissions", fail_validation) - task = AgentEvalTask(id="failure", intent="Fail.", inputs={"instruction": "fail"}) - - (trial,) = await runtime.run_tasks([task]) - - assert trial.status == "failed" - assert "status 23" in trial.metadata["error"] - assert "unsafe evidence" in trial.metadata["permission_cleanup_error"] - - -def test_failed_trial_survives_inaccessible_error_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - task = AgentEvalTask(id="failure", intent="Fail.", inputs={"instruction": "fail"}) - - def fail_write(path: Path, content: str) -> None: - raise PermissionError("inaccessible") - - monkeypatch.setattr(codex_runtime, "_write_private_text", fail_write) - - trial = codex_runtime._failed_codex_trial(task, tmp_path / "missing", RuntimeError("original")) - - assert trial.status == "failed" - assert trial.evidence is None - assert trial.metadata["error"] == "original" - assert "inaccessible" in trial.metadata["error_artifact_error"] - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_seeds_workspace_and_stamps_agent_ok( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - # Seed files are staged into the workspace before the agent runs. - workspace_dir = Path(self.command[self.command.index("--cd") + 1]) - assert (workspace_dir / "buggy.py").read_text(encoding="utf-8") == "def add(a, b)\n return a + b\n" - final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) - final_output_path.write_text("fixed it", encoding="utf-8") - return b"", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime( - work_root=tmp_path / "codex", - process_factory=fake_process_factory, - ) - task = AgentEvalTask( - id="fix-bug", - intent="Fix the syntax error.", - inputs={"instruction": "fix the bug", "files": {"buggy.py": "def add(a, b)\n return a + b\n"}}, - ) - - trials = await runtime.run_tasks([task]) - - assert trials[0].status == "completed" - # agent_ok is stamped so AgentPhaseSuccessMetric works over Codex trials. - assert trials[0].metadata["agent_ok"] is True - assert trials[0].metadata["seeded_files"] == ["buggy.py"] - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_seeds_off_the_event_loop_thread( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - # Seeding is synchronous and a handler may block (e.g. the plugin's fileset download). It runs on - # the event loop shared by every concurrent task, so it must be offloaded to a worker thread — - # otherwise one slow seed stalls the whole run. Register a probe handler that records the thread - # it resolves on and assert it is not the loop thread. - class _ProbeSeed(BaseModel): - kind: str = "thread_probe" - - class _ProbeHandler: - kind = "thread_probe" - resolved_on: int | None = None - - def parse(self, value: Mapping[str, Any]) -> BaseModel: - return _ProbeSeed() - - def resolve(self, seed: BaseModel) -> bytes: - _ProbeHandler.resolved_on = threading.get_ident() - return b"probe" - - monkeypatch.setitem(workspace_seeds._HANDLERS, "thread_probe", _ProbeHandler()) - - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) - final_output_path.write_text("ok", encoding="utf-8") - return b"", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime( - work_root=tmp_path / "codex", - process_factory=fake_process_factory, - ) - task = AgentEvalTask( - id="probe", intent="probe", inputs={"instruction": "run", "files": {"p.txt": {"kind": "thread_probe"}}} - ) - - trials = await runtime.run_tasks([task]) - - assert trials[0].status == "completed" - assert _ProbeHandler.resolved_on is not None - assert _ProbeHandler.resolved_on != threading.get_ident() - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_rejects_seed_path_escaping_workspace( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - async def fake_process_factory(*command: str, **kwargs: Any) -> Any: # pragma: no cover - never reached - raise AssertionError("agent should not run when seeding fails") - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime( - work_root=tmp_path / "codex", - process_factory=fake_process_factory, - ) - task = AgentEvalTask(id="evil", intent="escape", inputs={"files": {"../escape.txt": "x"}}) - - # A traversal path is surfaced as a failed trial (the exception is caught per-task). - trials = await runtime.run_tasks([task]) - assert trials[0].status == "failed" - assert trials[0].metadata["error_type"] == "WorkspaceSeedError" - assert trials[0].metadata["agent_ok"] is False - - -@pytest.mark.asyncio -async def test_codex_cli_agent_runtime_uses_injected_prompt_builder( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - assert input == b"CUSTOM: fix-bug\n" - final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) - final_output_path.write_text("ok", encoding="utf-8") - return b"", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime( - work_root=tmp_path / "codex", - prompt_builder=lambda task: f"CUSTOM: {task.id}\n", - process_factory=fake_process_factory, - ) - task = AgentEvalTask(id="fix-bug", intent="Fix.", inputs={}) - - trials = await runtime.run_tasks([task]) - assert trials[0].status == "completed" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime_live.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime_live.py deleted file mode 100644 index 2250d73ae2..0000000000 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime_live.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Live Docker regressions for host-readable Codex CLI evidence.""" - -from __future__ import annotations - -import os -import shutil -import stat -import subprocess -from pathlib import Path - -import pytest -from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexDockerCliAgentRuntime -from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask - -_IMAGE = "node:22-alpine" -_FAKE_PACKAGE_JSON = """{ - "name": "fake-codex", - "version": "1.0.0", - "bin": {"fake-codex": "fake-codex.js"} -} -""" -_FAKE_CODEX_JS = """#!/usr/bin/env node -const fs = require("fs"); -const path = require("path"); - -function argumentValue(name) { - const index = process.argv.indexOf(name); - if (index < 0 || index + 1 >= process.argv.length) { - throw new Error(`missing ${name}`); - } - return process.argv[index + 1]; -} - -const workspace = argumentValue("--cd"); -const finalOutput = argumentValue("--output-last-message"); -const cacheDir = path.join(workspace, "__pycache__"); -const cacheFile = path.join(cacheDir, "probe.pyc"); -fs.mkdirSync(cacheDir, {recursive: true}); -fs.writeFileSync(cacheFile, "fake bytecode"); -fs.writeFileSync(finalOutput, "fake codex answer"); -fs.symlinkSync("/etc/passwd", path.join(workspace, "host-link")); -fs.chmodSync(cacheFile, 0o000); -fs.chmodSync(cacheDir, 0o000); -fs.chmodSync(finalOutput, 0o000); - -const exitCodePath = path.join(workspace, "exit-code.txt"); -const exitCode = fs.existsSync(exitCodePath) ? Number(fs.readFileSync(exitCodePath, "utf8")) : 0; -process.exit(exitCode); -""" - - -def _docker_ready() -> bool: - if shutil.which("docker") is None: - return False - try: - return subprocess.run(["docker", "info"], capture_output=True, timeout=15).returncode == 0 - except subprocess.TimeoutExpired: - return False - - -pytestmark = pytest.mark.skipif(not _docker_ready(), reason="docker daemon not available") - - -def _fake_codex_task(task_id: str, *, exit_code: int | None = None) -> AgentEvalTask: - files = { - "package.json": _FAKE_PACKAGE_JSON, - "fake-codex.js": _FAKE_CODEX_JS, - } - if exit_code is not None: - files["exit-code.txt"] = str(exit_code) - return AgentEvalTask( - id=task_id, - intent="Exercise Docker evidence permissions.", - inputs={"instruction": "Create restrictive evidence.", "files": files}, - ) - - -def _assert_tree_host_private(root: Path) -> None: - assert root.is_dir() - for path in (root, *root.rglob("*")): - if path.is_symlink(): - continue - path_stat = path.stat() - mode = stat.S_IMODE(path_stat.st_mode) - assert path_stat.st_uid == os.getuid(), f"artifact is not owned by the host user: {path}" - assert mode & 0o077 == 0, f"artifact is accessible to group or other users: {path} ({mode:o})" - if path.is_dir(): - assert mode & 0o700 == 0o700, f"directory is not accessible to the host user: {path} ({mode:o})" - assert os.access(path, os.R_OK | os.W_OK | os.X_OK) - elif path.is_file(): - assert mode & 0o600 == 0o600, f"file is not readable and writable by the host user: {path} ({mode:o})" - assert os.access(path, os.R_OK | os.W_OK) - path.read_bytes() - - -async def test_codex_docker_normalizes_concurrent_workspace_and_evidence_trees(tmp_path: Path) -> None: - auth_path = tmp_path / "auth.json" - auth_path.write_text("{}", encoding="utf-8") - work_root = tmp_path / "codex-docker" - runtime = CodexDockerCliAgentRuntime( - work_root=work_root, - image=_IMAGE, - codex_package="/workspace", - auth_path=auth_path, - ) - - trials = await runtime.run_tasks( - [_fake_codex_task("success-a"), _fake_codex_task("success-b")], - AgentEvalRunConfig(parallelism=2), - ) - - assert [trial.status for trial in trials] == ["completed", "completed"] - for trial in trials: - assert trial.output is not None - assert trial.output.output_text == "fake codex answer" - assert trial.evidence is not None - workspace = await trial.evidence.filesystem("workspace") - verifier = await workspace.run_verifier(["test", "!", "-e", "host-link"]) - assert verifier.ok - - evidence_dir = Path(trial.output.metadata["evidence_dir"]) - _assert_tree_host_private(evidence_dir / "workspace") - _assert_tree_host_private(evidence_dir) - copy = tmp_path / "copies" / trial.task_id - shutil.copytree(evidence_dir, copy, symlinks=True) - assert (copy / "workspace" / "host-link").is_symlink() - - assert stat.S_IMODE(work_root.stat().st_mode) == 0o700 - - -async def test_codex_docker_preserves_failure_status_after_permission_normalization(tmp_path: Path) -> None: - auth_path = tmp_path / "auth.json" - auth_path.write_text("{}", encoding="utf-8") - work_root = tmp_path / "codex-docker" - runtime = CodexDockerCliAgentRuntime( - work_root=work_root, - image=_IMAGE, - codex_package="/workspace", - auth_path=auth_path, - ) - - (trial,) = await runtime.run_tasks([_fake_codex_task("failure", exit_code=23)]) - - assert trial.status == "failed" - assert trial.metadata["agent_ok"] is False - assert "status 23" in trial.metadata["error"] - assert "permission_cleanup_error" not in trial.metadata - evidence_dir = work_root / "000000-failure" - _assert_tree_host_private(evidence_dir / "workspace") - _assert_tree_host_private(evidence_dir) - assert (evidence_dir / "final_output.txt").read_text(encoding="utf-8") == "fake codex answer" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py index 798ee054a0..1ba812399d 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py @@ -47,7 +47,7 @@ def test_runner_info_is_required_by_the_runner_contract() -> None: def test_every_shipped_runner_reports_a_stable_name_and_result_shaping_config() -> None: - """All eight shipped runners: a curated name (not a class name) and the settings that change results. + """Every shipped runner: a curated name (not a class name) and the settings that change results. Provenance that omits a result-shaping setting is worse than none — two runs that behaved differently would record identical metadata — so assert each runner surfaces its own knobs. @@ -55,7 +55,6 @@ def test_every_shipped_runner_reports_a_stable_name_and_result_shaping_config() from pathlib import Path from nemo_evaluator_sdk.agent_eval.runtimes.callable_runtime import CallableAgentTaskRunner - from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime, CodexDockerCliAgentRuntime from nemo_evaluator_sdk.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime from nemo_evaluator_sdk.agent_eval.runtimes.fabric.container_runtime import FabricContainerRuntime from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime @@ -74,8 +73,6 @@ class _Provider: harness = {"harness": {"adapter_id": "nvidia.fabric.codex"}} runners = [ (CallableAgentTaskRunner(_agent_fn), "callable", {"agent_fn", "parallelism"}), - (CodexCliAgentRuntime(), "codex_cli", {"model", "timeout_s", "codex_bin", "prompt_builder"}), - (CodexDockerCliAgentRuntime(), "codex_docker_cli", {"model", "timeout_s", "codex_bin", "prompt_builder"}), (DockerSandboxAgentRuntime(), "docker_sandbox", {"model", "image", "timeout_s", "instructions"}), ( FabricAgentRuntime(config=harness), @@ -283,8 +280,8 @@ def test_model_provenance_records_the_endpoint_and_invocation_params() -> None: def _load_example(name: str): """Import an example runtime module. - The example runtimes use relative imports, so they are imported as a package (rather than loaded - by path like the standalone codex_docker example) with the SDK package root on sys.path. + The example runtimes use relative imports, so they are imported as a package with the SDK + package root on sys.path. """ import importlib import sys diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index d23445a7a8..e0d813ff95 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -2098,14 +2098,14 @@ components: anyOf: - $ref: '#/components/schemas/ModelTarget' - $ref: '#/components/schemas/AgentTarget' - - $ref: '#/components/schemas/CodexRunnerTarget' - $ref: '#/components/schemas/FabricRunnerTarget' - $ref: '#/components/schemas/GymRunnerTarget' - $ref: '#/components/schemas/HarborRunnerTarget' title: Target description: 'What generates trials online: a Model or Agent endpoint, or - an agent runner (e.g. Codex CLI). Endpoint targets carry their own request - config (prompt template / inference params). Mutually exclusive with `trials`.' + an agent runner (e.g. a Fabric harness). Endpoint targets carry their + own request config (prompt template / inference params). Mutually exclusive + with `trials`.' trials: title: Trials description: Precomputed trials to score directly (offline eval), instead @@ -2270,14 +2270,14 @@ components: anyOf: - $ref: '#/components/schemas/ModelTarget' - $ref: '#/components/schemas/AgentTarget' - - $ref: '#/components/schemas/CodexRunnerTarget' - $ref: '#/components/schemas/FabricRunnerTarget' - $ref: '#/components/schemas/GymRunnerTarget' - $ref: '#/components/schemas/HarborRunnerTarget' title: Target description: 'What generates trials online: a Model or Agent endpoint, or - an agent runner (e.g. Codex CLI). Endpoint targets carry their own request - config (prompt template / inference params). Mutually exclusive with `trials`.' + an agent runner (e.g. a Fabric harness). Endpoint targets carry their + own request config (prompt template / inference params). Mutually exclusive + with `trials`.' trials: title: Trials description: Precomputed trials to score directly (offline eval), instead @@ -3039,27 +3039,6 @@ components: polymorphically (typed as an abstract base), which renders as an opaque object in the spec; this concrete DTO documents the actual fields.' - CodexRunnerTarget: - properties: - kind: - type: string - const: codex - title: Kind - default: codex - model: - title: Model - description: Codex model to use (e.g. 'gpt-5.5'); CLI default when omitted. - type: string - timeout_s: - type: integer - minimum: 1.0 - title: Timeout S - description: Per-task timeout for the Codex CLI, in seconds. - default: 600 - additionalProperties: false - type: object - title: CodexRunnerTarget - description: Generate trials by driving the Codex CLI agent runner. DatetimeFilter: additionalProperties: false properties: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/entities.py b/plugins/nemo-evaluator/src/nemo_evaluator/entities.py index c7547c62e8..2d673f3e2a 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/entities.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/entities.py @@ -174,7 +174,7 @@ class _EvalResultCommon(BaseModel): job_id: str = Field(description="Identifier of the job run that produced this result (one result per run).") target_kind: str | None = Field( - description="Target discriminator: 'model', 'agent', or a runner kind e.g. 'codex'." + description="Target discriminator: 'model', 'agent', or a runner kind e.g. 'fabric'." ) target_name: str | None = Field(description="Model/agent entity name, or the runner's model — filterable trait.") target_url: str | None = Field(description="Endpoint URL, when the target is an HTTP model/agent.") @@ -246,7 +246,7 @@ class TaskEntity(_RevisionedCommon, EntityBase): A task is an evaluation unit; ``spec`` says what it is and which runner executes it. Both kinds live in one record type so a user manages every evaluation unit in one place, and so a taskset can group them without caring how each one runs — the same way ``AgentRunnerTarget`` already - treats codex/fabric/harbor as members of one union on the target side. + treats fabric/gym/harbor as members of one union on the target side. Content is nested under ``spec`` rather than flattened with nullable per-kind fields, so each variant's required fields stay genuinely required and the revision digest covers the spec as one diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_compiler.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_compiler.py index 137e4bf744..8ec96b21e1 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_compiler.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_compiler.py @@ -6,7 +6,7 @@ Parallels :mod:`nemo_evaluator.jobs.compiler` (row/model eval), emitting a single ``cpu-tasks`` step that runs ``python -m nemo_evaluator.tasks.agent_evaluate`` in the platform task environment. Metric/endpoint secrets are surfaced as -``from_secret`` environment variables; an agent *runner* target (e.g. Codex) +``from_secret`` environment variables; an agent *runner* target (e.g. Fabric) carries no endpoint secret of its own. """ diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index 4327d90f69..51d0e7fdb3 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -4,7 +4,7 @@ """SDK-backed agent-evaluation job for the evaluator plugin. Runs :class:`AgentEvaluator` over a set of tasks against a Model/Agent endpoint -or an agent runner (e.g. Codex CLI), producing an ``AgentEvalResult`` (trials + +or an agent runner (e.g. a Fabric harness), producing an ``AgentEvalResult`` (trials + per-trial scores + summary). The row-based counterpart is :class:`~nemo_evaluator.jobs.evaluate.EvaluateJob`. @@ -31,7 +31,6 @@ AgentEvalSpec, AgentEvalTaskSpec, AgentTarget, - CodexRunnerTarget, FabricRunnerTarget, GymRunnerTarget, HarborRunnerTarget, @@ -45,7 +44,6 @@ from nemo_evaluator.task_refs import resolve_agent_eval_tasks from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult -from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime from nemo_evaluator_sdk.agent_eval.runtimes.gym import GymAgentTaskRunner, GymRuntimeConfig from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner, HarborRuntimeConfig @@ -336,13 +334,6 @@ def _resolve_target( return target.model, target.prompt_template, target.params or RunConfigOnlineModel() if isinstance(target, AgentTarget): return target.agent, None, target.params or RunConfigOnline() - if isinstance(target, CodexRunnerTarget): - runtime = CodexCliAgentRuntime( - model=target.model, - timeout_s=target.timeout_s, - work_root=ctx.storage.persistent / "codex", - ) - return runtime, None, None if isinstance(target, FabricRunnerTarget): fabric_runtime = FabricAgentRuntime( config=target.config, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py index 6df82e9b06..e42bc06696 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py @@ -66,18 +66,6 @@ class AgentTarget(BaseModel): ) -class CodexRunnerTarget(BaseModel): - """Generate trials by driving the Codex CLI agent runner.""" - - model_config = ConfigDict(extra="forbid") - - kind: Literal["codex"] = "codex" - model: str | None = Field( - default=None, description="Codex model to use (e.g. 'gpt-5.5'); CLI default when omitted." - ) - timeout_s: int = Field(default=600, ge=1, description="Per-task timeout for the Codex CLI, in seconds.") - - class FabricRunnerTarget(BaseModel): """Generate trials by driving an agent harness through the NeMo Fabric runtime. @@ -209,7 +197,7 @@ class GymRunnerTarget(BaseModel): #: The agent-runner slot of the target union — the spec-side mirror of ``AgentTaskRunner``, resolved #: to a runtime at run time. ``kind``-discriminated; widen with more members as runners land. -AgentRunnerTarget: TypeAlias = CodexRunnerTarget | FabricRunnerTarget | GymRunnerTarget | HarborRunnerTarget +AgentRunnerTarget: TypeAlias = FabricRunnerTarget | GymRunnerTarget | HarborRunnerTarget #: What generates trials: a Model or Agent endpoint, or an agent runner. ``kind``-discriminated, and #: the spec-level analog of the SDK's runtime ``AgentEvalTarget`` (Model | Agent | AgentTaskRunner). @@ -239,7 +227,7 @@ def target_agent_identity(target: Target | Model | AgentBase | None) -> tuple[st return target.agent, None if isinstance(target, ModelTarget): return None, target.model.name - if isinstance(target, CodexRunnerTarget | FabricRunnerTarget): + if isinstance(target, FabricRunnerTarget): return None, target.model # Bare SDK values, as carried by the dataset-driven eval spec. if isinstance(target, AgentBase): @@ -313,9 +301,9 @@ class _AgentEvalSpecCommon(BaseModel): target: Target | None = Field( default=None, - description="What generates trials online: a Model or Agent endpoint, or an agent runner (e.g. Codex " - "CLI). Endpoint targets carry their own request config (prompt template / inference params). " - "Mutually exclusive with `trials`.", + description="What generates trials online: a Model or Agent endpoint, or an agent runner (e.g. a " + "Fabric harness). Endpoint targets carry their own request config (prompt template / inference " + "params). Mutually exclusive with `trials`.", ) trials: list[AgentEvalTrial] | None = Field( default=None, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py index 9ace50bc03..7f56dee72c 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py @@ -22,7 +22,6 @@ from nemo_evaluator.entities import AgentEvalResultEntity, EvaluateResultEntity from nemo_evaluator.jobs.agent_spec import ( AgentTarget, - CodexRunnerTarget, FabricRunnerTarget, GymRunnerTarget, HarborRunnerTarget, @@ -89,8 +88,6 @@ def _agent_target_fields(target: Target | None) -> tuple[str | None, str | None, return "model", target.model.name, _safe_target_url(target.model.url) if isinstance(target, AgentTarget): return "agent", getattr(target.agent, "name", None), _safe_target_url(target.agent.url) - if isinstance(target, CodexRunnerTarget): - return "codex", target.model, None if isinstance(target, FabricRunnerTarget): return "fabric", target.model, None if isinstance(target, GymRunnerTarget): diff --git a/plugins/nemo-evaluator/tests/api/service/test_result_service.py b/plugins/nemo-evaluator/tests/api/service/test_result_service.py index bd0dcb09ac..fa11b1f4eb 100644 --- a/plugins/nemo-evaluator/tests/api/service/test_result_service.py +++ b/plugins/nemo-evaluator/tests/api/service/test_result_service.py @@ -94,7 +94,7 @@ def _agent_entity(name: str, workspace: str = "default") -> AgentEvalResultEntit name=name, workspace=workspace, job_id=name, - target_kind="codex", + target_kind="fabric", target_name="gpt-5.5", target_url=None, scores=AggregatedMetricResult(scores=[]), diff --git a/plugins/nemo-evaluator/tests/api/v2/test_results_routes.py b/plugins/nemo-evaluator/tests/api/v2/test_results_routes.py index 2f60248d57..79c6acf0f4 100644 --- a/plugins/nemo-evaluator/tests/api/v2/test_results_routes.py +++ b/plugins/nemo-evaluator/tests/api/v2/test_results_routes.py @@ -96,7 +96,7 @@ def _agent_entity(name: str) -> AgentEvalResultEntity: name=name, workspace="default", job_id=name, - target_kind="codex", + target_kind="fabric", target_name="gpt-5.5", target_url=None, scores=AggregatedMetricResult(scores=[]), @@ -232,5 +232,5 @@ def test_collections_do_not_collide(client: TestClient, fake: _FakeEntityClient) fake.seed(_agent_entity("shared")) fake.seed(_eval_entity("shared")) - assert client.get(f"{_AGENT}/shared").json()["target_kind"] == "codex" + assert client.get(f"{_AGENT}/shared").json()["target_kind"] == "fabric" assert client.get(f"{_EVAL}/shared").json()["dataset_ref"] == "default/ds" diff --git a/plugins/nemo-evaluator/tests/integration/conftest.py b/plugins/nemo-evaluator/tests/integration/conftest.py index 70560ec777..dd46471158 100644 --- a/plugins/nemo-evaluator/tests/integration/conftest.py +++ b/plugins/nemo-evaluator/tests/integration/conftest.py @@ -200,11 +200,10 @@ def _materialize_subprocess_config(work_root: Path, *, base_url: str, auth_enabl def subprocess_platform(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: """Session-scoped platform with the subprocess jobs backend + IGW mock-provider mode. - Subprocess backend: the compiled task runs as a host process, so a Codex *runner* target finds - the host's codex CLI + ChatGPT login. IGW mock mode (``NMP_INFERENCE_GATEWAY_MOCK_PROVIDER_PREFIX``) - lets Model/Agent-target tests register a mock provider returning a canned response — no real model - or key. Codex-dependent tests gate themselves with ``@requires_codex``; this fixture does not, so - Model/Agent tests (which need only the running IGW) can use it without codex installed. + Subprocess backend: the compiled task runs as a host process, so a runner target sees the host's + agent toolchain. IGW mock mode (``NMP_INFERENCE_GATEWAY_MOCK_PROVIDER_PREFIX``) lets + Model/Agent-target tests register a mock provider returning a canned response — no real model or + key. """ work_root = tmp_path_factory.mktemp("platform") config_path = _materialize_subprocess_config(work_root, base_url=AGENT_PLATFORM_BASE_URL) @@ -288,10 +287,10 @@ def _materialize_docker_config(work_root: Path, *, base_url: str) -> Path: def docker_platform(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: """Session-scoped platform with the docker jobs backend (for the docker-backend submit test). - Gates on a reachable Docker daemon only — codex runs *inside* the task container, not on the - host, so host codex is irrelevant here. The agent-eval step runs in the ``cpu-tasks`` image; - runner targets aren't expected to succeed there yet (the image carries no codex CLI/auth — see - AALGO-301), which is why the test using this fixture is marked xfail. + Gates on a reachable Docker daemon only — a runner executes *inside* the task container, not on + the host, so the host toolchain is irrelevant here. The agent-eval step runs in the ``cpu-tasks`` + image; runner targets aren't expected to succeed there yet (the image carries no agent harness — + see AALGO-301), which is why the test using this fixture is marked xfail. """ if not _docker_available(): pytest.skip("docker daemon not available") diff --git a/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py b/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py index 05156539d7..238ceb669a 100644 --- a/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py @@ -6,8 +6,8 @@ These exercise the job against *real* execution seams, across the dimensions that matter for this work: -* target type — a Codex *runner*, plus Model and Agent endpoint targets pointed at an - IGW mock provider (canned response, so no real model or key); +* target type — Model and Agent endpoint targets pointed at an IGW mock provider + (canned response, so no real model or key); * metric form — an inline metric bundle, plus a stored ``MetricRef`` resolved against the live entity store; * execution mode — in-process ``run_local`` and service-side ``submit`` on both the @@ -15,9 +15,8 @@ ``docker_platform`` fixtures in ``conftest.py``. (Docker submit is xfail today — the cpu-tasks image predates this work; tracked in AALGO-301.) -Marked ``integration`` (auto-applied to ``/integration/`` paths). Codex-dependent tests -gate on ``@requires_codex`` (skip when the ``codex`` CLI is absent, needs a logged-in -ChatGPT account); Model/Agent tests need only the running platform's IGW. +Marked ``integration`` (auto-applied to ``/integration/`` paths). Model/Agent tests +need only the running platform's IGW. Run directly:: @@ -28,7 +27,6 @@ import json import os -import shutil import sys import uuid from pathlib import Path @@ -50,7 +48,6 @@ AgentEvalInputSpec, AgentEvalTaskInput, AgentTarget, - CodexRunnerTarget, HarborRunnerTarget, ModelTarget, ) @@ -69,9 +66,9 @@ from nmp.testing import add_mock_provider from nmp.testing.e2e import wait_for_platform_job -#: Opt-in: these tests spin real ``nemo services`` platforms (subprocess/docker/auth) and some need -#: a local ``codex`` CLI + login, so they're kept out of the standard CI integration job. Run them -#: locally (or on demand) with ``RUN_AGENT_EVAL_INTEGRATION=1``. +#: Opt-in: these tests spin real ``nemo services`` platforms (subprocess/docker/auth), so they're +#: kept out of the standard CI integration job. Run them locally (or on demand) with +#: ``RUN_AGENT_EVAL_INTEGRATION=1``. pytestmark = [ pytest.mark.integration, pytest.mark.skipif( @@ -80,9 +77,6 @@ ), ] -#: Codex model the runner drives. ChatGPT-account dependent; gpt-5.5 is known-good. -CODEX_MODEL = "gpt-5.5" - WORKSPACE = "default" #: Headers a job's task SDK carries (mirrors ``get_task_sdk``): an internal service principal the @@ -91,13 +85,6 @@ SERVICE_PRINCIPAL_HEADERS = {"X-NMP-Principal-Id": "service:evaluator", "X-NMP-Internal": "true"} -def _codex_available() -> bool: - return shutil.which("codex") is not None - - -requires_codex = pytest.mark.skipif(not _codex_available(), reason="codex CLI not on PATH") - - # Pickle metrics defined in this test module BY VALUE so the cloudpickle bundle embeds the class # itself — the submit-backend task runs in a subprocess that can't import this test module, and a # by-reference pickle would fail to hydrate there. @@ -261,52 +248,15 @@ def test_run_local_agent_target_scores_a_real_trial(subprocess_platform: str) -> assert scores[0]["outputs"][0]["value"] in (True, 1.0) -@requires_codex -@pytest.mark.timeout(300) -def test_run_local_codex_runner_scores_a_real_trial() -> None: - # One real Codex run covering dim 1 (runner) x dim 2 (inline metric) x dim 3 (run): the CLI - # produces a trial and a user-defined inline metric scores its output. (Every task must declare - # >=1 metric — the SDK evaluator rejects a metric-less task — so this is the minimal real run.) - input_spec = AgentEvalInputSpec( - tasks=[ - AgentEvalTaskInput( - id="say-done", - intent="Agent follows a trivial instruction and exits cleanly.", - inputs={"instruction": "Reply with the single word DONE and nothing else."}, - metrics=[_output_contains_metric("DONE")], - ) - ], - target=CodexRunnerTarget(model=CODEX_MODEL), - ) - - result = NemoJobScheduler().run_local(AgentEvalJob, input_spec.model_dump(mode="json")) - - assert result["status"] == "completed" - bundle = _bundle_dir(result) - - trials = _read_jsonl(bundle / "trials.jsonl") - assert len(trials) == 1 - assert trials[0]["task_id"] == "say-done" - assert trials[0]["status"] == "completed" - - scores = _read_jsonl(bundle / "scores.jsonl") - assert [score["metric_type"] for score in scores] == ["output-contains"] - assert scores[0]["trial_id"] == trials[0]["id"] - # The custom metric actually validated the agent's output (it replied "DONE"). - output = scores[0]["outputs"][0] - assert output["name"] == "contains" - assert output["value"] in (True, 1.0) - - # --- submit: service-side execution ----------------------------------------- def _offline_trials_input_spec() -> dict: """Submitter-facing spec: one precomputed trial scored offline by one inline metric. - No target — so no online generation, no codex, no IGW. Runs entirely inside the task container, - isolating the docker-backend-wiring + entrypoint condition (rather than also depending on a codex - CLI/auth the image doesn't carry).""" + No target — so no online generation and no IGW. Runs entirely inside the task container, + isolating the docker-backend-wiring + entrypoint condition (rather than also depending on a model + endpoint the image cannot reach).""" return AgentEvalInputSpec( tasks=[ AgentEvalTaskInput( @@ -425,21 +375,6 @@ def test_mixed_job_types_list_endpoints_do_not_cross_render(subprocess_platform: assert agent_name not in row_names -def _codex_eval_input_spec() -> dict: - """Submitter-facing spec: one Codex-runner task scored by one inline metric.""" - return AgentEvalInputSpec( - tasks=[ - AgentEvalTaskInput( - id="say-done", - intent="Agent follows a trivial instruction and exits cleanly.", - inputs={"instruction": "Reply with the single word DONE and nothing else."}, - metrics=[_output_contains_metric("DONE")], - ) - ], - target=CodexRunnerTarget(model=CODEX_MODEL), - ).model_dump(mode="json") - - def _harbor_eval_input_spec() -> dict: """Minimal Harbor target submission; compilation must reject it before task execution.""" return AgentEvalInputSpec( @@ -454,84 +389,12 @@ def _harbor_eval_input_spec() -> dict: ).model_dump(mode="json") -@requires_codex -@pytest.mark.timeout(600) -def test_submit_to_subprocess_backend_runs_agent_eval(subprocess_platform: str) -> None: - # dim 3 (submit) x dim 4 (subprocess backend): submit through the plugin route; the jobs service - # compiles + runs the task as a host subprocess (which has codex), to completion. - client = NeMoPlatform(base_url=subprocess_platform, max_retries=2) - client.workspaces.create(name=WORKSPACE, exist_ok=True) - - response = NemoJobScheduler().submit_remote( - AgentEvalJob, - _codex_eval_input_spec(), - base_url=subprocess_platform, - workspace=WORKSPACE, - profile="default", - ) - job_name = response.get("name") or response.get("id") - assert job_name, f"submit response carried no job name/id: {response}" - - job = wait_for_platform_job(client, job_name, WORKSPACE, timeout=480) - assert job.status == "completed", f"job {job_name} ended {job.status!r}: {getattr(job, 'status_details', None)}" - - # Persistence: run() wrote a queryable result record, retrievable via the typed SDK resource - # (client.evaluator.agent_eval_results -> the /agent-eval-results route). The record is keyed by - # the job id and denormalizes the target it ran against; the full bundle lives in bundle_ref. - result = client.evaluator.agent_eval_results.retrieve(job_name, workspace=WORKSPACE) - assert result.job_id == job_name - assert (result.target_kind, result.target_name) == ("codex", CODEX_MODEL) - assert result.bundle_ref - assert result.created_at is not None - - -@requires_codex -@pytest.mark.timeout(600) -def test_submit_with_stored_metric_ref_resolves_and_scores(subprocess_platform: str) -> None: - # dim 2 (stored MetricRef): store a metric in the platform, reference it by name, and submit. - # The server-side to_spec must resolve the ref against the live entity store + files service - # (not an inline bundle) before the job runs. - client = NeMoPlatform(base_url=subprocess_platform, max_retries=2) - client.workspaces.create(name=WORKSPACE, exist_ok=True) - - metric_name = _unique("done-contains") - stored = _output_contains_metric("DONE") - create = httpx.post( - f"{subprocess_platform}/apis/evaluator/v2/workspaces/{WORKSPACE}/metrics/{metric_name}", - content=stored.model_dump_json(), - headers={"content-type": "application/json"}, - timeout=30, - ) - assert create.status_code in (200, 201), f"metric create failed: {create.status_code} {create.text}" - - spec = AgentEvalInputSpec( - tasks=[ - AgentEvalTaskInput( - id="say-done", - intent="Agent follows a trivial instruction and exits cleanly.", - inputs={"instruction": "Reply with the single word DONE and nothing else."}, - metrics=[MetricRef(f"{WORKSPACE}/{metric_name}")], - ) - ], - target=CodexRunnerTarget(model=CODEX_MODEL), - ).model_dump(mode="json") - - response = NemoJobScheduler().submit_remote( - AgentEvalJob, spec, base_url=subprocess_platform, workspace=WORKSPACE, profile="default" - ) - job_name = response.get("name") or response.get("id") - assert job_name, f"submit response carried no job name/id: {response}" - - job = wait_for_platform_job(client, job_name, WORKSPACE, timeout=480) - assert job.status == "completed", f"job {job_name} ended {job.status!r}: {getattr(job, 'status_details', None)}" - - @pytest.mark.timeout(420) def test_submit_over_taskset_ref_resolves_and_scores(subprocess_platform: str) -> None: # dim 2 (stored taskset ref) x dim 3 (submit): store a metric + two tasks + a taskset, then submit # an agent eval whose `tasks` is a TasksetRef (no inline tasks). Server-side to_spec must load the # taskset, expand BOTH member tasks, and resolve each task's stored MetricRef — all against the - # live entity store — before the job runs. A Model target -> IGW mock provider keeps it codex-free. + # live entity store — before the job runs. A Model target -> IGW mock provider keeps it hermetic. client = NeMoPlatform(base_url=subprocess_platform, max_retries=2) client.workspaces.create(name=WORKSPACE, exist_ok=True) @@ -609,7 +472,6 @@ def test_submit_model_target_under_auth_forwards_identity_to_igw(auth_subprocess # inference client (AgentEvalJob._build_evaluator) — otherwise the IGW returns 401 and the job # fails. A clean completion proves the forwarded service-principal headers authenticate online # inference under auth, with no bearer. (Probed directly too: service headers -> 200, none -> 401.) - # No codex: the target is an IGW mock provider, so this runs without the runner toolchain. sdk = NeMoPlatform(base_url=auth_subprocess_platform, default_headers=SERVICE_PRINCIPAL_HEADERS, max_retries=2) sdk.workspaces.create(name=WORKSPACE, exist_ok=True) model_name = _unique("auth-model") @@ -693,7 +555,7 @@ def test_submit_harbor_target_to_docker_backend_fails_fast(docker_platform: str) reason="agent-eval can't run under the docker backend until the cpu-tasks image is rebuilt with " "this work: the published image predates the nemo_evaluator.tasks.agent_evaluate entrypoint " "(container exits with ModuleNotFoundError). This submits an offline trials spec (no online " - "generation, no codex, no IGW), so the stale image is the only remaining failure cause — the " + "generation, no online target, no IGW), so the stale image is the only remaining failure cause — the " "xfail flips the moment the image ships the entrypoint. Tracked in AALGO-301.", strict=False, ) @@ -703,7 +565,7 @@ def test_submit_to_docker_backend_runs_agent_eval(docker_platform: str) -> None: # cpu-tasks container. Verified to genuinely reach the docker backend (it creates a container # from the cpu-tasks image); it fails today because that image predates this work — hence xfail. # An offline trials spec keeps the task self-contained in-container, so this isolates the - # backend-wiring + entrypoint condition rather than also depending on a codex CLI/auth. + # backend-wiring + entrypoint condition rather than also depending on a live model endpoint. client = NeMoPlatform(base_url=docker_platform, max_retries=2) client.workspaces.create(name=WORKSPACE, exist_ok=True) diff --git a/plugins/nemo-evaluator/tests/integration/test_docs_manage_tasks_tasksets.py b/plugins/nemo-evaluator/tests/integration/test_docs_manage_tasks_tasksets.py index e4b5ca842f..c44f5838c2 100644 --- a/plugins/nemo-evaluator/tests/integration/test_docs_manage_tasks_tasksets.py +++ b/plugins/nemo-evaluator/tests/integration/test_docs_manage_tasks_tasksets.py @@ -13,7 +13,7 @@ claims. It deliberately mirrors the doc's own code rather than being written as an idiomatic test — when it fails, the fix is usually the doc. -Pure CRUD (no codex/IGW), so it only needs the host subprocess backend. Shares the evaluator-plugin +Pure CRUD (no online target or IGW), so it only needs the host subprocess backend. Shares the evaluator-plugin integration opt-in (``RUN_AGENT_EVAL_INTEGRATION``) and the session-scoped ``subprocess_platform``. """ diff --git a/plugins/nemo-evaluator/tests/integration/test_evaluate_job.py b/plugins/nemo-evaluator/tests/integration/test_evaluate_job.py index 8af2e1d605..8e238619bc 100644 --- a/plugins/nemo-evaluator/tests/integration/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/integration/test_evaluate_job.py @@ -5,7 +5,7 @@ Shares the evaluator-plugin integration harness (conftest's session-scoped ``subprocess_platform``) and the ``RUN_AGENT_EVAL_INTEGRATION`` opt-in. Submits an *offline* metric eval — inline dataset, no -model target / IGW / codex — so the only requirement is the host subprocess backend. Asserts the run +model target / IGW / agent runner — so the only requirement is the host subprocess backend. Asserts the run persisted a queryable ``EvaluateResult`` retrievable via ``client.evaluator.eval_results``, covering the row-eval half of result persistence (the agent-eval half lives in ``test_agent_evaluate_job.py``). """ @@ -60,7 +60,7 @@ def _offline_exact_match_spec() -> dict: def test_submit_offline_row_eval_persists_result(subprocess_platform: str) -> None: # dim: submit x subprocess backend, row (EvaluateJob) path. The jobs service compiles + runs # EvaluateJob.run() as a host subprocess; run() writes an EvaluateResult through the async task - # SDK + entity store. Offline (no target/IGW/codex): the dataset already carries the outputs. + # SDK + entity store. Offline (no target or IGW): the dataset already carries the outputs. client = NeMoPlatform(base_url=subprocess_platform, max_retries=2) client.workspaces.create(name=WORKSPACE, exist_ok=True) diff --git a/plugins/nemo-evaluator/tests/integration/test_metric_filtering.py b/plugins/nemo-evaluator/tests/integration/test_metric_filtering.py index ed351085eb..c2768854f5 100644 --- a/plugins/nemo-evaluator/tests/integration/test_metric_filtering.py +++ b/plugins/nemo-evaluator/tests/integration/test_metric_filtering.py @@ -5,7 +5,7 @@ Verifies the route's custom-field filter actually works end-to-end: ``metric_type`` is a ``data.*`` entity field, so without the ``DataFilter`` translation the entity store 500s. Pure CRUD (create + -list) — no codex/IGW — so it only needs the host subprocess backend. Shares the evaluator-plugin +list) — no online target or IGW — so it only needs the host subprocess backend. Shares the evaluator-plugin integration opt-in (``RUN_AGENT_EVAL_INTEGRATION``) and the session-scoped ``subprocess_platform``. """ diff --git a/plugins/nemo-evaluator/tests/integration/test_task_derived_metrics.py b/plugins/nemo-evaluator/tests/integration/test_task_derived_metrics.py index 8b9d8106e9..7c804d10e8 100644 --- a/plugins/nemo-evaluator/tests/integration/test_task_derived_metrics.py +++ b/plugins/nemo-evaluator/tests/integration/test_task_derived_metrics.py @@ -12,7 +12,7 @@ - the derived metric is real (retrievable, ``derived=True``, Files-backed); - it is hidden from the default ``/metrics`` listing but visible with ``include_derived``. -Pure CRUD (no codex/IGW), so it only needs the host subprocess backend. Shares the evaluator-plugin +Pure CRUD (no online target or IGW), so it only needs the host subprocess backend. Shares the evaluator-plugin integration opt-in (``RUN_AGENT_EVAL_INTEGRATION``) and the session-scoped ``subprocess_platform``. """ diff --git a/plugins/nemo-evaluator/tests/integration/test_task_revisions.py b/plugins/nemo-evaluator/tests/integration/test_task_revisions.py index bbe7f34d9c..2f0f3c9c60 100644 --- a/plugins/nemo-evaluator/tests/integration/test_task_revisions.py +++ b/plugins/nemo-evaluator/tests/integration/test_task_revisions.py @@ -17,7 +17,7 @@ - a published revision is immutable in practice: reading a pinned digest returns the old content after the task has moved on. -Pure CRUD (no codex/IGW), so it only needs the host subprocess backend. Shares the evaluator-plugin +Pure CRUD (no online target or IGW), so it only needs the host subprocess backend. Shares the evaluator-plugin integration opt-in (``RUN_AGENT_EVAL_INTEGRATION``) and the session-scoped ``subprocess_platform``. """ diff --git a/plugins/nemo-evaluator/tests/jobs/test_publication.py b/plugins/nemo-evaluator/tests/jobs/test_publication.py index 48d1a4ba2d..969ae85672 100644 --- a/plugins/nemo-evaluator/tests/jobs/test_publication.py +++ b/plugins/nemo-evaluator/tests/jobs/test_publication.py @@ -23,7 +23,6 @@ AgentEvalTaskInput, AgentEvalTaskSpec, AgentTarget, - CodexRunnerTarget, FabricRunnerTarget, GymRunnerTarget, HarborRunnerTarget, @@ -232,7 +231,6 @@ def _publish(client: AsyncNeMoPlatform | None, *, required: bool = True, agent_n GymRunnerTarget(agent="simple_agent", agent_config="conf/agent.yaml", resources_server="mcqa"), ("simple_agent", None), ), - (CodexRunnerTarget(model="gpt-5.5"), (None, "gpt-5.5")), (FabricRunnerTarget(config={}, model="p/m"), (None, "p/m")), (None, (None, None)), ], @@ -296,7 +294,6 @@ def test_agent_name_derived_from_gym_target_needs_no_override() -> None: "target", [ ModelTarget(model=Model(name="gpt-4o", url="http://model")), - CodexRunnerTarget(model="gpt-5.5"), FabricRunnerTarget(config={}), None, ], @@ -317,7 +314,7 @@ def test_blank_identity_fields_are_rejected() -> None: @pytest.mark.parametrize( "target", - [ModelTarget(model=Model(name="gpt-4o", url="http://model")), CodexRunnerTarget(model="gpt-5.5"), None], + [ModelTarget(model=Model(name="gpt-4o", url="http://model")), FabricRunnerTarget(config={}), None], ) def test_explicit_agent_name_satisfies_undeducible_targets(target: Target | None) -> None: spec = _input_spec( @@ -509,7 +506,7 @@ def _job_context(tmp_path: Path, *, job_id: str | None = None) -> JobContext: def _job_spec(*, required: bool = True) -> AgentEvalSpec: return AgentEvalSpec( tasks=[AgentEvalTaskSpec(id="task-1", intent="Answer.")], - target=CodexRunnerTarget(model="gpt-5.5"), + target=FabricRunnerTarget(config={}, model="p/m"), publication=PublicationSpec( intake=IntakePublicationSpec(evaluation_id="eval-1", agent_name="a", required=required) ), @@ -520,7 +517,7 @@ def test_job_does_not_publish_without_a_publication_spec(tmp_path: Path, mocker: mocker.patch.object(AgentEvalJob, "_build_evaluator", return_value=_FakeEvaluator()) client = _FakeClient() - spec = AgentEvalSpec(tasks=[AgentEvalTaskSpec(id="task-1", intent="Answer.")], target=CodexRunnerTarget()) + spec = AgentEvalSpec(tasks=[AgentEvalTaskSpec(id="task-1", intent="Answer.")], target=FabricRunnerTarget(config={})) result = AgentEvalJob().run( spec.model_dump(), ctx=_job_context(tmp_path), async_sdk=cast(AsyncNeMoPlatform, client) ) diff --git a/plugins/nemo-evaluator/tests/sdk/test_result_sdk_resources.py b/plugins/nemo-evaluator/tests/sdk/test_result_sdk_resources.py index c931f76360..009d7e0dff 100644 --- a/plugins/nemo-evaluator/tests/sdk/test_result_sdk_resources.py +++ b/plugins/nemo-evaluator/tests/sdk/test_result_sdk_resources.py @@ -24,15 +24,15 @@ _BASE = "http://localhost:8080/apis/evaluator/v2/workspaces/default" -def _agent_payload(name: str) -> dict[str, Any]: +def _agent_payload(name: str, *, target_kind: str = "fabric", target_name: str = "openai/gpt-5.4") -> dict[str, Any]: now = datetime.now(timezone.utc) return AgentEvalResult( id=f"agent_eval_result-{name}", name=name, workspace="default", job_id=name, - target_kind="codex", - target_name="gpt-5.5", + target_kind=target_kind, + target_name=target_name, target_url=None, scores=AggregatedMetricResult(scores=[]), bundle_ref="fileset://default/agent-eval-results#b", @@ -102,10 +102,24 @@ def test_sync_retrieve_agent_eval_targets_item_url_and_parses_dto() -> None: assert isinstance(result, AgentEvalResult) assert result.job_id == "job-1" - assert result.target_kind == "codex" + assert result.target_kind == "fabric" assert http_client.get.call_args[0][0] == f"{_BASE}/agent-eval-results/job-1" +def test_sync_retrieve_agent_eval_parses_a_retired_runner_kind() -> None: + # The wire DTO must keep reading rows written by a runner that has since been removed; nothing + # in the response schema constrains `target_kind` to the runners that currently exist. + http_client = MagicMock() + http_client.get.return_value = _response(_agent_payload("job-legacy", target_kind="codex", target_name="gpt-5.5")) + resource = EvaluatorAgentEvalResultsResource(_platform(http_client)) + + result = resource.retrieve("job-legacy") + + assert isinstance(result, AgentEvalResult) + assert result.target_kind == "codex" + assert result.target_name == "gpt-5.5" + + def test_sync_list_eval_results_parses_dtos_and_targets_collection() -> None: http_client = MagicMock() http_client.get.return_value = _response(_page([_eval_payload("a"), _eval_payload("b")])) diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index 957a3c7572..7e46904ed0 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -26,7 +26,6 @@ AgentEvalTaskInput, AgentEvalTaskSpec, AgentTarget, - CodexRunnerTarget, FabricRunnerTarget, GymRunnerTarget, HarborRunnerTarget, @@ -39,7 +38,6 @@ from nemo_evaluator.tasks.agent_evaluate import main as agent_eval_task_main from nemo_evaluator.tasks.runner import SDK_INITIALIZATION_EXIT_CODE from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary -from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime from nemo_evaluator_sdk.agent_eval.runtimes.gym import GymAgentTaskRunner from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner @@ -96,6 +94,13 @@ def _task_spec() -> AgentEvalTaskSpec: ) +def _runner_target(model: str | None = None) -> FabricRunnerTarget: + """A minimal agent-runner target, for tests about runners in general rather than a specific one.""" + return FabricRunnerTarget( + config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex"}}, model=model + ) + + def _job_context(tmp_path: Path) -> JobContext: storage = StoragePaths(ephemeral=tmp_path / "ephemeral", persistent=tmp_path / "persistent") storage.ephemeral.mkdir() @@ -155,7 +160,7 @@ async def test_reference_round_trips_from_input_spec_to_runtime_task() -> None: # metrics can grade against held-out ground truth (never seeded into the agent workspace). reference = {"test_calculator.py": "def test_add(): assert add(2, 3) == 5"} input_spec = AgentEvalInputSpec( - target=CodexRunnerTarget(), + target=_runner_target(), tasks=[ AgentEvalTaskInput( id="fix-bug", @@ -207,7 +212,7 @@ def test_agent_eval_job_reconstructs_tasks_and_persists_bundle(tmp_path: Path, m mocker.patch.object(AgentEvalJob, "_build_evaluator", return_value=fake) ctx = _job_context(tmp_path) - spec = AgentEvalSpec(tasks=[_task_spec()], target=CodexRunnerTarget(model="gpt-5.5")) + spec = AgentEvalSpec(tasks=[_task_spec()], target=_runner_target("openai/gpt-5.4")) result = AgentEvalJob().run(spec.model_dump(), ctx=ctx) # The job reconstructed runtime tasks (bundled metric round-tripped) before handing off. @@ -235,7 +240,7 @@ def test_agent_eval_job_survives_result_persistence_failure(tmp_path: Path, mock ) ctx = _job_context(tmp_path) - spec = AgentEvalSpec(tasks=[_task_spec()], target=CodexRunnerTarget(model="gpt-5.5")) + spec = AgentEvalSpec(tasks=[_task_spec()], target=_runner_target("openai/gpt-5.4")) result = AgentEvalJob().run(spec.model_dump(), ctx=ctx) # Persistence was attempted and raised, yet the job still completed with its artifacts intact. @@ -260,15 +265,6 @@ def _agent() -> Agent: ) -def test_resolve_target_builds_codex_runtime_from_runner_target(tmp_path: Path) -> None: - ctx = _job_context(tmp_path) - target, prompt_template, params = AgentEvalJob._resolve_target(CodexRunnerTarget(model="gpt-5.5"), ctx) - # A runner shapes its own request, so it contributes no prompt template or inference params. - assert isinstance(target, CodexCliAgentRuntime) - assert prompt_template is None - assert params is None - - def test_resolve_target_builds_fabric_runtime_from_runner_target(tmp_path: Path) -> None: ctx = _job_context(tmp_path) fabric_target = FabricRunnerTarget( @@ -360,8 +356,8 @@ def test_resolve_target_builds_gym_runtime_from_runner_target(tmp_path: Path) -> def test_runner_target_is_accepted(tmp_path: Path) -> None: - spec = AgentEvalSpec(tasks=[_task_spec()], target=CodexRunnerTarget(model="gpt-5.5")) - assert isinstance(spec.target, CodexRunnerTarget) + spec = AgentEvalSpec(tasks=[_task_spec()], target=_runner_target("openai/gpt-5.4")) + assert isinstance(spec.target, FabricRunnerTarget) def test_harbor_runner_target_is_accepted() -> None: @@ -436,9 +432,9 @@ def test_build_evaluator_sends_no_identity_to_third_party_target( def test_build_evaluator_runner_target_forwards_no_headers() -> None: - # A runner (Codex CLI) has no platform HTTP endpoint, so there's no identity to forward. + # A runner has no platform HTTP endpoint of its own, so there's no identity to forward. sdk = _sdk_with_identity(NeMoPlatform) - assert AgentEvalJob._build_evaluator(sdk, CodexRunnerTarget(model="gpt-5.5")).default_headers is None + assert AgentEvalJob._build_evaluator(sdk, _runner_target("openai/gpt-5.4")).default_headers is None def test_build_evaluator_without_platform_forwards_no_headers() -> None: @@ -449,13 +445,13 @@ def test_build_evaluator_without_platform_forwards_no_headers() -> None: def test_input_spec_accepts_stored_metric_reference() -> None: spec = AgentEvalInputSpec( tasks=[AgentEvalTaskInput(id="task-1", intent="Answer.", inputs={}, metrics=[MetricRef("stored-metric")])], - target=CodexRunnerTarget(model="gpt-5.5"), + target=_runner_target("openai/gpt-5.4"), ) assert isinstance(spec.tasks[0].metrics[0], MetricRef) def test_input_spec_accepts_a_taskset_reference() -> None: - spec = AgentEvalInputSpec(tasks=TasksetRef("default/geo-suite"), target=CodexRunnerTarget(model="gpt-5.5")) + spec = AgentEvalInputSpec(tasks=TasksetRef("default/geo-suite"), target=_runner_target("openai/gpt-5.4")) assert isinstance(spec.tasks, TasksetRef) assert spec.tasks.root == "default/geo-suite" # A JSON string round-trips back to the TasksetRef arm of the union, not a list. @@ -464,7 +460,7 @@ def test_input_spec_accepts_a_taskset_reference() -> None: def test_input_spec_rejects_empty_inline_task_list() -> None: with pytest.raises(ValueError, match="at least one task"): - AgentEvalInputSpec(tasks=[], target=CodexRunnerTarget(model="gpt-5.5")) + AgentEvalInputSpec(tasks=[], target=_runner_target("openai/gpt-5.4")) async def test_to_spec_resolves_inline_task_metrics_without_a_platform() -> None: @@ -478,7 +474,7 @@ async def test_to_spec_resolves_inline_task_metrics_without_a_platform() -> None metrics=[_inline_metric()], ) ], - target=CodexRunnerTarget(model="gpt-5.5"), + target=_runner_target("openai/gpt-5.4"), ) spec = await AgentEvalJob.to_spec(input_spec, workspace="dev", entity_client=None, async_sdk=None, is_local=True) @@ -495,7 +491,7 @@ async def test_to_spec_requires_platform_to_resolve_a_metric_reference() -> None # fail loudly rather than silently drop the metric. input_spec = AgentEvalInputSpec( tasks=[AgentEvalTaskInput(id="task-1", intent="Answer.", inputs={}, metrics=[MetricRef("stored-metric")])], - target=CodexRunnerTarget(model="gpt-5.5"), + target=_runner_target("openai/gpt-5.4"), ) with pytest.raises(ValueError, match="platform connection"): await AgentEvalJob.to_spec(input_spec, workspace="dev", entity_client=None, async_sdk=None, is_local=True) @@ -569,7 +565,6 @@ async def _compile_harbor(*, async_sdk: AsyncNeMoPlatform | None, profile: str | @pytest.mark.parametrize( ("target", "expected_kind", "expected_endpoint_name", "expected_image_name"), [ - (CodexRunnerTarget(model="gpt-5.5"), "codex", None, "nmp-cpu-tasks"), ( FabricRunnerTarget(config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex"}}), "fabric", @@ -765,7 +760,7 @@ async def test_compile_non_harbor_target_does_not_resolve_execution_profiles(moc "nemo_evaluator.jobs.agent_evaluate.client_from_platform", side_effect=AssertionError("non-Harbor compilation must not query execution profiles"), ) - spec = AgentEvalSpec(tasks=[_task_spec()], target=CodexRunnerTarget(model="gpt-5.5")) + spec = AgentEvalSpec(tasks=[_task_spec()], target=_runner_target("openai/gpt-5.4")) compiled = await AgentEvalJob.compile( workspace="default", @@ -775,7 +770,7 @@ async def test_compile_non_harbor_target_does_not_resolve_execution_profiles(moc async_sdk=None, ) - assert cast(dict[str, Any], PlatformJobSpec.model_validate(compiled).steps[0].config)["target"]["kind"] == "codex" + assert cast(dict[str, Any], PlatformJobSpec.model_validate(compiled).steps[0].config)["target"]["kind"] == "fabric" async def test_compile_injects_target_api_key_secret() -> None: @@ -829,7 +824,7 @@ async def test_compile_rejects_reserved_secret_env_name() -> None: model=Model(url="http://model.test/v1/chat/completions", name="test-model"), params=RunConfigOnlineModel() ), AgentTarget(agent=_agent(), params=RunConfigOnline()), - CodexRunnerTarget(model="gpt-5.5"), + _runner_target("openai/gpt-5.4"), HarborRunnerTarget(agent_name="oracle"), GymRunnerTarget( agent="simple_agent", @@ -858,8 +853,8 @@ def test_run_local_executes_each_target_type(target: Target, mocker: MockerFixtu assert result["artifact"]["name"] == DEFAULT_RESULT_NAME assert [task.id for task in fake.received_tasks] == ["task-1"] assert fake.received_trials is None # online generation, not precomputed - if isinstance(target, CodexRunnerTarget): - assert isinstance(fake.received_target, CodexCliAgentRuntime) + if isinstance(target, FabricRunnerTarget): + assert isinstance(fake.received_target, FabricAgentRuntime) elif isinstance(target, HarborRunnerTarget): assert isinstance(fake.received_target, HarborAgentTaskRunner) elif isinstance(target, GymRunnerTarget): @@ -898,7 +893,7 @@ def test_spec_requires_exactly_one_of_target_or_trials() -> None: with pytest.raises(ValueError, match="exactly one"): AgentEvalSpec(tasks=[_task_spec()]) # neither target nor trials with pytest.raises(ValueError, match="exactly one"): - AgentEvalSpec(tasks=[_task_spec()], target=CodexRunnerTarget(model="gpt-5.5"), trials=[trial]) # both + AgentEvalSpec(tasks=[_task_spec()], target=_runner_target("openai/gpt-5.4"), trials=[trial]) # both # --- container task entrypoint ---------------------------------------------- diff --git a/plugins/nemo-evaluator/tests/test_result_entity.py b/plugins/nemo-evaluator/tests/test_result_entity.py index 96b8f54fa7..1ea2bcd366 100644 --- a/plugins/nemo-evaluator/tests/test_result_entity.py +++ b/plugins/nemo-evaluator/tests/test_result_entity.py @@ -36,13 +36,38 @@ def _roundtrip(entity: _E) -> _E: return cls.model_validate({"name": entity.name, "workspace": entity.workspace, **data}) +def test_agent_eval_result_roundtrip_reads_a_retired_runner_kind() -> None: + """A result stored before its runner was removed still loads: ``target_kind`` is a free-form + trait, not a live discriminator, so retiring a runner never strands the rows it wrote. + + Built from the raw string rather than a target class on purpose — the class is gone, and a test + that needed it could not express this. + """ + entity = AgentEvalResultEntity( + name="job-legacy", + workspace="default", + job_id="job-legacy", + target_kind="codex", + target_name="gpt-5.5", + target_url=None, + scores=_scores(), + bundle_ref="fileset://default/agent-eval-results#legacy", + ) + + restored = _roundtrip(entity) + + assert restored.target_kind == "codex" + assert restored.target_name == "gpt-5.5" + assert restored.scores == entity.scores + + def test_agent_eval_result_roundtrip_preserves_scores_and_target() -> None: entity = AgentEvalResultEntity( name="job-123", workspace="default", job_id="job-123", - target_kind="codex", - target_name="gpt-5.5", + target_kind="fabric", + target_name="openai/gpt-5.4", target_url=None, scores=_scores(), bundle_ref="fileset://default/agent-eval-results#bundle", @@ -51,8 +76,8 @@ def test_agent_eval_result_roundtrip_preserves_scores_and_target() -> None: restored = _roundtrip(entity) assert restored.job_id == "job-123" - assert restored.target_kind == "codex" - assert restored.target_name == "gpt-5.5" + assert restored.target_kind == "fabric" + assert restored.target_name == "openai/gpt-5.4" assert restored.target_url is None assert restored.bundle_ref == entity.bundle_ref # The nested AggregatedMetricResult must survive the JSON column intact. diff --git a/plugins/nemo-evaluator/tests/test_result_persistence.py b/plugins/nemo-evaluator/tests/test_result_persistence.py index 2600c29f25..589315fb83 100644 --- a/plugins/nemo-evaluator/tests/test_result_persistence.py +++ b/plugins/nemo-evaluator/tests/test_result_persistence.py @@ -17,7 +17,6 @@ from nemo_evaluator.jobs import result_persistence from nemo_evaluator.jobs.agent_spec import ( AgentTarget, - CodexRunnerTarget, FabricRunnerTarget, GymRunnerTarget, HarborRunnerTarget, @@ -92,9 +91,11 @@ def _agent() -> Agent: [ (ModelTarget(model=_model()), ("model", "my-model", "https://model.test/v1/chat/completions")), (AgentTarget(agent=_agent()), ("agent", "my-agent", "http://agent.test")), - (CodexRunnerTarget(model="gpt-5.5"), ("codex", "gpt-5.5", None)), ( - FabricRunnerTarget(config={"metadata": {"name": "a"}}, model="openai/gpt-5.4"), + FabricRunnerTarget( + config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex"}}, + model="openai/gpt-5.4", + ), ("fabric", "openai/gpt-5.4", None), ), ( @@ -187,7 +188,10 @@ def test_persist_agent_eval_result_builds_entity_and_saves(tmp_path: Path, mocke persist_agent_eval_result( _agent_result(), - target=CodexRunnerTarget(model="gpt-5.5"), + target=FabricRunnerTarget( + config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex"}}, + model="openai/gpt-5.4", + ), ctx=_ctx(tmp_path, "job-1"), bundle_ref="fileset://dev/agent-eval-results#b", async_sdk=_ASYNC_SDK, @@ -199,8 +203,8 @@ def test_persist_agent_eval_result_builds_entity_and_saves(tmp_path: Path, mocke assert entity.name == "job-1" assert entity.job_id == "job-1" assert entity.workspace == "dev" - assert entity.target_kind == "codex" - assert entity.target_name == "gpt-5.5" + assert entity.target_kind == "fabric" + assert entity.target_name == "openai/gpt-5.4" assert entity.target_url is None assert entity.bundle_ref == "fileset://dev/agent-eval-results#b" diff --git a/plugins/nemo-evaluator/tests/test_skill_examples.py b/plugins/nemo-evaluator/tests/test_skill_examples.py index f59e67bc1b..850ca803c7 100644 --- a/plugins/nemo-evaluator/tests/test_skill_examples.py +++ b/plugins/nemo-evaluator/tests/test_skill_examples.py @@ -15,7 +15,7 @@ import pytest import yaml from nemo_evaluator.api.schemas import TasksetRef -from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec, CodexRunnerTarget, FabricRunnerTarget +from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec, FabricRunnerTarget from nemo_evaluator.jobs.evaluate import EvaluateInputSpec from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, bundle_metric, unbundle_metric from nemo_evaluator.shared.metric_bundles.inline import InlineMetricBundlePackager @@ -161,13 +161,14 @@ def test_skill_python_examples_import_and_build_agent_spec() -> None: assert not isinstance(spec.tasks, TasksetRef) assert len(spec.tasks) == 1 - assert isinstance(spec.target, CodexRunnerTarget) + assert isinstance(spec.target, FabricRunnerTarget) + assert spec.target.config["harness"]["adapter_id"] == "nvidia.fabric.codex" assert spec.target.model is None reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/agent-evaluation.md").read_text( encoding="utf-8" ) - assert 'CodexRunnerTarget(model="")' not in reference + assert "CodexRunnerTarget" not in reference assert 'labels={"benchmark": "geography-smoke"}' in reference diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py deleted file mode 100644 index a66e4c99a9..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py +++ /dev/null @@ -1,641 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Codex-backed agent-eval runtimes.""" - -# ruff: noqa: I001, T201 - the vendored SDK mirror uses different import-order and print settings. - -from __future__ import annotations - -import asyncio -import contextlib -import json -import os -import shlex -import shutil -import stat -import subprocess -import tempfile -from collections.abc import Awaitable, Callable, Mapping, Sequence -from enum import StrEnum -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTrial, - AgentEvalTrialStatus, - AgentOutput, - RunnerInfo, - callable_identity, -) -from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor - -#: Wall-clock ceiling for a single task's Codex CLI invocation — one ``process.communicate()`` covering -#: the agent's whole run on that task, not a per-request or per-turn limit. Tasks run independently, so -#: this is not a budget for the evaluation as a whole. On expiry the process is terminated and the task -#: is recorded as a failed trial; it does not abort the run. -DEFAULT_CODEX_TIMEOUT_S = 600 -DEFAULT_CODEX_DOCKER_MODEL = "gpt-5.4" -DEFAULT_CODEX_DOCKER_CLI_IMAGE = "node:22-alpine" -DEFAULT_CODEX_DOCKER_CLI_PACKAGE = "@openai/codex@0.137.0" -ProcessFactory = Callable[..., Awaitable[Any]] - - -class RuntimeChoice(StrEnum): - """Which Codex execution mode the caller wants.""" - - DOCKER = "docker" - LOCAL = "local" - - -class EffectiveCodexRuntime(StrEnum): - """The concrete runtime chosen for a :class:`RuntimeChoice` + environment.""" - - DOCKER_SANDBOX = "docker_sandbox" - DOCKER_CLI = "docker_cli" - LOCAL_CLI = "local_cli" - - -#: Builds the prompt handed to Codex on stdin for a task. Swap it to change how a task is framed -#: (e.g. a benchmark-specific preamble); the default presents the task and invites workspace edits. -CodexPromptBuilder = Callable[[AgentEvalTask], str] - - -class CodexCliAgentRuntime: - """AgentTaskRunner that uses the locally installed Codex CLI credentials.""" - - def __init__( - self, - *, - model: str | None = None, - work_root: str | Path | None = None, - codex_bin: str = "codex", - timeout_s: int = DEFAULT_CODEX_TIMEOUT_S, - prompt_builder: CodexPromptBuilder | None = None, - process_factory: ProcessFactory | None = None, - runtime_name: str = "codex_cli", - ) -> None: - self._model = model - self._work_root = Path(work_root).expanduser() if work_root is not None else None - self._codex_bin = codex_bin - self._timeout_s = timeout_s - self._prompt_builder = prompt_builder or AgentEvalTask.agent_prompt - self._process_factory = process_factory or asyncio.create_subprocess_exec - self._runtime_name = runtime_name - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the Codex CLI settings that shape its results. - - Uses ``runtime_name``, which subclasses already set (the Docker variant reports - ``codex_docker_cli``) and which trials are stamped with, so provenance agrees with them. - """ - return RunnerInfo( - name=self._runtime_name, - kind="runner", - config={ - "model": self._model, - "timeout_s": self._timeout_s, - "codex_bin": self._codex_bin, - "prompt_builder": callable_identity(self._prompt_builder), - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - if shutil.which(self._codex_bin) is None: - raise RuntimeError(f"Codex CLI executable {self._codex_bin!r} was not found on PATH") - - resolved_config = config or AgentEvalRunConfig() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(index, task, resolved_config) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> AgentEvalTrial: - evidence_dir = self._evidence_dir(index, task, config) - workspace_dir = evidence_dir / "workspace" - - try: - # The task directory is mounted into Docker, but its private parent is not. Keeping that - # parent host-owned and 0700 preserves the local confidentiality boundary even when a - # container is interrupted before its recursive cleanup completes. - _ensure_private_directory(evidence_dir.parent) - _ensure_private_directory(evidence_dir) - _ensure_private_directory(workspace_dir) - except Exception as exc: - # The path that failed setup is not safe to use for artifact persistence. In particular, - # writing through a rejected evidence-directory symlink would escape the private tree. - return _failed_codex_trial(task, None, exc, runtime_name=self._runtime_name) - - prompt_path = evidence_dir / "prompt.txt" - task_path = evidence_dir / "task.json" - stdout_path = evidence_dir / "stdout.jsonl" - stderr_path = evidence_dir / "stderr.txt" - final_output_path = evidence_dir / "final_output.txt" - - # Persist the task for debugging, but never the grader-only fields: the docker variant mounts - # this evidence dir into the sandbox (danger-full-access), so serializing `intent` (desired - # behavior) or `reference` (held-out ground truth) here would let the agent read them back out - # of `/evidence/task.json` — the same reward-hacking leak the intent-free prompt closes. - try: - _write_private_text(task_path, task.model_dump_json(indent=2, exclude={"intent", "reference"})) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - - command = self._command(workspace_dir=workspace_dir, final_output_path=final_output_path) - process: Any | None = None - try: - # Seed inside the guarded block so a bad seed (e.g. a path escaping the workspace) fails - # just this task rather than aborting the whole run. Offload to a worker thread: seeding is - # synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this - # runs on the event loop shared by every concurrent task, so a blocking seed would stall them all. - seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - # Build the prompt after seeding and inside the guarded block: an instruction-less task - # raises here, failing just this task instead of aborting the run (and seeding wins if both). - prompt = self._prompt_builder(task) - _write_private_text(prompt_path, prompt) - process = await self._process_factory( - *command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if process is None: - raise RuntimeError("process factory failed to create a process") - stdout, stderr = await asyncio.wait_for( - process.communicate(prompt.encode("utf-8")), - timeout=self._timeout_s, - ) - except TimeoutError as exc: - await _terminate_process(process) - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - - stdout_text = _decode_process_output(stdout) - stderr_text = _decode_process_output(stderr) - artifact_persistence_error: str | None = None - try: - _write_private_text(stdout_path, stdout_text) - _write_private_text(stderr_path, stderr_text) - except Exception as exc: - artifact_persistence_error = f"{exc.__class__.__name__}: {exc}" - - permission_cleanup_error: str | None = None - try: - self._validate_artifact_permissions(evidence_dir) - except Exception as exc: - permission_cleanup_error = f"{exc.__class__.__name__}: {exc}" - - if process.returncode != 0: - return _failed_codex_trial( - task, - evidence_dir, - RuntimeError(f"codex exec exited with status {process.returncode}: {stderr_text.strip()}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - artifact_persistence_error=artifact_persistence_error, - ) - - if artifact_persistence_error is not None: - return _failed_codex_trial( - task, - evidence_dir, - RuntimeError(f"failed to persist Codex evidence: {artifact_persistence_error}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - artifact_persistence_error=artifact_persistence_error, - ) - if permission_cleanup_error is not None: - return _failed_codex_trial( - task, - evidence_dir, - PermissionError(f"Codex evidence permission normalization failed: {permission_cleanup_error}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - ) - - try: - output_text = _read_private_final_output(final_output_path, fallback=stdout_text) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - return AgentEvalTrial( - id=f"{task.id}:codex", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=output_text, - metadata={ - "runtime": self._runtime_name, - "agent": "codex", - "agent_model": self._model, - "evidence_dir": str(evidence_dir), - }, - ), - evidence=CandidateEvidence( - descriptors={ - "workspace": EvidenceDescriptor(kind="filesystem", ref=str(workspace_dir)), - "prompt": EvidenceDescriptor(kind="text", format="txt", ref=str(prompt_path)), - "task": EvidenceDescriptor(kind="json", format="json", ref=str(task_path)), - "stdout": EvidenceDescriptor(kind="codex_stdout", format="jsonl", ref=str(stdout_path)), - "stderr": EvidenceDescriptor(kind="text", format="txt", ref=str(stderr_path)), - "final_output": EvidenceDescriptor(kind="text", format="txt", ref=str(final_output_path)), - }, - metadata={"runtime": self._runtime_name, "agent": "codex"}, - ), - metadata={ - "runtime": self._runtime_name, - "agent": "codex", - "agent_model": self._model, - "agent_ok": True, - "seeded_files": seeded_files, - "generated": True, - }, - ) - - def _command(self, *, workspace_dir: Path, final_output_path: Path) -> list[str]: - command = [ - self._codex_bin, - "exec", - "--skip-git-repo-check", - "--ephemeral", - "--ignore-user-config", - "--sandbox", - "workspace-write", - "--cd", - str(workspace_dir), - "--output-last-message", - str(final_output_path), - "--json", - ] - if self._model is not None: - command.extend(["--model", self._model]) - command.append("-") - return command - - def _validate_artifact_permissions(self, evidence_dir: Path) -> None: - """Validate runtime-specific artifact postconditions after the process exits.""" - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = self._work_root - if root is None: - root = (config.work_dir or Path.cwd()) / "evidence" / "codex" - safe_task_id = _safe_path_name(task.id) - task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return Path(root) / task_dir - - -class CodexDockerCliAgentRuntime(CodexCliAgentRuntime): - """AgentTaskRunner that runs Codex CLI inside a Docker container.""" - - def __init__( - self, - *, - model: str | None = None, - work_root: str | Path | None = None, - docker_bin: str = "docker", - image: str = DEFAULT_CODEX_DOCKER_CLI_IMAGE, - codex_package: str = DEFAULT_CODEX_DOCKER_CLI_PACKAGE, - auth_path: str | Path | None = None, - timeout_s: int = DEFAULT_CODEX_TIMEOUT_S, - prompt_builder: CodexPromptBuilder | None = None, - process_factory: ProcessFactory | None = None, - ) -> None: - super().__init__( - model=model, - work_root=work_root, - timeout_s=timeout_s, - prompt_builder=prompt_builder, - process_factory=process_factory, - runtime_name="codex_docker_cli", - ) - self._docker_bin = docker_bin - self._image = image - self._codex_package = codex_package - self._auth_path = ( - Path(auth_path).expanduser() if auth_path is not None else Path.home() / ".codex" / "auth.json" - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - if shutil.which(self._docker_bin) is None: - raise RuntimeError(f"Docker executable {self._docker_bin!r} was not found on PATH") - if not self._auth_path.exists(): - raise RuntimeError( - f"Codex auth file was not found at {self._auth_path}. Run `codex login` or use OPENAI_API_KEY " - "so --runtime docker can use DockerSandboxAgentRuntime." - ) - - resolved_config = config or AgentEvalRunConfig() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(index, task, resolved_config) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - def _command(self, *, workspace_dir: Path, final_output_path: Path) -> list[str]: - evidence_dir = final_output_path.parent - inner_command = [ - "npx", - "-y", - self._codex_package, - "exec", - "--skip-git-repo-check", - "--ephemeral", - "--sandbox", - "danger-full-access", - "--cd", - "/workspace", - "--output-last-message", - "/evidence/final_output.txt", - "--json", - ] - if self._model is not None: - inner_command.extend(["--model", self._model]) - inner_command.append("-") - # Codex intentionally runs as root: the container mounts its auth under /root and coding tasks - # may need to install tools. Repair the bind-mounted trees before Docker returns so the host can - # score and persist every artifact the agent created without widening access to other host users. - # Capture the bind mount's owner as seen inside this container before Codex runs: raw host UID/GID - # values are not portable across Docker Desktop and rootless user-namespace mappings. Keep Codex - # failures authoritative; only surface the required chmod status when Codex itself succeeded. - shell_command = ( - "host_owner=\"$(stat -c '%u:%g' /evidence 2>/dev/null)\" || true; " - f"{shlex.join(inner_command)}; " - "codex_status=$?; " - 'if [ -n "$host_owner" ]; then ' - 'chown -R "$host_owner" /workspace /evidence 2>/dev/null || true; ' - "fi; " - "chmod -R u+rwX,go-rwx /workspace /evidence; " - "permissions_status=$?; " - 'if [ "$codex_status" -ne 0 ]; then exit "$codex_status"; fi; ' - 'exit "$permissions_status"' - ) - return [ - self._docker_bin, - "run", - "--rm", - "-i", - "-e", - "PYTHONDONTWRITEBYTECODE=1", - "-v", - f"{self._auth_path.resolve()}:/root/.codex/auth.json:ro", - "-v", - f"{workspace_dir.resolve()}:/workspace", - "-v", - f"{evidence_dir.resolve()}:/evidence", - self._image, - "sh", - "-lc", - shell_command, - ] - - def _validate_artifact_permissions(self, evidence_dir: Path) -> None: - _validate_private_tree(evidence_dir) - - -def resolve_codex_runtime( - *, - runtime: RuntimeChoice, - model: str | None, - output_dir: Path, - env: Mapping[str, str] = os.environ, - prompt_builder: CodexPromptBuilder | None = None, -) -> tuple[CodexCliAgentRuntime | CodexDockerCliAgentRuntime | DockerSandboxAgentRuntime, EffectiveCodexRuntime]: - """Pick and construct a Codex runtime for a run-mode + environment. - - ``local`` runs the on-PATH Codex CLI. ``docker`` prefers the OpenAI-Agents ``DockerSandbox`` when - ``OPENAI_API_KEY`` is an OpenAI platform secret (``sk-...``) and otherwise falls back to the - containerized Codex CLI (which mounts ``~/.codex/auth.json``). ``prompt_builder`` is threaded into - the CLI runtimes; the sandbox runtime does its own prompting. Returns the runtime plus the - :class:`EffectiveCodexRuntime` actually chosen so callers can label/report it. - """ - effective_runtime = _resolve_codex_runtime(runtime, env) - if effective_runtime == EffectiveCodexRuntime.LOCAL_CLI: - return ( - CodexCliAgentRuntime( - model=model, - work_root=output_dir / "evidence" / "codex", - prompt_builder=prompt_builder, - ), - effective_runtime, - ) - if effective_runtime == EffectiveCodexRuntime.DOCKER_CLI: - return ( - CodexDockerCliAgentRuntime( - model=model, - work_root=output_dir / "evidence" / "codex-docker", - prompt_builder=prompt_builder, - ), - effective_runtime, - ) - if effective_runtime == EffectiveCodexRuntime.DOCKER_SANDBOX: - return DockerSandboxAgentRuntime(model=model or DEFAULT_CODEX_DOCKER_MODEL), effective_runtime - raise ValueError(f"unsupported Codex runtime {runtime!r}") - - -def _resolve_codex_runtime(runtime: RuntimeChoice, env: Mapping[str, str] = os.environ) -> EffectiveCodexRuntime: - if runtime == RuntimeChoice.LOCAL: - return EffectiveCodexRuntime.LOCAL_CLI - if runtime == RuntimeChoice.DOCKER: - if _openai_sdk_secret_key_is_set(env): - return EffectiveCodexRuntime.DOCKER_SANDBOX - return EffectiveCodexRuntime.DOCKER_CLI - raise ValueError(f"unsupported Codex runtime {runtime!r}") - - -def _openai_sdk_secret_key_is_set(env: Mapping[str, str] = os.environ) -> bool: - return env.get("OPENAI_API_KEY", "").strip().startswith("sk-") - - -def list_codex_agent_models(*, codex_bin: str = "codex") -> list[dict[str, Any]]: - """Return visible Codex model descriptors from the local Codex CLI.""" - if shutil.which(codex_bin) is None: - raise RuntimeError(f"Codex CLI executable {codex_bin!r} was not found on PATH") - result = subprocess.run( - [codex_bin, "debug", "models"], - check=True, - capture_output=True, - text=True, - ) - payload = json.loads(result.stdout) - models = payload.get("models") - if not isinstance(models, list): - raise RuntimeError("Codex model catalog did not contain a models list") - visible = [model for model in models if isinstance(model, dict) and model.get("visibility") == "list"] - return sorted(visible, key=lambda model: int(model.get("priority") or 0), reverse=True) - - -def print_codex_agent_models(*, codex_bin: str = "codex") -> None: - """Print local Codex model slugs and display names.""" - for model in list_codex_agent_models(codex_bin=codex_bin): - slug = model.get("slug") - if not isinstance(slug, str): - continue - display_name = model.get("display_name") - if isinstance(display_name, str) and display_name != slug: - print(f"{slug}\t{display_name}") - else: - print(slug) - - -def _failed_codex_trial( - task: AgentEvalTask, - evidence_dir: Path | None, - exc: Exception, - *, - runtime_name: str = "codex_cli", - permission_cleanup_error: str | None = None, - artifact_persistence_error: str | None = None, -) -> AgentEvalTrial: - evidence: CandidateEvidence | None = None - error_artifact_error: str | None = None - if evidence_dir is not None: - error_path = evidence_dir / "error.json" - try: - _write_private_text( - error_path, json.dumps({"error_type": exc.__class__.__name__, "error": str(exc)}) + "\n" - ) - except Exception as artifact_exc: - error_artifact_error = f"{artifact_exc.__class__.__name__}: {artifact_exc}" - else: - evidence = CandidateEvidence( - descriptors={"error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path))}, - metadata={"runtime": runtime_name, "agent": "codex"}, - ) - - metadata: dict[str, Any] = { - "runtime": runtime_name, - "agent": "codex", - "agent_ok": False, - "error_type": exc.__class__.__name__, - "error": str(exc), - } - if permission_cleanup_error is not None: - metadata["permission_cleanup_error"] = permission_cleanup_error - if artifact_persistence_error is not None: - metadata["artifact_persistence_error"] = artifact_persistence_error - if error_artifact_error is not None: - metadata["error_artifact_error"] = error_artifact_error - return AgentEvalTrial( - id=f"{task.id}:codex", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=evidence, - metadata=metadata, - ) - - -def _ensure_private_directory(path: Path) -> None: - """Create or repair a host-owned directory without following a leaf symlink.""" - path.mkdir(mode=0o700, parents=True, exist_ok=True) - descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - try: - path_stat = os.fstat(descriptor) - if path_stat.st_uid != os.getuid(): - raise PermissionError(f"directory is not owned by the invoking host user: {path}") - os.fchmod(descriptor, 0o700) - finally: - os.close(descriptor) - - -def _write_private_text(path: Path, content: str) -> None: - """Atomically publish a host-created evidence artifact with owner-only access.""" - descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) - temporary_path = Path(temporary_name) - try: - os.fchmod(descriptor, 0o600) - temporary_file = os.fdopen(descriptor, "w", encoding="utf-8") - descriptor = -1 - with temporary_file: - temporary_file.write(content) - os.replace(temporary_path, path) - finally: - if descriptor != -1: - with contextlib.suppress(OSError): - os.close(descriptor) - temporary_path.unlink(missing_ok=True) - - -def _read_private_final_output(path: Path, *, fallback: str) -> str: - """Read a regular agent-created final output without following it, then republish it privately.""" - try: - descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) - except FileNotFoundError: - _write_private_text(path, fallback) - return fallback - - try: - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise PermissionError(f"final output is not a regular file: {path}") - with os.fdopen(descriptor, "r", encoding="utf-8") as output_file: - descriptor = -1 - output_text = output_file.read() - finally: - if descriptor != -1: - os.close(descriptor) - - _write_private_text(path, output_text) - return output_text - - -def _validate_private_tree(root: Path) -> None: - """Require a host-owned, owner-only tree without following agent-created symlinks.""" - expected_uid = os.getuid() - pending = [root] - while pending: - path = pending.pop() - path_stat = path.lstat() - if stat.S_ISLNK(path_stat.st_mode): - continue - if path_stat.st_uid != expected_uid: - raise PermissionError(f"artifact is not owned by the invoking host user: {path}") - - mode = stat.S_IMODE(path_stat.st_mode) - if mode & 0o077: - raise PermissionError(f"artifact grants group or other access: {path} ({mode:o})") - if stat.S_ISDIR(path_stat.st_mode): - if mode & 0o700 != 0o700: - raise PermissionError(f"directory is not owner-readable, writable, and traversable: {path} ({mode:o})") - with os.scandir(path) as entries: - pending.extend(Path(entry.path) for entry in entries) - elif stat.S_ISREG(path_stat.st_mode): - if mode & 0o600 != 0o600: - raise PermissionError(f"file is not owner-readable and writable: {path} ({mode:o})") - else: - raise PermissionError(f"artifact is not a regular file or directory: {path}") - - -async def _terminate_process(process: Any | None) -> None: - if process is None or process.returncode is not None: - return - process.kill() - with contextlib.suppress(Exception): - await process.wait() - - -def _decode_process_output(value: bytes | str | None) -> str: - if value is None: - return "" - if isinstance(value, str): - return value - return value.decode("utf-8", errors="replace") - - -def _safe_path_name(value: str) -> str: - return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/runtime.py index dd15928376..85e525d903 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/runtime.py @@ -71,8 +71,8 @@ def __init__(self, *, config: GymRuntimeConfig) -> None: def config(self) -> GymRuntimeConfig: """The settings this runner was constructed with. - Read-only, and the whole config rather than a property per field: unlike the Codex and - Fabric runtimes, everything shaping a Gym run already lives in one validated object. + Read-only, and the whole config rather than a property per field: unlike the Fabric + runtimes, everything shaping a Gym run already lives in one validated object. Exposed so a live runner can be described as the job-spec target that reproduces it, without reaching into a private attribute from another package. ``runner_info()`` cannot serve that diff --git a/skills/nemo-evaluator-plugin/SKILL.md b/skills/nemo-evaluator-plugin/SKILL.md index 248abf92d0..097ad7c476 100644 --- a/skills/nemo-evaluator-plugin/SKILL.md +++ b/skills/nemo-evaluator-plugin/SKILL.md @@ -151,9 +151,9 @@ Use `AgentEvaluator().run(...)` for standalone task-driven SDK evaluation. Its **Platform job evaluation** Use the plugin `agent-evaluate submit` job for platform task evaluation. Its -target is a `ModelTarget`, `AgentTarget`, `CodexRunnerTarget`, -`FabricRunnerTarget`, or `HarborRunnerTarget`; alternatively provide -precomputed `trials`. Provide exactly one of `target` or `trials`. +target is a `ModelTarget`, `AgentTarget`, `FabricRunnerTarget`, or +`HarborRunnerTarget`; alternatively provide precomputed `trials`. Provide +exactly one of `target` or `trials`. Submission accepts inline tasks or a stored `TasksetRef`. Stored tasksets are resolved in the target workspace. diff --git a/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py b/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py index d047758c55..16e6157254 100644 --- a/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py +++ b/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py @@ -91,7 +91,7 @@ def build_agent_eval_spec(metric_bundle: Any) -> Any: from nemo_evaluator.jobs.agent_spec import ( AgentEvalInputSpec, AgentEvalTaskInput, - CodexRunnerTarget, + FabricRunnerTarget, ) return AgentEvalInputSpec( @@ -103,7 +103,12 @@ def build_agent_eval_spec(metric_bundle: Any) -> Any: metrics=[metric_bundle], ) ], - target=CodexRunnerTarget(), + target=FabricRunnerTarget( + config={ + "metadata": {"name": "geography-smoke"}, + "harness": {"adapter_id": "nvidia.fabric.codex"}, + } + ), max_concurrent_tasks=2, labels={"benchmark": "geography-smoke"}, ) diff --git a/skills/nemo-evaluator-plugin/references/agent-evaluation.md b/skills/nemo-evaluator-plugin/references/agent-evaluation.md index 676c129abc..351bd403a4 100644 --- a/skills/nemo-evaluator-plugin/references/agent-evaluation.md +++ b/skills/nemo-evaluator-plugin/references/agent-evaluation.md @@ -60,7 +60,7 @@ from nemo_evaluator.api.schemas import TaskInputs from nemo_evaluator.jobs.agent_spec import ( AgentEvalInputSpec, AgentEvalTaskInput, - CodexRunnerTarget, + FabricRunnerTarget, ) spec = AgentEvalInputSpec( @@ -72,7 +72,12 @@ spec = AgentEvalInputSpec( metrics=[metric_bundle], ) ], - target=CodexRunnerTarget(), + target=FabricRunnerTarget( + config={ + "metadata": {"name": "geography-smoke"}, + "harness": {"adapter_id": "nvidia.fabric.codex"}, + } + ), max_concurrent_tasks=2, fail_fast=False, labels={"benchmark": "geography-smoke"}, @@ -100,7 +105,6 @@ reported score. See | --- | --- | | `ModelTarget` | Generate trials through an OpenAI-compatible model endpoint | | `AgentTarget` | Generate trials through a generic HTTP or NeMo Agent Toolkit agent | -| `CodexRunnerTarget` | Drive the Codex CLI runner | | `FabricRunnerTarget` | Run a configured NeMo [Fabric](https://github.com/nvidia/nemo-fabric) runner | | `HarborRunnerTarget` | Run a Harbor task suite in Docker | diff --git a/skills/nemo-evaluator-plugin/references/troubleshooting.md b/skills/nemo-evaluator-plugin/references/troubleshooting.md index 98a08bd85e..864cb7e77d 100644 --- a/skills/nemo-evaluator-plugin/references/troubleshooting.md +++ b/skills/nemo-evaluator-plugin/references/troubleshooting.md @@ -31,7 +31,7 @@ nemo evaluator agent-evaluate explain | Result download fails while progress shows 100% | Metric progress finished before the platform job finalized artifacts | Call `job.wait_until_done()` before `get_result()` or `download_artifacts()` | | Agent-eval rejects the spec | Both or neither of `target` and `trials` were provided | Provide exactly one | | Taskset evaluation lacks held-out reference data | Stored tasks do not carry grader-only `reference` | Use inline `AgentEvalTaskInput` when the metric needs held-out per-task data | -| Runner target fails to start | The runtime dependency, CLI, config, credentials, or Docker access is missing | Check the selected Codex, Fabric, or Harbor runner prerequisites | +| Runner target fails to start | The runtime dependency, CLI, config, credentials, or Docker access is missing | Check the selected runner's prerequisites | ## Debug in the smallest scope