Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/wheel-constraints/nemo-platform-services.txt
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,10 @@ langchain-openai==1.5.0
lark==1.3.1
matplotlib==3.11.1
nemo-anonymizer==0.3.3
nemo-fabric==0.1.1
nemo-fabric-adapters-hermes==0.1.1
nemo-fabric==0.2.0
nemo-fabric-adapters-hermes==0.2.0
nemoguardrails==0.23.0
nemo-relay==0.6.0
nemo-relay==0.7.2
nemo-safe-synthesizer==0.1.7
ngcsdk==4.34.10
numpy==2.5.2
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# Keep the Fabric runtime aligned with the packaged agent contract.
nemo-fabric==0.1.1
nemo-fabric==0.2.0
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,13 @@ async def main() -> int:
# requirements.env); it resolves via the default LocalSecretResolver (from the host env) and injects
# the value into the container. No raw credential on the API surface.
runtime = FabricContainerRuntime(
# Typed hermes-SDK agent config: in-library transport, model-only (no codex/node). The harness is
# chosen by harness.adapter_id, never inferred from the model.
# Typed Hermes agent config: model-only (no codex/node). The harness is chosen by
# harness.adapter_id, never inferred from the model; Fabric owns its execution mechanism.
FabricConfig(
metadata=MetadataConfig(name="hermes-eval"),
harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"),
models={"default": {"provider": "nvidia", "model": model}},
runtime=RuntimeConfig.from_mapping(
{"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"}
),
runtime=RuntimeConfig(input_schema="chat", output_schema="message"),
),
provider=DockerSandboxProvider(),
secrets={"NVIDIA_API_KEY": SecretRef(root="NVIDIA_API_KEY")},
Expand Down
23 changes: 11 additions & 12 deletions packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,17 @@
The same :class:`~nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime.FabricAgentRuntime` targets
different agent harnesses purely via the Fabric config — the harness is selected by its
``harness.adapter_id``, never inferred from a model. Across harnesses the shape differs mainly in that
``adapter_id``, ``runtime.transport``, and any harness-specific ``harness.settings``:
``adapter_id``, optional input/output schemas, and any harness-specific ``harness.settings``:

* **Codex CLI** (``nvidia.fabric.codex``) runs the agent as a subprocess — ``transport="cli"`` —
and takes codex-specific ``harness.settings`` such as sandbox and approval modes.
* **Hermes SDK** (``nvidia.fabric.hermes``) runs in-library — ``transport="library"`` — and
declares its ``input``/``output`` schemas instead.
* **Codex** (``nvidia.fabric.codex``) takes codex-specific ``harness.settings`` such as sandbox and
approval modes.
* **Hermes** (``nvidia.fabric.hermes``) declares its ``input``/``output`` schemas.

Fabric owns each adapter's execution mechanism; callers select the adapter rather than configuring a
CLI-versus-library transport.

An optional ``model=`` slug (e.g. ``"openai/gpt-5.4"``) can be passed to ``FabricAgentRuntime`` to
overlay the model as a final profile, mirroring Fabric's own Harbor integration.
apply the model to each task config, mirroring Fabric's own Harbor integration.

The configs are built from ``nemo_fabric``'s typed config objects (``FabricConfig`` etc.), which
validate structure at construction. That makes this module — like any real Fabric use — require the
Expand All @@ -36,25 +38,22 @@
RuntimeConfig,
)

#: Codex CLI harness — subprocess transport, codex-specific harness settings.
#: Codex harness with codex-specific settings.
CODEX_CLI_CONFIG = FabricConfig(
metadata=MetadataConfig(name="codex-eval"),
harness=HarnessConfig(
adapter_id="nvidia.fabric.codex",
settings={"sandbox": "read-only"},
),
models={"default": ModelConfig(provider="openai", model="gpt-5.4")},
runtime=RuntimeConfig.from_mapping({"mode": "oneshot", "transport": "cli"}),
)

#: Hermes SDK harness — in-library transport, explicit chat/message schemas.
#: Hermes harness with explicit chat/message schemas.
HERMES_SDK_CONFIG = FabricConfig(
metadata=MetadataConfig(name="hermes-eval"),
harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"),
models={"default": ModelConfig(provider="nvidia", model="qwen2.5-coder-32b")},
runtime=RuntimeConfig.from_mapping(
{"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"}
),
runtime=RuntimeConfig(input_schema="chat", output_schema="message"),
)

#: Named Fabric configs, one per harness, keyed by a short label.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,14 @@ def _fabric_config(harness: str, *, provider: str, model: str, api_key_env: str)
"metadata": {"name": "lab-fabric-eval"},
"models": {"default": {"provider": provider, "model": model, "api_key_env": api_key_env}},
}
# Adapter ids are the base harness name; transport is set separately in `runtime` (newer nemo-fabric
# dropped the `.cli`/`.sdk` suffix from adapter ids).
# Adapter ids select the harness implementation directly; Fabric owns its execution mechanism.
if harness == "deepagents":
# LangChain Deep Agents — provider-agnostic; for provider=nvidia it targets NVIDIA's
# OpenAI-compatible endpoint. The recommended harness for NVIDIA-hosted models.
return {
**common,
"harness": {"adapter_id": "nvidia.fabric.langchain.deepagents"},
"runtime": {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"},
"runtime": {"input_schema": "chat", "output_schema": "message"},
}
if harness == "codex-cli":
# codex runs as the SDK adapter here; CLI-only settings (e.g. skip_git_repo_check) are rejected.
Expand All @@ -85,12 +84,11 @@ def _fabric_config(harness: str, *, provider: str, model: str, api_key_env: str)
"config_overrides": {"web_search": "disabled", "sandbox_workspace_write.network_access": False},
},
},
"runtime": {"mode": "oneshot", "transport": "cli"},
}
return { # hermes-sdk
**common,
"harness": {"adapter_id": "nvidia.fabric.hermes", "resolution": "preinstalled"},
"runtime": {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"},
"runtime": {"input_schema": "chat", "output_schema": "message"},
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ async def _main() -> int:
"settings": {"max_iterations": 50},
},
"models": {"default": {"provider": "nvidia", "model": MODEL}},
"runtime": {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"},
"runtime": {"input_schema": "chat", "output_schema": "message"},
}

baseline_runtime = FabricAgentRuntime(config=fabric_config)
Expand Down
2 changes: 1 addition & 1 deletion packages/nemo_evaluator_sdk/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ dependencies = [
"ragas==0.4.3",
"langchain-openai>=1.3.5",
"langchain-nvidia-ai-endpoints>=1.4.3,<2.0.0",
"nemo-relay>=0.6.0,<0.7",
"nemo-relay>=0.7.2,<0.8",
# Fabric's typed config surface (FabricConfig/RuntimeConfig/ModelConfig/the relay models) — what the
# agent-eval runtimes compose against. The metapackage alone is ~2 MB / 3 packages; it requires
# nemo-fabric-runtime unconditionally as of rc4, which is why the `runtime` extra it used to expose
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ def _resolve_skill_mode(self) -> SkillMode | None:
probe_config = FabricConfig.from_mapping(self._config)
probe_config.add_skill_path(_SKILL_PROBE_PATH)
plan = Fabric().plan(probe_config)
return resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness)
return resolve_skill_mode(capability_plan=plan.capability_plan, adapter_id=self._adapter_id())

def _adapter_id(self) -> str:
"""The harness adapter id declared by the config mapping (for provenance + error messages)."""
Expand Down Expand Up @@ -348,7 +348,7 @@ def _composed_config(self, skill_paths: Sequence[str] = ()) -> dict[str, Any]:
config = dict(self._config)

# Each section is spread over the caller's, so sibling keys survive — pinning
# ``runtime.artifacts`` must not drop a configured ``runtime.transport``.
# ``runtime.artifacts`` must not drop configured input/output schemas or timeouts.
config["runtime"] = {**_section(config, "runtime"), "artifacts": _ARTIFACTS_DIR}
# ``provider: local`` is required by the native planner in the container (it does not inject the
# Python default), and the workspace pins the harness cwd to the retrievable /out subtree.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,9 @@ class FabricAgentRuntime:
"""AgentTaskRunner that generates trials by running tasks through NeMo Fabric.

The harness is selected entirely by ``config["harness"]["adapter_id"]``. Across harnesses the
config shape differs mainly in that ``adapter_id``, ``runtime.transport``, and any harness-specific
``harness.settings`` — e.g. Codex runs as a subprocess (``transport="cli"``) while the Hermes SDK
harness runs in-library (``transport="library"``). See
``examples/fabric_harness_runtimes.py`` for full Codex-CLI and Hermes-SDK config examples.
config shape differs mainly in that ``adapter_id``, optional input/output schemas, and any
harness-specific ``harness.settings``. Fabric owns each adapter's execution mechanism. See
``examples/fabric_harness_runtimes.py`` for full Codex and Hermes config examples.
"""

def __init__(
Expand Down Expand Up @@ -271,7 +270,10 @@ def _resolve_skill_mode(self, client: Fabric, agent_config: FabricConfig) -> Ski
probe_config = agent_config.model_copy(deep=True)
probe_config.add_skill_path(_SKILL_PROBE_PATH)
plan = client.plan(probe_config, base_dir=self._base_dir)
return resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness)
return resolve_skill_mode(
capability_plan=plan.capability_plan,
adapter_id=agent_config.harness.adapter_id,
)

async def _run_task(
self,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@
# runtime plans a probe skill path and reads these to decide the injection mode (see resolve_skill_mode).
_SKILLS_ROUTE_KIND = "skills"
_SKILLS_TARGET_NATIVE = "harness_native"
# Fabric harness name of the Codex CLI adapter, which self-discovers ``.agents/skills/`` rather than
# Fabric adapter id of Codex, which self-discovers ``.agents/skills/`` rather than
# accepting the native ``skills`` config.
_CODEX_HARNESS = "codex"
_CODEX_ADAPTER_ID = "nvidia.fabric.codex"


class SkillInjectionError(ValueError):
Expand Down Expand Up @@ -165,7 +165,7 @@ def native_skills_route(capability_plan: Mapping[str, object]) -> bool:
)


def resolve_skill_mode(*, capability_plan: Mapping[str, object], harness: str) -> SkillMode | None:
def resolve_skill_mode(*, capability_plan: Mapping[str, object], adapter_id: str) -> SkillMode | None:
"""Resolve how a skill would reach the selected harness, or ``None`` if it can't.

Driven by Fabric's own capability routing (queried at runtime via ``Fabric.plan``) rather than a
Expand All @@ -178,7 +178,7 @@ def resolve_skill_mode(*, capability_plan: Mapping[str, object], harness: str) -
"""
if native_skills_route(capability_plan):
return SKILL_MODE_NATIVE
if harness.strip().lower() == _CODEX_HARNESS:
if adapter_id.strip().lower() == _CODEX_ADAPTER_ID:
return SKILL_MODE_CODEX_SKILLS_DIR
return None

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -270,8 +270,6 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None
"settings": {"sandbox": "workspace-write"},
},
"runtime": {
"mode": "oneshot",
"transport": "cli",
"input_schema": "text",
"output_schema": "message",
"timeout_seconds": 180,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,22 +79,22 @@ def test_native_skills_route(capability_plan: dict[str, object], expected: bool)


@pytest.mark.parametrize(
("capability_plan", "harness", "expected"),
("capability_plan", "adapter_id", "expected"),
[
# Native routing wins regardless of harness name (e.g. Hermes, or an end-user adapter).
(_plan(native=True), "hermes", SKILL_MODE_NATIVE),
(_plan(native=True), "acme-custom", SKILL_MODE_NATIVE),
# Not native, but a codex harness -> self-discovered .agents/skills dir.
(_plan(native=False), "codex", SKILL_MODE_CODEX_SKILLS_DIR),
(_plan(native=None), "codex", SKILL_MODE_CODEX_SKILLS_DIR),
(_plan(native=False), "CODEX", SKILL_MODE_CODEX_SKILLS_DIR), # case-insensitive
# Native routing wins regardless of adapter id (e.g. Hermes, or an end-user adapter).
(_plan(native=True), "nvidia.fabric.hermes", SKILL_MODE_NATIVE),
(_plan(native=True), "acme.custom", SKILL_MODE_NATIVE),
# Not native, but the Codex adapter -> self-discovered .agents/skills dir.
(_plan(native=False), "nvidia.fabric.codex", SKILL_MODE_CODEX_SKILLS_DIR),
(_plan(native=None), "nvidia.fabric.codex", SKILL_MODE_CODEX_SKILLS_DIR),
(_plan(native=False), "NVIDIA.FABRIC.CODEX", SKILL_MODE_CODEX_SKILLS_DIR), # case-insensitive
# Neither native nor codex -> unsupported (runtime fails fast).
(_plan(native=False), "hermes", None),
(_plan(native=None), "some-other", None),
(_plan(native=False), "nvidia.fabric.hermes", None),
(_plan(native=None), "acme.other", None),
],
)
def test_resolve_skill_mode(capability_plan: dict[str, object], harness: str, expected: str | None) -> None:
assert resolve_skill_mode(capability_plan=capability_plan, harness=harness) == expected
def test_resolve_skill_mode(capability_plan: dict[str, object], adapter_id: str, expected: str | None) -> None:
assert resolve_skill_mode(capability_plan=capability_plan, adapter_id=adapter_id) == expected


def test_install_native_stages_named_dir_and_overlay(tmp_path: Path) -> None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,13 +79,13 @@ def _codex_adapter_installed() -> bool:
_HERMES_CONFIG = {
"metadata": {"name": "fabric-surface-hermes"},
"harness": {"adapter_id": _HERMES_ADAPTER_ID, "resolution": "preinstalled"},
"runtime": {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"},
"runtime": {"input_schema": "chat", "output_schema": "message"},
"environment": {"provider": "local"},
}
_CODEX_CONFIG = {
"metadata": {"name": "fabric-surface"},
"harness": {"adapter_id": _CODEX_ADAPTER_ID, "resolution": "preinstalled"},
"runtime": {"mode": "oneshot", "transport": "cli", "input_schema": "text", "output_schema": "message"},
"runtime": {"input_schema": "text", "output_schema": "message"},
"environment": {"provider": "local"},
}

Expand Down Expand Up @@ -203,7 +203,10 @@ def test_hermes_routes_skills_natively_per_the_real_planner() -> None:
plan = Fabric().plan(probe)

assert plan.adapter.adapter_id == _HERMES_ADAPTER_ID
assert resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) == SKILL_MODE_NATIVE
assert (
resolve_skill_mode(capability_plan=plan.capability_plan, adapter_id=plan.adapter.adapter_id)
== SKILL_MODE_NATIVE
)


@requires_harness_adapters
Expand All @@ -225,4 +228,7 @@ def test_codex_also_routes_skills_natively_so_the_workspace_branch_is_a_fallback
plan = Fabric().plan(probe)

assert "skills" in plan.capability_plan.get("routes", [])[0].get("kind", "")
assert resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) == SKILL_MODE_NATIVE
assert (
resolve_skill_mode(capability_plan=plan.capability_plan, adapter_id=plan.adapter.adapter_id)
== SKILL_MODE_NATIVE
)
6 changes: 3 additions & 3 deletions packages/nemo_platform/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@ nemo-agents-plugin = [
"pyyaml>=6.0",
"anthropic>=0.88.0",
"rich>=13.7.1",
"nemo-fabric[claude,codex,deepagents,relay]>=0.1.0,<0.3.0",
"nemo-fabric-adapters-hermes>=0.1.0,<0.3.0; python_version < '3.14'",
"nemo-fabric[claude,codex,deepagents,relay]>=0.2.0,<0.3.0",
"nemo-fabric-adapters-hermes>=0.2.0,<0.3.0; python_version < '3.14'",
]

# Generated from [tool.bundle-package]; do not edit by hand.
Expand Down Expand Up @@ -317,7 +317,7 @@ nemo-evaluator-sdk = [
"ragas==0.4.3",
"langchain-openai>=1.3.5",
"langchain-nvidia-ai-endpoints>=1.4.3,<2.0.0",
"nemo-relay>=0.6.0,<0.7",
"nemo-relay>=0.7.2,<0.8",
"nemo-fabric>=0.1.1,<0.3.0",
]

Expand Down
6 changes: 2 additions & 4 deletions plugins/nemo-agents/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,11 @@ dependencies = [
# improvement/ subpackage — agent-improvement workflow (POC).
"anthropic>=0.88.0",
"rich>=13.7.1",
# Floor stays at 0.1.0 so published wheels remain PyPI-resolvable (wheelcheck / consumers).
# Workspace uv.sources pins the mid-Aug main SHA (0.2.0 content) until PyPI ships 0.2.0.
"nemo-fabric[claude,codex,deepagents,relay]>=0.1.0,<0.3.0",
"nemo-fabric[claude,codex,deepagents,relay]>=0.2.0,<0.3.0",
# TODO(AIRCORE-952): Switch to the metapackage's `hermes-agent` extra once hermes-agent
# relaxes its vulnerable exact dependency pins — that extra applies [harness], which
# pins requests==2.33.0.
"nemo-fabric-adapters-hermes>=0.1.0,<0.3.0; python_version < '3.14'",
"nemo-fabric-adapters-hermes>=0.2.0,<0.3.0; python_version < '3.14'",
]
version = "0.0.0"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
from __future__ import annotations

import os
from pathlib import Path
from typing import Any

# CI type-checks this plugin via ty extra-paths without installing nemo-agents deps.
import nemo_fabric as fabric # ty: ignore[unresolved-import]
import nemo_fabric as fabric
from nemo_agents_plugin.agent_config import AgentConfig, HarnessConfig, ModelConfig
from nemo_agents_plugin.fabric.gateway_credentials import (
bind_platform_gateway_model_credential,
Expand Down Expand Up @@ -133,7 +133,9 @@ def _instructions_config(config: AgentConfig) -> Any:
def _skills_config(config: AgentConfig) -> Any:
if config.skills is None:
return None
return fabric.SkillConfig(paths=config.skills.paths)
paths: list[str | Path] = []
paths.extend(config.skills.paths)
return fabric.SkillConfig(paths=paths)


def _mcp_config(config: AgentConfig) -> Any:
Expand Down Expand Up @@ -168,7 +170,7 @@ def _apply_telemetry(fabric_config: Any, config: AgentConfig, model: ModelConfig

def _relay_observability_config(config: AgentConfig, model: ModelConfig) -> dict[str, Any]:
telemetry = config.telemetry
observability: dict[str, Any] = {"version": 2}
observability: dict[str, Any] = {"version": 3}

if telemetry.atif is not None:
atif = dict(telemetry.atif)
Expand Down
Loading
Loading