feat(eval-author)!: replace the CLI with skills for Harbor eval discovery

docs/agents/insight-driven-optimization.mdx

@@ -285,11 +285,6 @@ evaluation suite for validating candidates that address the Insight. The
 Experimenter invokes the Eval Author workflow in Insight mode and reads the
 `eval_author` section of the experiment configuration.
 
-The plugin also exposes the canonical `nemo agents eval-author` command
-namespace with `discover`, `audit`, `propose`, `run`, and `doctor` verbs. These
-standalone verbs are currently scaffolding and exit with a nonzero status until
-their implementations are available.
-
 ## Get Started
 
 ### Set Up
@@ -325,7 +320,6 @@ Confirm the plugins are installed and discoverable:
 ```bash
 nemo agents analyst --help
 nemo agents experimentalist --help
-nemo agents eval-author --help
 ```
 
 From an agent directory with an `optimizer.yaml` profile, check that the
@@ -529,8 +523,7 @@ seconds and submits a run after the scheduled daily or weekly window is reached.
 ## Command Reference
 
 The Analyst lives under `nemo agents analyst`, and the Experimenter lives under
-`nemo agents experimentalist`. The Eval Author command surface lives under
-`nemo agents eval-author`. These command namespaces share the `optimizer.yaml`
+`nemo agents experimentalist`. These command namespaces share the `optimizer.yaml`
 profile, and each validates its own configuration. Scheduled analysis remains
 under `nemo insights analysis`.
 
@@ -586,12 +579,6 @@ Run the local Experimenter loop.
 Diagnose the Experimenter setup: profile, configured models, Insight resolution,
 datasets, and the experiment plan.
 
-### `nemo agents eval-author`
-
-Discover the current Eval Author command surface with `--help`. The
-`discover`, `audit`, `propose`, `run`, and `doctor` verbs are placeholders and
-exit with a nonzero status until their implementations are available.
-
 ### Models
 
 The optimization agents use the active Platform CLI context:

packages/nemo_platform/pyproject.toml

@@ -291,16 +291,6 @@ nemo-deployments-plugin = [
   "pyyaml>=6.0",
 ]
 
-# Generated from [tool.bundle-package]; do not edit by hand.
-nemo-eval-author-plugin = [
-  "pydantic>=2",
-  "harbor>=0.18",
-  "nemo-experimentalist-plugin",
-  "nemo-insights-plugin",
-  "nemo-platform-plugin",
-  "pyyaml>=6.0.3",
-]
-
 # Generated from [tool.bundle-package]; do not edit by hand.
 nemo-evaluator-plugin = [
   "cloudpickle>=3.1.1",
@@ -488,7 +478,6 @@ plugins = [
     "nemo-platform[nemo-auditor-plugin]",
     "nemo-platform[nemo-data-designer-plugin]",
     "nemo-platform[nemo-deployments-plugin]",
-    "nemo-platform[nemo-eval-author-plugin]",
     "nemo-platform[nemo-evaluator-plugin]",
     "nemo-platform[nemo-experimentalist-plugin]",
     "nemo-platform[nemo-guardrails-plugin]",
@@ -569,7 +558,6 @@ insights = "nemo_insights_plugin.cli:InsightsCLI"
 
 # Generated from [tool.bundle-package]; do not edit this table by hand.
 [project.entry-points."nemo.cli.agents"]
-eval-author = "nemo_eval_author_plugin.cli:EvalAuthorCLI"
 experimentalist = "nemo_experimentalist_plugin.cli:ExperimentalistCLI"
 analyst = "nemo_insights_plugin.analyst.cli:AnalystCLI"
 
@@ -728,11 +716,6 @@ nemo-anonymizer-plugin = { source = "../../plugins/nemo-anonymizer/src/nemo_anon
 nemo-auditor-plugin = { source = "../../plugins/nemo-auditor/src/nemo_auditor", module = "nemo_auditor", inherit = { "entry-points" = ["nemo.*"] } }
 nemo-data-designer-plugin = { source = "../../plugins/nemo-data-designer/src/nemo_data_designer_plugin", module = "nemo_data_designer_plugin", inherit = { "entry-points" = ["nemo.*"] } }
 nemo-deployments-plugin = { source = "../../plugins/nemo-deployments/src/nemo_deployments_plugin", module = "nemo_deployments_plugin", inherit = { "entry-points" = ["nemo.*"] } }
-# Eval Author, Experimentalist, and Insights ship together: Experimentalist imports
-# EvalAuthor and EvalAuthorConfig at module scope, Eval Author still borrows
-# Experimentalist helpers, and both read the Insights profile contract. Bundling all
-# three into one distribution collapses that dependency cycle into intra-wheel extras.
-nemo-eval-author-plugin = { source = "../../plugins/nemo-eval-author/src/nemo_eval_author_plugin", module = "nemo_eval_author_plugin", inherit = { "entry-points" = ["nemo.*"] } }
 nemo-evaluator-plugin = { source = "../../plugins/nemo-evaluator/src/nemo_evaluator", module = "nemo_evaluator", inherit = { "entry-points" = ["nemo.*"] } }
 nemo-experimentalist-plugin = { source = "../../plugins/nemo-experimentalist/src/nemo_experimentalist_plugin", module = "nemo_experimentalist_plugin", inherit = { "entry-points" = ["nemo.*"] } }
 nemo-insights-plugin = { source = "../../plugins/nemo-insights/src/nemo_insights_plugin", module = "nemo_insights_plugin", inherit = { "entry-points" = ["nemo.*"] } }

plugins/README.md

@@ -64,7 +64,6 @@ The package name is the `name` field in the plugin's `pyproject.toml`, not the d
 | `nemo-agents/` | `nemo-agents-plugin` |
 | `nemo-anonymizer/` | `nemo-anonymizer-plugin` |
 | `nemo-data-designer/` | `nemo-data-designer-plugin` |
-| `nemo-eval-author/` | `nemo-eval-author-plugin` |
 | `nemo-evaluator/` | `nemo-evaluator-plugin` |
 | `nemo-experimentalist/` | `nemo-experimentalist-plugin` |
 | `nemo-guardrails/` | `nemo-guardrails-plugin` |

plugins/nemo-eval-author/README.md

@@ -1,26 +1,50 @@
 <!-- SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
 <!-- SPDX-License-Identifier: Apache-2.0 -->
 
-# NeMo Eval Author Plugin
+# NeMo Eval Author
 
-Owns the `nemo agents eval-author` command group, registered under `nemo.cli.agents` and
-mounted by the agents plugin. `discover` is implemented; `audit`, `propose`, `run`, and
-`doctor` are placeholders.
+Two skills that an agent reads to work on the evaluation suites in a user's own
+repository. There is no CLI, no service, and no importable code, so this directory
+builds no package at all. A customer points their agent at `skills/` and nothing
+gets installed.
 
-Use `discover` only with a trusted repository, because importing an agent runs
-module top-level code.
+| Skill | Role |
+| --- | --- |
+| [`eval-author`](skills/eval-author/SKILL.md) | Core. Owns the standard every sub-flow follows and routes to one. |
+| [`eval-author-discover`](skills/eval-author-discover/SKILL.md) | Sub-flow. Records whether a repository's Harbor evals are ready to run. |
 
-The Eval Author agent moved into the Experimentalist plugin, at
-[`nemo_experimentalist_plugin.eval_author`](../nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md).
-Experimentalist insight mode is its only caller, so the agent sits beside the evaluator,
-staging, and trace helpers it depends on.
+## Where findings go
 
-## Direction of travel
+`eval-author-discover` leaves a report at `.eval-author/discovery.md`, carrying the
+JSON as front matter so a later model reads the verdict without Harbor. It is
+visible and worth committing: a teammate who reads it skips the discovery pass.
 
-The dependency is one arrow. `discovery/run.py` borrows `make_client` from Experimentalist,
-and Experimentalist imports nothing from here, so there is no package cycle for `uv` to
-resolve. Install both plugins with:
+The scripts write no files. They report to stdout and the skill tells the agent
+where to save, because that is a judgement about someone's repository.
 
-```bash
-uv sync --group experimentalist
-```
+## Why skills instead of an agent
+
+Harbor tasks live in the customer's repository, so an agent that proposes changes
+has to write to that repository. Customers were unwilling to grant that, sandboxed
+or not. A skill inverts the arrangement: the customer's own agent does the work,
+and this directory only supplies the instructions and the deterministic scripts.
+
+The Eval Author agent that Experimentalist insight mode still uses lives in
+[the Experimentalist plugin](../nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md).
+
+## Dependencies
+
+The scripts under `skills/*/scripts/` import nothing beyond the standard library and
+Harbor itself, so they run on whatever Python the customer already has. Where a real
+answer needs a provider, the skill defers to the provider's own validators rather
+than guessing from file layout, which is why `eval-author-discover` probes for an
+installed Harbor and asks Harbor to judge each config.
+
+`tests/test_skill_contract.py` holds to the same boundary and imports nothing from
+the platform, so `pytest` and `pyyaml` are enough to run it. The five tests that
+make Harbor judge a fixture suite skip when Harbor is absent, which is why this
+directory declares no dependencies and appears in no dependency group.
+
+Adding a runtime dependency to a bundled script is a breaking change for anyone who
+copied the skill, so the contract test walks each script's imports and fails on
+anything outside the standard library, a sibling module, or Harbor.

plugins/nemo-eval-author/pyproject.toml

@@ -4,30 +4,16 @@
 [project]
 name = "nemo-eval-author-plugin"
 version = "0.1.0"
-description = "Eval Author commands for NeMo Platform (borrows the Experimentalist platform client)."
+description = "Eval Author skills that an agent reads to discover a repository's Harbor eval setup."
 requires-python = ">=3.12,<3.14"
-dependencies = [
-    "pydantic>=2",
-    # Harbor 0.18 provides the discovery APIs used by this plugin.
-    "harbor>=0.18",
-    "nemo-experimentalist-plugin",
-    "nemo-insights-plugin",
-    "nemo-platform",
-    "nemo-platform-plugin",
-    "pyyaml>=6.0.3",
-]
 
-[project.entry-points."nemo.cli.agents"]
-eval-author = "nemo_eval_author_plugin.cli:EvalAuthorCLI"
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.wheel]
-packages = ["src/nemo_eval_author_plugin"]
+[tool.uv]
+# Nothing here is importable. The directory holds two skills and the test that guards
+# them, so there is no package to build and nothing to install. That is also why the
+# skills sit at the top level instead of under `src/<module>/skills/`: that layout exists
+# to make skills importable and shippable through the `nemo.skills` entry point, and this
+# plugin deliberately does not plug into skill discovery yet.
+package = false
 
 [tool.pytest.ini_options]
-asyncio_mode = "auto"
-pythonpath = ["src"]
 testpaths = ["tests"]