feat(mcp-readability): compliance orchestrator, LLM judge, and metrics scorer

datasets/mcp_readability/run_config.yaml

@@ -12,24 +12,32 @@
 
 orchestrator: mcp_readability
 
+# Plug-and-play scorers. Each key names a registered mcp_readability scorer
+# (SCORER_REGISTRY in the orchestrator) and carries that scorer's own config.
+# The orchestrator runs every scorer declared here against each endpoint and
+# merges their result columns; the shared analyzer aggregates a per-scorer pass
+# rate (comparator = scorer name) for the scores / summary reports. Add a scorer
+# (e.g. conformance) by registering the class and adding a block here.
+scorers:
+  # Deterministic size metrics. Pass = within token budget. `token_budget` feeds
+  # mcp_readability_token_budget_used_percent (endpoints may override per-entry).
+  mcp_tool_metrics:
+    token_budget: 200000
+  # LLM judge vs the style guide. Pass = no P0 findings. Owns its model config
+  # and style-guide path. Run with EVAL_GCP_PROJECT_REGION=global.
+  mcp_style_readability:
+    model_config: datasets/model_configs/gemini_2.5_pro_model.yaml
+    style_guide: datasets/mcp_readability/style_guide.md
+
 # Inputs (paths relative to the repo root / run cwd).
 endpoints_config: datasets/mcp_readability/endpoints.yaml
 exceptions_config: datasets/mcp_readability/exceptions.yaml          # optional
 tools_generator_config: datasets/mcp_readability/tools_generator.yaml
 
-# Token budget for token_budget_used_percent (endpoints may override per-entry).
-token_budget: 200000
-
 # Optional filter: only check endpoints whose endpoint_type is in this list.
 # Empty = check all. e.g. [PROD]
 endpoint_types: []
 
-# LLM judge: consumes a model config and the style-guide path. Reuses the
-# shared repo model config.
-readability_judge:
-  model_config: datasets/model_configs/gemini_2.5_pro_model.yaml
-  style_guide: datasets/mcp_readability/style_guide.md
-
 # Per-endpoint check concurrency.
 runners:
   endpoint_runners: 4

evalbench/dataset/dataset.py

@@ -242,6 +242,17 @@ def load_json(json_file_path):
 
 
 def load_dataset_from_json(json_file_path, config):
+    # No dataset path (e.g. orchestrators driven by their run config rather than a
+    # prompt dataset): nothing to load. flatten_dataset({}) yields []. Log it so
+    # a run that *did* expect a dataset (missing/misconfigured dataset_config)
+    # doesn't fail silently.
+    if not json_file_path:
+        logging.info(
+            "load_dataset_from_json: no dataset path provided; returning an "
+            "empty dataset. Expected only for orchestrators that drive their "
+            "own inputs from the run config."
+        )
+        return {}
     input_items = {}
     dataset_format = config.get("dataset_format", "evalbench-standard-format")
     if dataset_format == "bird-interact-format":

evalbench/eval_service.py

@@ -126,7 +126,9 @@ async def ListEvalInputs(
         session = SESSIONMANAGER.get_session(rpc_id_var.get())
         logging.info("Retrieving Evals for: %s.", rpc_id_var.get())
         experiment_config = session["config"]
-        dataset_config_json = experiment_config["dataset_config"]
+        # dataset_config is optional: some orchestrators drive their work from
+        # the run config itself, in which case load_dataset_from_json returns {}.
+        dataset_config_json = experiment_config.get("dataset_config")
         dataset = load_dataset_from_json(
             dataset_config_json, experiment_config)
         for _, eval_inputs in dataset.items():
@@ -316,8 +318,9 @@ async def Interact(
             context.set_details(error_msg)
             return
 
-        # Load dataset and instantiate the Orchestrator
-        dataset_config_json = config["dataset_config"]
+        # Load dataset and instantiate the Orchestrator. dataset_config is
+        # optional (datasetless orchestrators drive from the run config).
+        dataset_config_json = config.get("dataset_config")
         dataset_dict = load_dataset_from_json(dataset_config_json, config)
 
         dataset = []

evalbench/evaluator/__init__.py

@@ -8,6 +8,7 @@
 from evaluator.dataengineeringagentorchestrator import (
     DataEngineeringAgentOrchestrator,
 )
+from evaluator.mcp_readability import McpReadabilityOrchestrator
 import logging
 
 
@@ -27,6 +28,10 @@ def get_orchestrator(config, db_configs, setup_config, report_progress=False):
         return DataEngineeringAgentOrchestrator(
             config, db_configs, setup_config, report_progress
         )
+    elif orchestrator_type == "mcp_readability":
+        return McpReadabilityOrchestrator(
+            config, db_configs, setup_config, report_progress
+        )
     else:
         return Orchestrator(config, db_configs, setup_config, report_progress)
 

evalbench/evaluator/mcp_readability/__init__.py

@@ -0,0 +1,3 @@
+from evaluator.mcp_readability.orchestrator import McpReadabilityOrchestrator
+
+__all__ = ["McpReadabilityOrchestrator"]

evalbench/evaluator/mcp_readability/exceptions.py

@@ -0,0 +1,71 @@
+"""Loading and matching of per-endpoint style-rule exceptions (waivers).
+
+An exceptions file lets an operator waive a specific style requirement for a
+specific endpoint when it legitimately cannot comply. Waived rules are passed to
+the scorer so they are excluded from the P0/P1/P2 counts and surfaced separately
+in the feedback.
+
+File schema (see ``datasets/mcp_readability/exceptions.yaml``)::
+
+    exceptions:
+      - product_name: "Cloud SQL"     # any of product_name / endpoint_type
+        rule_id: "tool-names"
+        reason: "..."
+      - endpoint_type: AUTOPUSH       # "*" or omitted field = match-all
+        rule_id: "use-enums"
+        reason: "..."
+"""
+
+import logging
+from util.config import load_yaml_config
+
+
+def load_exceptions(path: str) -> list[dict]:
+    """Load the exceptions list from a YAML file. Missing/empty -> []."""
+    if not path:
+        return []
+    parsed = load_yaml_config(path)
+    if not parsed:
+        return []
+    exceptions = parsed.get("exceptions") or []
+    if not isinstance(exceptions, list):
+        logging.warning(
+            "mcp_readability: 'exceptions' in %s is not a list; ignoring.", path
+        )
+        return []
+    return exceptions
+
+
+def _matches(field_value, exception_value) -> bool:
+    """A matcher field matches when it is absent, '*', or equal (case-insensitive)."""
+    if exception_value is None or exception_value == "*":
+        return True
+    if field_value is None:
+        return False
+    return str(field_value).strip().lower() == str(exception_value).strip().lower()
+
+
+def applicable_exceptions(endpoint: dict, all_exceptions: list[dict]) -> list[dict]:
+    """Return exceptions whose matchers all apply to ``endpoint``.
+
+    Matchers considered: ``product_name`` and ``endpoint_type`` (the endpoint's
+    identity in #469's ``endpoints.yaml``). Each exception keeps its ``rule_id``
+    and ``reason`` for the scorer prompt.
+    """
+    matched = []
+    for exc in all_exceptions:
+        if not isinstance(exc, dict):
+            continue
+        if not exc.get("rule_id"):
+            continue
+        if (
+            _matches(endpoint.get("product_name"), exc.get("product_name"))
+            and _matches(endpoint.get("endpoint_type"), exc.get("endpoint_type"))
+        ):
+            matched.append(
+                {
+                    "rule_id": exc.get("rule_id"),
+                    "reason": exc.get("reason", ""),
+                }
+            )
+    return matched