fix(ci): skip C4 deepeval tests when dep missing, drop pr_drafts links from docs nav, run pre-commit auto-fixes

Author: javierdejesusda

.pre-commit-config.yaml

@@ -22,7 +22,7 @@ repos:
     hooks:
       - id: codespell
         args:
-          - "--ignore-words-list=iff,nd,te,hist,ot,ist,ment,worl,bu,som,covert,strat,intoto,unparseable,re-use,socio-economic,rouge"
+          - "--ignore-words-list=iff,nd,te,hist,ot,ist,ment,worl,bu,som,covert,strat,intoto,unparseable,re-use,socio-economic,rouge,unx,ord,cov"
         exclude: |
           (?x)^(
             CHANGELOG\.md|

benchmarks/paper/experiments/01_controlled_noise.py

@@ -192,18 +192,11 @@ def run_experiment(
                         user_instruction=task.user_instruction,
                         tools=task.tools,
                     )
-                    _, subs, passed = _score_trajectory(
-                        predicted, task.reference_actions
-                    )
+                    _, subs, passed = _score_trajectory(predicted, task.reference_actions)
                     traj_path = (
-                        output_root
-                        / domain
-                        / noise.name
-                        / f"seed{seed}_{task.task_id}.jsonl"
-                    )
-                    _write_trajectory_jsonl(
-                        traj_path, predicted, task.reference_actions
+                        output_root / domain / noise.name / f"seed{seed}_{task.task_id}.jsonl"
                     )
+                    _write_trajectory_jsonl(traj_path, predicted, task.reference_actions)
                     timestamp = datetime.now(timezone.utc).isoformat()
 
                     manifest_row: dict[str, Any] = {
@@ -219,9 +212,7 @@ def run_experiment(
                         "passed": bool(passed),
                         "reference_action_count": len(task.reference_actions),
                         "predicted_action_count": len(predicted),
-                        "trajectory_path": str(
-                            traj_path.relative_to(output_root)
-                        ),
+                        "trajectory_path": str(traj_path.relative_to(output_root)),
                         "timestamp_utc": timestamp,
                         "model_version_sha": None,
                         "benchmark_sha": None,
@@ -254,25 +245,19 @@ def run_experiment(
             fh.write(json.dumps(row) + "\n")
 
     summary = _build_summary(jsonl_rows)
-    summary_path.write_text(
-        json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8"
-    )
+    summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
 
     manifest = {
         "schema": "https://checkllm.dev/schemas/paper_manifest/v1",
         "experiment_id": EXPERIMENT_ID,
         "generated_at_utc": datetime.now(timezone.utc).isoformat(),
-        "noise_levels": [
-            {"name": n.name, **n.as_params()} for n in NOISE_LEVELS
-        ],
+        "noise_levels": [{"name": n.name, **n.as_params()} for n in NOISE_LEVELS],
         "seeds": list(SEEDS),
         "domains": list(DOMAINS),
         "limit_tasks": limit_tasks,
         "rows": rows,
     }
-    manifest_path.write_text(
-        json.dumps(manifest, indent=2), encoding="utf-8"
-    )
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
 
     return {
         "total_rows": len(rows),
@@ -302,35 +287,23 @@ def _build_summary(jsonl_rows: list[dict[str, Any]]) -> dict[str, Any]:
         domain_noise_means[domain] = {}
         for noise in NOISE_LEVELS:
             bucket = [
-                r
-                for r in jsonl_rows
-                if r["domain"] == domain and r["noise_level"] == noise.name
+                r for r in jsonl_rows if r["domain"] == domain and r["noise_level"] == noise.name
             ]
             stats: dict[str, dict[str, float | int]] = {}
             for key in sub_keys:
                 values = [float(r[key]) for r in bucket]
                 stats[key] = {
                     "mean": float(statistics.fmean(values)) if values else 0.0,
-                    "stdev": (
-                        float(statistics.pstdev(values))
-                        if len(values) > 1
-                        else 0.0
-                    ),
+                    "stdev": (float(statistics.pstdev(values)) if len(values) > 1 else 0.0),
                     "n": len(values),
                 }
             domain_noise_stats[domain][noise.name] = stats
-            domain_noise_means[domain][noise.name] = float(
-                stats["overall"]["mean"]
-            )
+            domain_noise_means[domain][noise.name] = float(stats["overall"]["mean"])
 
     monotonicity_check: dict[str, bool] = {}
     for domain in DOMAINS:
-        ordered_means = [
-            domain_noise_means[domain][n.name] for n in NOISE_LEVELS
-        ]
-        monotonicity_check[domain] = _is_monotonic_non_increasing(
-            ordered_means
-        )
+        ordered_means = [domain_noise_means[domain][n.name] for n in NOISE_LEVELS]
+        monotonicity_check[domain] = _is_monotonic_non_increasing(ordered_means)
 
     return {
         "experiment_id": EXPERIMENT_ID,
@@ -347,8 +320,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     """Build the argparse namespace for the CLI entry point."""
     parser = argparse.ArgumentParser(
         description=(
-            "Controlled-noise validation experiment for TrajectoryMetric "
-            "(Task C1-zero)."
+            "Controlled-noise validation experiment for TrajectoryMetric " "(Task C1-zero)."
         )
     )
     parser.add_argument(
@@ -376,9 +348,7 @@ def main(argv: list[str] | None = None) -> int:
         Exit code; ``0`` on success.
     """
     args = _parse_args(argv)
-    summary = run_experiment(
-        output_dir=args.output_dir, limit_tasks=args.limit_tasks
-    )
+    summary = run_experiment(output_dir=args.output_dir, limit_tasks=args.limit_tasks)
     print(json.dumps(summary, indent=2))
     return 0
 

benchmarks/paper/experiments/02_metric_vs_truth.py

@@ -140,9 +140,7 @@ def _score_trajectory(
     return metric.compute_subscores(predicted_names).as_dict()
 
 
-def _auroc_mann_whitney(
-    scores: np.ndarray, labels: np.ndarray
-) -> float:
+def _auroc_mann_whitney(scores: np.ndarray, labels: np.ndarray) -> float:
     """Compute AUROC via the Mann-Whitney U statistic on average ranks.
 
     Equivalent to :func:`sklearn.metrics.roc_auc_score` for binary
@@ -232,9 +230,7 @@ def _bootstrap_ci(
         resample produced a defined statistic.
     """
 
-    def _vectorized_statistic(
-        sample_scores: np.ndarray, sample_labels: np.ndarray
-    ) -> float:
+    def _vectorized_statistic(sample_scores: np.ndarray, sample_labels: np.ndarray) -> float:
         # scipy.stats.bootstrap calls the statistic on each resample;
         # vectorize=False (the default for non-vectorised callables) is
         # set in the bootstrap call so each resample arrives as 1-D.
@@ -297,9 +293,7 @@ def _statistics_block(
         ("auroc", _auroc_mann_whitney),
     ):
         value = fn(scores, labels)
-        ci_lower, ci_upper = _bootstrap_ci(
-            scores, labels, fn, n_bootstrap=n_bootstrap, rng=rng
-        )
+        ci_lower, ci_upper = _bootstrap_ci(scores, labels, fn, n_bootstrap=n_bootstrap, rng=rng)
         block[name] = {
             "value": float(value),
             "ci_lower": float(ci_lower),
@@ -410,26 +404,20 @@ def run_experiment(
             fh.write(json.dumps(row) + "\n")
 
     summary = _build_summary(score_rows, n_bootstrap=n_bootstrap)
-    summary_path.write_text(
-        json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8"
-    )
+    summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
 
     manifest = {
         "schema": "https://checkllm.dev/schemas/paper_manifest/v1",
         "experiment_id": EXPERIMENT_ID,
         "generated_at_utc": datetime.now(timezone.utc).isoformat(),
-        "noise_levels": [
-            {"name": n.name, **n.as_params()} for n in NOISE_LEVELS
-        ],
+        "noise_levels": [{"name": n.name, **n.as_params()} for n in NOISE_LEVELS],
         "seeds": list(SEEDS),
         "domains": list(DOMAINS),
         "limit_tasks": limit_tasks,
         "n_bootstrap": n_bootstrap,
         "rows": manifest_rows,
     }
-    manifest_path.write_text(
-        json.dumps(manifest, indent=2), encoding="utf-8"
-    )
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
 
     return {
         "n_trajectories": len(manifest_rows),
@@ -439,9 +427,7 @@ def run_experiment(
     }
 
 
-def _build_summary(
-    score_rows: list[dict[str, Any]], n_bootstrap: int
-) -> dict[str, Any]:
+def _build_summary(score_rows: list[dict[str, Any]], n_bootstrap: int) -> dict[str, Any]:
     """Aggregate per-trajectory rows into the paper-summary statistics.
 
     Args:
@@ -458,26 +444,20 @@ def _build_summary(
     labels = np.asarray([r["label"] for r in score_rows], dtype=int)
     overall_scores = np.asarray([r["score"] for r in score_rows], dtype=float)
 
-    overall_block = _statistics_block(
-        overall_scores, labels, n_bootstrap=n_bootstrap, rng=rng
-    )
+    overall_block = _statistics_block(overall_scores, labels, n_bootstrap=n_bootstrap, rng=rng)
 
     per_domain: dict[str, dict[str, dict[str, float]]] = {}
     auroc_gate_passed: dict[str, bool] = {}
     for domain in DOMAINS:
         bucket = [r for r in score_rows if r["domain"] == domain]
         d_scores = np.asarray([r["score"] for r in bucket], dtype=float)
         d_labels = np.asarray([r["label"] for r in bucket], dtype=int)
-        block = _statistics_block(
-            d_scores, d_labels, n_bootstrap=n_bootstrap, rng=rng
-        )
+        block = _statistics_block(d_scores, d_labels, n_bootstrap=n_bootstrap, rng=rng)
         per_domain[domain] = block
         ci_lower = block["auroc"]["ci_lower"]
         # Gate passes when the 95% CI lower bound is at or above 0.70.
         # NaN (undefined AUROC) fails the gate.
-        auroc_gate_passed[domain] = bool(
-            not np.isnan(ci_lower) and ci_lower >= AUROC_GATE_LOWER
-        )
+        auroc_gate_passed[domain] = bool(not np.isnan(ci_lower) and ci_lower >= AUROC_GATE_LOWER)
 
     per_metric: dict[str, dict[str, dict[str, float]]] = {}
     for sub_metric in SUB_METRIC_KEYS:
@@ -507,9 +487,7 @@ def _build_summary(
 def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     """Build the argparse namespace for the CLI entry point."""
     parser = argparse.ArgumentParser(
-        description=(
-            "Metric-vs-synthetic-truth correlation study (Task C2-zero)."
-        )
+        description=("Metric-vs-synthetic-truth correlation study (Task C2-zero).")
     )
     parser.add_argument(
         "--output-dir",
@@ -527,10 +505,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
         "--n-bootstrap",
         type=int,
         default=DEFAULT_N_BOOTSTRAP,
-        help=(
-            "Number of percentile bootstrap resamples for 95%% CIs "
-            "(default: 1000)."
-        ),
+        help=("Number of percentile bootstrap resamples for 95%% CIs " "(default: 1000)."),
     )
     return parser.parse_args(argv)
 

benchmarks/paper/experiments/03_trajectory_ablation.py

@@ -204,12 +204,9 @@ def _build_trajectory_pool(
                             "task_id": task.task_id,
                             "seed": seed,
                             "noise_level": noise.name,
-                            "predicted_names": [
-                                str(a.get("name", "")) for a in predicted
-                            ],
+                            "predicted_names": [str(a.get("name", "")) for a in predicted],
                             "reference_names": [
-                                str(a.get("name", ""))
-                                for a in task.reference_actions
+                                str(a.get("name", "")) for a in task.reference_actions
                             ],
                             "label": 1 if noise.name == "clean" else 0,
                         }
@@ -342,9 +339,7 @@ def _build_heatmap(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
                 vals = [
                     r["spearman_rho"]
                     for r in rows
-                    if r[key_a] == va
-                    and r[key_b] == vb
-                    and not np.isnan(r["spearman_rho"])
+                    if r[key_a] == va and r[key_b] == vb and not np.isnan(r["spearman_rho"])
                 ]
                 row.append(float(np.mean(vals)) if vals else None)
             grid.append(row)
@@ -386,9 +381,7 @@ def _weight_correlations(rows: list[dict[str, Any]]) -> dict[str, dict[str, floa
     return out
 
 
-def _is_pareto_optimal(
-    rows: list[dict[str, Any]], default_row: dict[str, Any]
-) -> bool:
+def _is_pareto_optimal(rows: list[dict[str, Any]], default_row: dict[str, Any]) -> bool:
     """True iff no other cell strictly dominates the default on both metrics.
 
     A cell strictly dominates iff its Spearman rho and its AUROC are
@@ -410,10 +403,7 @@ def _is_pareto_optimal(
             continue
         if np.isnan(r["spearman_rho"]) or np.isnan(r["auroc"]):
             continue
-        if (
-            r["spearman_rho"] > default_rho
-            and r["auroc"] > default_auc
-        ):
+        if r["spearman_rho"] > default_rho and r["auroc"] > default_auc:
             return False
     return True
 
@@ -448,9 +438,7 @@ def run_experiment(
     pool = _build_trajectory_pool(seeds, noise_levels, domains, limit_tasks)
 
     all_cells = list(_iter_grid_cells())
-    n_skipped_degenerate = sum(
-        1 for cell in all_cells if _is_degenerate(cell[:4])
-    )
+    n_skipped_degenerate = sum(1 for cell in all_cells if _is_degenerate(cell[:4]))
     valid_cells = [cell for cell in all_cells if not _is_degenerate(cell[:4])]
 
     if limit_cells is not None and limit_cells < len(valid_cells):
@@ -558,14 +546,10 @@ def run_experiment(
         "weight_correlations": _weight_correlations(rows),
         "wall_seconds": float(wall_seconds),
     }
-    summary_path.write_text(
-        json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8"
-    )
+    summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
 
     heatmap = _build_heatmap(rows)
-    heatmap_path.write_text(
-        json.dumps(heatmap, indent=2, sort_keys=True), encoding="utf-8"
-    )
+    heatmap_path.write_text(json.dumps(heatmap, indent=2, sort_keys=True), encoding="utf-8")
 
     return {
         "n_cells": len(rows),