feat(evals): eval architecture + probe wave 1 — first BEHAVIORAL skill verdicts

cli/internal/adapters/eval/runtime.go

@@ -161,7 +161,7 @@ func (runtime Runtime) RunStats(args []string) ([]byte, error) {
 		return nil, fmt.Errorf("eval suite: substrate venv not found; provision via `python3 -m venv ~/.agents/evals/.venv && pip install numpy scipy`")
 	}
 	command := exec.Command(python, args...)
-	command.Env = append(os.Environ(), "PYTHONPATH="+runtime.Root())
+	command.Env = append(os.Environ(), "PYTHONPATH="+runtime.statsPythonPath())
 	output, err := command.Output()
 	if err != nil {
 		stderr := ""
@@ -174,6 +174,30 @@ func (runtime Runtime) RunStats(args []string) ([]byte, error) {
 	return output, nil
 }
 
+// statsPythonPath prefers the repo-vendored stats package (evals/_stats,
+// found by a bounded walk up from the working directory) over the legacy
+// home-directory copy at Root(). The vendored copy is first on PYTHONPATH so
+// the statistics implementation ships with the repo instead of depending on
+// uncommitted state in ~/.agents/evals.
+func (runtime Runtime) statsPythonPath() string {
+	paths := []string{}
+	if dir, err := os.Getwd(); err == nil {
+		for range [8]struct{}{} {
+			if _, statErr := os.Stat(filepath.Join(dir, "evals", "_stats", "cli.py")); statErr == nil {
+				paths = append(paths, filepath.Join(dir, "evals"))
+				break
+			}
+			parent := filepath.Dir(dir)
+			if parent == dir {
+				break
+			}
+			dir = parent
+		}
+	}
+	paths = append(paths, runtime.Root())
+	return strings.Join(paths, string(os.PathListSeparator))
+}
+
 func (runtime Runtime) pythonBinary() string {
 	if override := strings.TrimSpace(os.Getenv("AGENTOPS_EVALS_VENV")); override != "" {
 		return override

cli/internal/gates/checks/seed.go

@@ -294,7 +294,7 @@ func init() {
 		// (Blocking:false, warn never fail) exactly like skill.isolation and the
 		// egwt gates: the spine is probed first, the ratchet drives the rest, and
 		// the Blocking:false->true flip is made deliberately once covered. age-e508.1.
-		{ID: "skill.probe-coverage", Tiers: gates.Fast | gates.Full, Match: skillProbePaths, Blocking: false, Backing: "check-skill-probe-coverage.sh", RepairHint: "bash scripts/probe-skill.sh --probe <skill> then record it in the MEASURED ledger of skills/SKILL-TIERS.md; advisory — probe the spine, ratchet the rest"},
+		{ID: "skill.probe-coverage", Tiers: gates.Fast | gates.Full, Match: skillProbePaths, Blocking: false, Backing: "check-skill-probe-coverage.sh", RepairHint: "bash scripts/probe-skill.sh --probe <skill> then record it in the MEASURED ledger at evals/skill-probes/LEDGER.md (hand-maintained; never inside generated SKILL-TIERS.md); advisory — probe the spine, ratchet the rest"},
 		{ID: "skill.no-operator-leakage", Tiers: gates.Fast | gates.Full, Match: operatorLeakPaths, Blocking: true, Backing: "check-no-operator-skills.sh"},
 		{ID: "skill.heal-strict", Tiers: gates.Full, Match: skillPaths, Blocking: true, Backing: "skills/skill-builder/scripts/heal.sh", Args: []string{"--check", "--strict"}},
 		{ID: "skill.frontmatter-v2", Tiers: gates.Full, Match: skillPaths, Blocking: true, Backing: "validate-skill-frontmatter.sh"},

docs/evals/2026-08-04-probe-wave-1.md

@@ -0,0 +1,148 @@
+# Skill behavioral probe wave 1 — 2026-08-04
+
+> **HONESTY.** Probes measure **BEHAVIOR-CHANGE, not quality-uplift**: did loading the
+> skill's guidance change what the agent DID? N=2 per arm per config is
+> **directional, not statistical** (ADR-0011 — do not overclaim). Each arm differs
+> only by `treatment-prelude.md`; discriminators are deterministic and pass a
+> planted-reference selftest before any live scoring (known-good → PRESENT,
+> known-bad → ABSENT, empty → DEGRADED). Probes measure **injected-prelude
+> efficacy** — they do not measure whether a real session loads the SKILL.md file
+> (the router lane; wave-2 scope).
+
+## What ran
+
+5 new probes + 1 re-run, each at TWO producer configs (`gpt-5.6-luna` at
+`model_reasoning_effort` xhigh and low), 2 reps per arm per config — 48 live
+`codex exec` runs total. New harness capabilities this wave: `--effort` flag
+(the second weak-producer ratchet), `producer` provenance in every scorecard,
+discriminator selftests, and per-config fixture snapshots
+(`fixtures-xhigh-2026-08-04/`) so re-runs never destroy prior evidence.
+
+Reproduce any cell from committed fixtures:
+
+```bash
+bash scripts/probe-skill.sh --probe premortem-self-validation --replay
+```
+
+## Results
+
+| Probe | Skill (tier) | xhigh C→T | low C→T | Verdict |
+|---|---|---|---|---|
+| `premortem-self-validation` | premortem (judgment) | 0.5 → 1.0 | 0.0 → 1.0 | **BEHAVIORAL** |
+| `standards-go-conventions` | standards (knowledge) | 0.5 → 1.0 | 0.0 → 1.0 | **BEHAVIORAL** |
+| `validate-not-proven` | validate (judgment) | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) |
+| `security-coverage-gap` | security (product) | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) |
+| `reality-check-gap` | reality-check (judgment) | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) |
+| `crank-luna` | crank (execution) | 1.0 → 1.0 | 1.0 → 1.0 | INERT (3rd config) |
+
+## Findings
+
+1. **First measured skill effects in this repo.** Two skills produced clean
+   behavioral separation, and both show the **gradient the SOTA literature
+   predicts: the effect grows as the producer weakens.** At xhigh the control
+   arm gets it right half the time; at low effort the control NEVER does and
+   the treatment ALWAYS does (0.0 → 1.0, both probes, 2/2 reps). This is the
+   product story in miniature — the harness's guidance matters most exactly
+   where the config is cheapest.
+   - premortem: naming the self-validation closure flaw (implementer closes on
+     its own tests) in a premortem over a plan that plants it.
+   - standards: producing `fmt.Errorf(...%w...)` wrapping + a table-driven
+     test without being asked for either.
+2. **The ceiling class is scenario-difficulty, not effort.** validate,
+   security, and reality-check stayed saturated at BOTH efforts — even a
+   low-effort luna answers NOT_PROVEN / GAPPED / names-the-gap unaided. The
+   planted flaws are too obvious. The instrument needs harder scenarios
+   (subtler flaws, signal buried in longer context — the context-rot
+   direction), not weaker producers. **Do not read these INERTs as "the skill
+   is worthless"; read them as "this scenario cannot measure it."**
+3. **crank is INERT across three configs** (gpt-5.5 xhigh 2026-07-08; luna
+   xhigh; luna low): write-scope-collision separation appears native to
+   current frontier models at every tested effort. This is the strongest
+   cull-or-reshape signal in the ledger — either the skill's value lives in
+   behaviors no probe yet measures, or the wave-planning core has been
+   absorbed by the models.
+
+## Skill changes made from these measurements
+
+The treatment preludes that produced the separation were **hoisted into the
+shipped skills** as front-loaded, imperative, load-bearing blocks (marked
+MEASURED with probe citations):
+
+- `skills/premortem/SKILL.md` — new first-check section: evidence-shape /
+  who-verifies-and-are-they-fresh, before any technical risk.
+- `skills/standards/SKILL.md` — new load-bearing-conventions section:
+  inline-imperative Go core (the probe proved inline works where the graphify
+  probe proved behind-the-link does not).
+
+Codex twins regenerated (`codex-sync`), frontmatter + heal checks green.
+Honest limit: the probes prove the **prelude content** changes behavior when
+injected; whether the edited SKILL.md files get loaded and obeyed in real
+sessions is the router question — measured next by the telemetry hook
+(the opt-in `scripts/hooks/skill-telemetry.sh`) plus a routing probe batch.
+
+No changes were made to validate / security / reality-check from this wave:
+their probes lack headroom, so any edit would be uninstrumented guessing.
+
+## Infrastructure shipped this wave
+
+- `evals/skill-probes/LEDGER.md` — MEASURED ledger moved out of generated
+  SKILL-TIERS.md (root-cause fix for the regen wipe); gate + README + seed.go
+  hint repointed; coverage now 4/11 product-judgment skills measured.
+- `evals/_stats/` — paired-bootstrap statistics package vendored in-repo
+  (42 tests green); `RunStats` prefers the vendored copy.
+- `scripts/hooks/skill-telemetry.sh` (opt-in, never wired by default) — Skill invocations logged to
+  `.agents/ao/skill-telemetry.jsonl`. Wiring is per-machine opt-in via
+  gitignored `.claude/settings.json` (PostToolUse matcher `Skill` -> the
+  script); never shipped to plugin users, never forced on contributors.
+- `scripts/probe-skill.sh` — `--effort` ratchet, producer provenance in
+  scorecards.
+- 6 new probe packages under `evals/skill-probes/` with deterministic
+  selftested discriminators and dual-config fixtures.
+
+## Wave 2a — hardened ceiling scenarios (2026-08-05)
+
+The three ceiling probes were re-authored as v2 with the flaw buried the way
+it hides in real life: validate's `not_checked` euphemized as "low-risk
+housekeeping" inside a green-heavy completion report under release-train
+pressure; security's semgrep failure softened to a mid-log WARN whose next
+line reads "0 findings"; reality-check's gap derivable only by arithmetic
+across separate outputs (19 files − 2 helpers = 17 claimed; grep count 14)
+against two narrative confirmations. Same selftested discriminators.
+
+| Probe | xhigh C→T | low C→T | Verdict |
+|---|---|---|---|
+| `validate-not-proven-v2` | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) |
+| `security-coverage-gap-v2` | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) |
+| `reality-check-gap-v2` | 1.0 → 1.0 | n=2: 0.5 → 0.5; **n=6: 1.0 → 1.0** | INERT (ceiling) |
+
+Two findings:
+
+1. **A sequential-stopping object lesson.** reality-check-v2's first n=2 batch
+   showed apparent headroom (control 0.5) with an apparent null skill effect
+   (treatment 0.5). Extending to n=6 per the architecture's D4 rule resolved
+   both as sampling noise: 6/6 PRESENT in both arms. An n=2 conclusion here
+   would have been wrong twice over — this is exactly why probe scorecards
+   carry the "directional, not statistical" honesty header and why ambiguous
+   batches extend before concluding.
+2. **The quiz format is the ceiling, not the scenario dressing.** Two rounds
+   of hardening failed to open headroom: an isolated question with an explicit
+   output menu (VERDICT:/STATUS:/RESULT:) telegraphs "this is a test," and
+   frontier models at every effort apply the doctrine flawlessly in that
+   frame. Real false-PASSes happen mid-task, in long agentic flows, with no
+   menu — the repo's own ~3% live refute rate proves the failure exists in
+   production shape. Conclusion recorded: **these three doctrines are
+   quiz-robust on current frontier models; further measurement of these
+   skills moves to task-embedded Tier-2 fixtures where the doctrine must fire
+   unprompted mid-work.** No more quiz-hardening rounds.
+
+## Next wave queue
+
+1. ~~Harden the three ceiling scenarios and re-measure~~ — done (wave 2a);
+   outcome above: quiz-robust, move to Tier-2 task-embedded form.
+2. Routing probe batch: does a real session load the right skill unprompted
+   (P(loaded | applicable)) — the multiplier on every number above.
+3. Tier-2 outcome ablation for premortem on a task corpus (the Stage-A screen
+   of docs/architecture/eval-architecture.md) — now also carrying the
+   validate/security/reality-check doctrines as task-embedded checks.
+4. crank disposition decision (Bo): cull, reshape toward unmeasured behaviors,
+   or keep as documentation.

docs/evals/scorecards/2026-08-04/crank-luna-low.json

@@ -0,0 +1,14 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "crank-luna",
+  "skill": "crank",
+  "mode": "live",
+  "generated_at": "2026-08-05T00:02:50Z",
+  "reps": 2,
+  "producer": {"model": "gpt-5.6-luna", "effort": "low"},
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 2, "usable": 2, "rate": 1.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"PRESENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/crank-luna-xhigh.json

@@ -0,0 +1,13 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "crank-luna",
+  "skill": "crank",
+  "mode": "live",
+  "generated_at": "2026-08-04T23:58:09Z",
+  "reps": 2,
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 2, "usable": 2, "rate": 1.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"PRESENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/premortem-self-validation-low.json

@@ -0,0 +1,14 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "premortem-self-validation",
+  "skill": "premortem",
+  "mode": "live",
+  "generated_at": "2026-08-05T00:02:05Z",
+  "reps": 2,
+  "producer": {"model": "gpt-5.6-luna", "effort": "low"},
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 0, "usable": 2, "rate": 0.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "BEHAVIORAL",
+  "per_rep": [{"rep":1,"control":"ABSENT","treatment":"PRESENT"},{"rep":2,"control":"ABSENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/premortem-self-validation-xhigh.json

@@ -0,0 +1,13 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "premortem-self-validation",
+  "skill": "premortem",
+  "mode": "live",
+  "generated_at": "2026-08-04T23:57:23Z",
+  "reps": 2,
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 1, "usable": 2, "rate": 0.5},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "BEHAVIORAL",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"ABSENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/reality-check-gap-low.json

@@ -0,0 +1,14 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "reality-check-gap",
+  "skill": "reality-check",
+  "mode": "live",
+  "generated_at": "2026-08-05T00:02:21Z",
+  "reps": 2,
+  "producer": {"model": "gpt-5.6-luna", "effort": "low"},
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 2, "usable": 2, "rate": 1.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"PRESENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/reality-check-gap-xhigh.json

@@ -0,0 +1,13 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "reality-check-gap",
+  "skill": "reality-check",
+  "mode": "live",
+  "generated_at": "2026-08-04T23:57:46Z",
+  "reps": 2,
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 2, "usable": 2, "rate": 1.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"PRESENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/security-coverage-gap-low.json

@@ -0,0 +1,14 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "security-coverage-gap",
+  "skill": "security",
+  "mode": "live",
+  "generated_at": "2026-08-05T00:02:02Z",
+  "reps": 2,
+  "producer": {"model": "gpt-5.6-luna", "effort": "low"},
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 2, "usable": 2, "rate": 1.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"PRESENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/security-coverage-gap-xhigh.json

@@ -0,0 +1,13 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "security-coverage-gap",
+  "skill": "security",
+  "mode": "live",
+  "generated_at": "2026-08-04T23:56:35Z",
+  "reps": 2,
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 2, "usable": 2, "rate": 1.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"PRESENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/standards-go-conventions-low.json

@@ -0,0 +1,14 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "standards-go-conventions",
+  "skill": "standards",
+  "mode": "live",
+  "generated_at": "2026-08-05T00:03:16Z",
+  "reps": 2,
+  "producer": {"model": "gpt-5.6-luna", "effort": "low"},
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 0, "usable": 2, "rate": 0.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "BEHAVIORAL",
+  "per_rep": [{"rep":1,"control":"ABSENT","treatment":"PRESENT"},{"rep":2,"control":"ABSENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/standards-go-conventions-xhigh.json

@@ -0,0 +1,13 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "standards-go-conventions",
+  "skill": "standards",
+  "mode": "live",
+  "generated_at": "2026-08-05T00:00:00Z",
+  "reps": 2,
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 1, "usable": 2, "rate": 0.5},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "BEHAVIORAL",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"ABSENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/validate-not-proven-low.json

@@ -0,0 +1,14 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "validate-not-proven",
+  "skill": "validate",
+  "mode": "live",
+  "generated_at": "2026-08-05T00:01:40Z",
+  "reps": 2,
+  "producer": {"model": "gpt-5.6-luna", "effort": "low"},
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 2, "usable": 2, "rate": 1.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"PRESENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-04/validate-not-proven-xhigh.json

@@ -0,0 +1,13 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "validate-not-proven",
+  "skill": "validate",
+  "mode": "live",
+  "generated_at": "2026-08-04T23:56:15Z",
+  "reps": 2,
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 2, "usable": 2, "rate": 1.0},
+  "treatment": {"present": 2, "usable": 2, "rate": 1.0},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"PRESENT"},{"rep":2,"control":"PRESENT","treatment":"PRESENT"}]
+}

docs/evals/scorecards/2026-08-05/reality-check-gap-v2-low.json

@@ -0,0 +1,14 @@
+{
+  "schema": "agentops-skill-probe.v1",
+  "probe": "reality-check-gap-v2",
+  "skill": "reality-check",
+  "mode": "live",
+  "generated_at": "2026-08-05T00:51:09Z",
+  "reps": 2,
+  "producer": {"model": "gpt-5.6-luna", "effort": "low"},
+  "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)",
+  "control": {"present": 1, "usable": 2, "rate": 0.5},
+  "treatment": {"present": 1, "usable": 2, "rate": 0.5},
+  "verdict": "INERT",
+  "per_rep": [{"rep":1,"control":"PRESENT","treatment":"ABSENT"},{"rep":2,"control":"ABSENT","treatment":"PRESENT"}]
+}