diff --git a/AGENTS.md b/AGENTS.md index 118ff469e..7eb34bf19 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -133,6 +133,12 @@ AgentOps work ownership. Inspect the final subject, map acceptance to evidence, disclose `checked` and `not_checked`, and obtain one fresh validation result over the exact content. -Include a verdict reference only when persistence was requested. Report residual -risk plainly. Git status, pushing, merging, release, and rollback are handled -by the caller's repository policy, outside semantic completion. +`not_checked` names in-scope acceptance surface that went unverified, so a PASS +has none by construction and any entry makes the result `NOT_PROVEN`. Scope +limits are disclosed, never deleted: a bounded proof of a criterion belongs in +that criterion's `reason`, a declared non-goal belongs to the intent source +(optionally restated as an evidence-backed boundary criterion), and residual +risk belongs in this report. Include a verdict reference only when persistence +was requested. Report residual risk plainly. Git status, pushing, merging, +release, and rollback are handled by the caller's repository policy, outside +semantic completion. diff --git a/docs/architecture/operating-loop.md b/docs/architecture/operating-loop.md index 180c30e06..e769b30b5 100644 --- a/docs/architecture/operating-loop.md +++ b/docs/architecture/operating-loop.md @@ -82,6 +82,16 @@ the evidence, and judges every acceptance criterion. - Complete evidence satisfying every criterion, with nonempty checked scope and evidence references: `PASS`. +`not_checked` names in-scope acceptance surface that this validation did not +verify. PASS asserts that the whole declared acceptance surface was verified, +so a PASS carries no `not_checked` entries and any entry makes the result +`NOT_PROVEN`. That strictness never rewards deleting an honest caveat, because +each kind of scope limit has a home that survives inside a PASS: a bounded +proof of a criterion goes in `criteria[].reason`, a declared non-goal stays in +the intent source (optionally restated as an evidence-backed boundary +criterion), and residual risk goes in the caller-facing report. The full table +lives in `skills/validate/SKILL.md` under Scope disclosure. + The validation result records criterion results, findings, evidence references, checked and not-checked surfaces, identities, and freshness. It carries no WARN, confidence, disposition, learning, owner, next action, retry, closure, diff --git a/images/gemini/skills/validate/SKILL.md b/images/gemini/skills/validate/SKILL.md index 98365d02d..03cc9c8a9 100644 --- a/images/gemini/skills/validate/SKILL.md +++ b/images/gemini/skills/validate/SKILL.md @@ -77,10 +77,53 @@ fresh context produced the PASS (`e9b6cdb8...37b9`). If a mutating check is genuinely required by acceptance, run it against a disposable copy or a committed subject, never the judged working tree. +## Scope disclosure + +`not_checked` has exactly one meaning: **in-scope acceptance surface this +validation did not verify**. PASS asserts that the whole declared acceptance +surface was verified, so a PASS carries no `not_checked` entries; the helper +refuses one and records a `validate.integrity` finding. + +That rule never pays for deleting an honest caveat, because every kind of scope +limit has a home that survives inside a PASS: + +| Scope limit | Home | Example | +|---|---|---| +| A criterion proven by a bounded check | `criteria[].reason` on that criterion | "proven by the unit suite; the full integration matrix was not replayed" | +| A declared non-goal or out-of-scope area | the intent source's non-goals, optionally restated as an evidence-backed boundary criterion in `criteria` | "`cli/**` is a declared non-goal; the diff proves it untouched" | +| Residual risk or judgment caveat | the caller-facing report | "the migration path is untested against pre-3.0 stores" | +| Acceptance that genuinely went unverified | `not_checked`, and the result is `NOT_PROVEN` rather than PASS | "criterion 3 needs hardware this context cannot reach" | + +Emptying `not_checked` to obtain PASS is a contract violation, not a +workaround. If acceptance really went unverified, the honest result is +`NOT_PROVEN`. If the entry was never acceptance in the first place, it belongs +in one of the other homes, where it stays visible in the stored artifact +instead of being deleted. + +## Helper commands + +The helper ships beside this file. Invoke it through this skill's own +directory rather than a checkout-relative path: `$SKILL_DIR` is the directory +containing this `SKILL.md` — `skills/validate/` in a repository checkout, +`.agents/skills/validate/` in an installed runtime. + +| Command | Required | Optional | +|---|---|---| +| `manifest` | `--root `, `--include ` (repeatable, at least one) | `--exclude ` (repeatable), `--base-manifest `, `--git-metadata-json `, `--output ` | +| `verify-manifest` | `--root `, `--manifest ` | `--base-manifest ` | +| `snapshot-intent` | `--source ` (`-` reads stdin) | `--workspace `, `--intent-dir ` | +| `digest` | `` positional | none | +| `store-verdict` | `--draft`, `--intent-source`, `--subject-manifest`, `--author-context-id`, `--validator-context-id`, `--freshness-source `, `--freshness-attester-id`, `--scope-result ` | `--workspace `, `--verdict-dir ` | + +```sh +python3 "$SKILL_DIR/scripts/validate.py" manifest \ + --root . --include skills/validate --exclude '**/*.log' --output manifest.json +``` + ## Workflow -1. Recompute and compare `subject-manifest.v1` using - `python3 skills/validate/scripts/validate.py manifest`. The helper uses only +1. Recompute and compare `subject-manifest.v1` with the `manifest` command + above (`--root` plus at least one `--include`). The helper uses only filesystem content; Git commit/tree IDs are optional metadata. Derive the manifest at the start of validation and re-derive it at the end; any mismatch between the two is subject mutation and returns `NOT_PROVEN`. @@ -95,15 +138,19 @@ committed subject, never the judged working tree. claims, not evidence: re-execute the claimed proofs that bear on acceptance (see the freshness rules below for when a digest-bound receipt suffices). Judge every acceptance criterion and record criterion-level results, - findings, evidence references, `checked`, and `not_checked`. + findings, evidence references, `checked`, and any acceptance surface that + went unverified in `not_checked` (see Scope disclosure). 5. Choose exactly one semantic result: `PASS`, `FAIL`, or `NOT_PROVEN`. Return it with criterion results, findings, evidence references, `checked`, `not_checked`, the acceptance and subject identities, distinct author and validator context IDs, and the freshness attestation. PASS requires distinct identities, explicit freshness, nonempty checked scope, top-level evidence, - and evidence for every criterion. + evidence for every criterion, and an empty `not_checked`; route bounded + proofs, declared non-goals, and residual risk to the homes named in Scope + disclosure rather than deleting them or downgrading a proven result. 6. Only when the caller requests machine-readable evidence or a declared - downstream consumer requires it, persist canonical `verdict.v2` with + downstream consumer requires it, persist canonical `verdict.v2` with the + helper's `store-verdict --draft --intent-source --subject-manifest --author-context-id --validator-context-id --freshness-source diff --git a/skills-codex/.agentops-manifest.json b/skills-codex/.agentops-manifest.json index 74a11fd79..7bc3b2933 100644 --- a/skills-codex/.agentops-manifest.json +++ b/skills-codex/.agentops-manifest.json @@ -639,8 +639,8 @@ { "name": "validate", "source_skill": "skills/validate", - "source_hash": "3f8fd13efb33dabd114dbb66deb0eea5d3762de888b081c753a27c1fba557e03", - "generated_hash": "93b79b850fe0c6581305e1357ad2e83923d3a83cd3d3c23be342e70c9e0902ca" + "source_hash": "4cad5b18f30ecba7944b1502624e38bb75fc329666464c614fc9051d4d3bd10c", + "generated_hash": "1938a59144b5aaf9de48e0c3e03e00750107b60904d517742dfcedf63d011304" }, { "name": "workflow-builder", diff --git a/skills-codex/validate/.agentops-generated.json b/skills-codex/validate/.agentops-generated.json index d3777098b..28db5c37c 100644 --- a/skills-codex/validate/.agentops-generated.json +++ b/skills-codex/validate/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/validate", "layout": "modular", - "source_hash": "3f8fd13efb33dabd114dbb66deb0eea5d3762de888b081c753a27c1fba557e03", - "generated_hash": "93b79b850fe0c6581305e1357ad2e83923d3a83cd3d3c23be342e70c9e0902ca" + "source_hash": "4cad5b18f30ecba7944b1502624e38bb75fc329666464c614fc9051d4d3bd10c", + "generated_hash": "1938a59144b5aaf9de48e0c3e03e00750107b60904d517742dfcedf63d011304" } diff --git a/skills-codex/validate/SKILL.md b/skills-codex/validate/SKILL.md index bf689bca1..05d18813d 100644 --- a/skills-codex/validate/SKILL.md +++ b/skills-codex/validate/SKILL.md @@ -49,10 +49,53 @@ fresh context produced the PASS (`e9b6cdb8...37b9`). If a mutating check is genuinely required by acceptance, run it against a disposable copy or a committed subject, never the judged working tree. +## Scope disclosure + +`not_checked` has exactly one meaning: **in-scope acceptance surface this +validation did not verify**. PASS asserts that the whole declared acceptance +surface was verified, so a PASS carries no `not_checked` entries; the helper +refuses one and records a `validate.integrity` finding. + +That rule never pays for deleting an honest caveat, because every kind of scope +limit has a home that survives inside a PASS: + +| Scope limit | Home | Example | +|---|---|---| +| A criterion proven by a bounded check | `criteria[].reason` on that criterion | "proven by the unit suite; the full integration matrix was not replayed" | +| A declared non-goal or out-of-scope area | the intent source's non-goals, optionally restated as an evidence-backed boundary criterion in `criteria` | "`cli/**` is a declared non-goal; the diff proves it untouched" | +| Residual risk or judgment caveat | the caller-facing report | "the migration path is untested against pre-3.0 stores" | +| Acceptance that genuinely went unverified | `not_checked`, and the result is `NOT_PROVEN` rather than PASS | "criterion 3 needs hardware this context cannot reach" | + +Emptying `not_checked` to obtain PASS is a contract violation, not a +workaround. If acceptance really went unverified, the honest result is +`NOT_PROVEN`. If the entry was never acceptance in the first place, it belongs +in one of the other homes, where it stays visible in the stored artifact +instead of being deleted. + +## Helper commands + +The helper ships beside this file. Invoke it through this skill's own +directory rather than a checkout-relative path: `$SKILL_DIR` is the directory +containing this `SKILL.md` — `skills/validate/` in a repository checkout, +`.agents/skills/validate/` in an installed runtime. + +| Command | Required | Optional | +|---|---|---| +| `manifest` | `--root `, `--include ` (repeatable, at least one) | `--exclude ` (repeatable), `--base-manifest `, `--git-metadata-json `, `--output ` | +| `verify-manifest` | `--root `, `--manifest ` | `--base-manifest ` | +| `snapshot-intent` | `--source ` (`-` reads stdin) | `--workspace `, `--intent-dir ` | +| `digest` | `` positional | none | +| `store-verdict` | `--draft`, `--intent-source`, `--subject-manifest`, `--author-context-id`, `--validator-context-id`, `--freshness-source `, `--freshness-attester-id`, `--scope-result ` | `--workspace `, `--verdict-dir ` | + +```sh +python3 "$SKILL_DIR/scripts/validate.py" manifest \ + --root . --include skills/validate --exclude '**/*.log' --output manifest.json +``` + ## Workflow -1. Recompute and compare `subject-manifest.v1` using - `python3 skills/validate/scripts/validate.py manifest`. The helper uses only +1. Recompute and compare `subject-manifest.v1` with the `manifest` command + above (`--root` plus at least one `--include`). The helper uses only filesystem content; Git commit/tree IDs are optional metadata. Derive the manifest at the start of validation and re-derive it at the end; any mismatch between the two is subject mutation and returns `NOT_PROVEN`. @@ -67,15 +110,19 @@ committed subject, never the judged working tree. claims, not evidence: re-execute the claimed proofs that bear on acceptance (see the freshness rules below for when a digest-bound receipt suffices). Judge every acceptance criterion and record criterion-level results, - findings, evidence references, `checked`, and `not_checked`. + findings, evidence references, `checked`, and any acceptance surface that + went unverified in `not_checked` (see Scope disclosure). 5. Choose exactly one semantic result: `PASS`, `FAIL`, or `NOT_PROVEN`. Return it with criterion results, findings, evidence references, `checked`, `not_checked`, the acceptance and subject identities, distinct author and validator context IDs, and the freshness attestation. PASS requires distinct identities, explicit freshness, nonempty checked scope, top-level evidence, - and evidence for every criterion. + evidence for every criterion, and an empty `not_checked`; route bounded + proofs, declared non-goals, and residual risk to the homes named in Scope + disclosure rather than deleting them or downgrading a proven result. 6. Only when the caller requests machine-readable evidence or a declared - downstream consumer requires it, persist canonical `verdict.v2` with + downstream consumer requires it, persist canonical `verdict.v2` with the + helper's `store-verdict --draft --intent-source --subject-manifest --author-context-id --validator-context-id --freshness-source diff --git a/skills-codex/validate/scripts/test_validate.py b/skills-codex/validate/scripts/test_validate.py index b94494f9d..4069779fa 100755 --- a/skills-codex/validate/scripts/test_validate.py +++ b/skills-codex/validate/scripts/test_validate.py @@ -278,6 +278,68 @@ def test_runtime_facts_override_model_authored_digests(self): self.assertEqual(artifact["subject_manifest_digest"], manifest["canonical_manifest_digest"]) self.assertEqual(artifact["verdict"], "PASS") + def test_honest_scoped_pass_round_trips_through_documented_homes(self): + """An honest draft with declared non-goals is representable as PASS. + + Both drafts below carry the same honest content. Draft A parks the + declared non-goals in ``not_checked``, which is reserved for unverified + in-scope acceptance: the result is NOT_PROVEN and the finding names + where each caveat belongs. Draft B moves the same caveats into the + documented homes and stores PASS with every caveat still readable in + the persisted artifact. Nothing is deleted to earn the PASS. + """ + bounded = "proven by the unit suite; the full integration matrix was not replayed" + boundary = "declared non-goal; the diff proves cli/** untouched" + + with tempfile.TemporaryDirectory() as raw: + draft_a = self.draft() + draft_a["not_checked"] = [ + "cli/** (declared non-goal)", + "Windows runners (declared non-goal)", + ] + artifact_a, _path, _existed = self.store_bound(draft_a, Path(raw)) + self.assertEqual(artifact_a["verdict"], "NOT_PROVEN") + summary = artifact_a["findings"][-1]["summary"] + self.assertIn("PASS cannot contain not_checked items", summary) + for home in ("criteria[].reason", "non-goal", "report"): + self.assertIn(home, summary) + self.assert_schema_valid(artifact_a) + + with tempfile.TemporaryDirectory() as raw: + draft_b = self.draft() + draft_b["criteria"][0]["reason"] = bounded + draft_b["criteria"].append( + { + "id": "non-goal:cli-untouched", + "result": "PASS", + "evidence_refs": ["git-diff:cli"], + "reason": boundary, + } + ) + draft_b["evidence_refs"] = ["e1", "git-diff:cli"] + draft_b["not_checked"] = [] + artifact_b, path, _existed = self.store_bound(draft_b, Path(raw)) + self.assertEqual(artifact_b["verdict"], "PASS") + self.assert_schema_valid(artifact_b) + # Round-trip: the caveats survive in the persisted PASS artifact. + stored = json.loads(path.read_text(encoding="utf-8")) + reasons = [criterion.get("reason") for criterion in stored["criteria"]] + self.assertIn(bounded, reasons) + self.assertIn(boundary, reasons) + self.assertEqual(stored["not_checked"], []) + self.assertEqual(stored["verdict"], "PASS") + + def test_criteria_field_error_names_the_allowed_set(self): + with tempfile.TemporaryDirectory() as raw: + draft = self.draft() + draft["criteria"][0]["confidence"] = "high" + with self.assertRaisesRegex( + tool.ContractError, + r"unknown confidence.*allowed fields are \{id, result, evidence_refs, reason\}", + ): + self.store_bound(draft, Path(raw)) + self.assertEqual(list(Path(raw).iterdir()), []) + def test_runtime_scope_failure_forces_fail(self): with tempfile.TemporaryDirectory() as raw: artifact, _path, _existed = self.store_bound(self.draft(), Path(raw), scope="FAIL") diff --git a/skills-codex/validate/scripts/validate.py b/skills-codex/validate/scripts/validate.py index b43d08103..b077b382b 100755 --- a/skills-codex/validate/scripts/validate.py +++ b/skills-codex/validate/scripts/validate.py @@ -22,6 +22,33 @@ HEX64 = set("0123456789abcdef") +# ``not_checked`` names the *in-scope acceptance surface a validator did not +# verify*. PASS asserts that the whole declared acceptance surface was +# verified, so a PASS carries no ``not_checked`` entries by construction. +# +# That rule only pays for honest disclosure if every kind of scope limit has a +# home that survives inside a PASS. Each does, so nothing is ever deleted to +# earn a PASS: +# +# * a bounded proof of a criterion -> ``criteria[].reason`` +# * a declared non-goal -> the intent source's non-goals, and +# optionally an evidence-backed boundary +# criterion in ``criteria`` +# * residual risk -> the caller-facing report +# +# ``not_checked`` stays reserved for its one meaning: acceptance that genuinely +# went unverified, which is NOT_PROVEN and not PASS. +NOT_CHECKED_HOMES = ( + "not_checked lists unverified in-scope acceptance surface, so a PASS has none by " + "construction; record a bounded proof of a criterion in criteria[].reason, a declared " + "non-goal in the intent source's non-goals (optionally as an evidence-backed boundary " + "criterion), and residual risk in the report; keep a not_checked entry only when " + "acceptance genuinely went unverified, which is NOT_PROVEN" +) + +CRITERION_KEYS = ("id", "result", "evidence_refs", "reason") +CRITERION_REQUIRED = ("id", "result", "evidence_refs") + class ContractError(ValueError): pass @@ -263,7 +290,7 @@ def enforce_identity(draft: dict[str, Any]) -> dict[str, Any]: if not isinstance(freshness, dict) or freshness.get("source") not in ("runtime", "caller") or not freshness.get("attester_identity"): problems.append("freshness attestation is missing or invalid") if draft.get("verdict") == "PASS" and (draft.get("not_checked") or []): - problems.append("PASS cannot contain not_checked items") + problems.append(f"PASS cannot contain not_checked items: {NOT_CHECKED_HOMES}") criteria = draft.get("criteria") if draft.get("verdict") == "PASS" and ( not isinstance(criteria, list) @@ -312,6 +339,20 @@ def require_string_list(value: Any, field: str, *, nonempty: bool = False) -> No raise ContractError(f"verdict.v2 {field} entries must be nonempty strings") +def criterion_fields_error(index: int, *, missing: list[str], unknown: list[str]) -> str: + """Return an actionable criteria-shape message naming the allowed field set.""" + detail: list[str] = [] + if missing: + detail.append(f"missing {', '.join(missing)}") + if unknown: + detail.append(f"unknown {', '.join(unknown)}") + problem = "; ".join(detail) if detail else "not an object" + return ( + f"verdict.v2 criteria[{index}] has invalid fields ({problem}); allowed fields are " + f"{{{', '.join(CRITERION_KEYS)}}}, of which {', '.join(CRITERION_REQUIRED)} are required" + ) + + def validate_verdict_v2(artifact: dict[str, Any]) -> None: """Enforce the complete bundled verdict.v2 contract before persistence.""" missing = sorted(VERDICT_KEYS - artifact.keys()) @@ -345,9 +386,14 @@ def validate_verdict_v2(artifact: dict[str, Any]) -> None: if not isinstance(criteria, list) or not criteria: raise ContractError("verdict.v2 criteria must be a nonempty array") for index, criterion in enumerate(criteria): - allowed = {"id", "result", "evidence_refs", "reason"} - if not isinstance(criterion, dict) or not {"id", "result", "evidence_refs"}.issubset(criterion) or not set(criterion).issubset(allowed): - raise ContractError(f"verdict.v2 criteria[{index}] has invalid fields") + if not isinstance(criterion, dict): + raise ContractError(criterion_fields_error(index, missing=[], unknown=[])) + missing_keys = [key for key in CRITERION_REQUIRED if key not in criterion] + unknown_keys = sorted(set(criterion) - set(CRITERION_KEYS)) + if missing_keys or unknown_keys: + raise ContractError( + criterion_fields_error(index, missing=missing_keys, unknown=unknown_keys) + ) if not isinstance(criterion["id"], str) or not criterion["id"]: raise ContractError(f"verdict.v2 criteria[{index}].id must be nonempty") if criterion["result"] not in {"PASS", "FAIL", "NOT_PROVEN"}: @@ -386,7 +432,9 @@ def validate_verdict_v2(artifact: dict[str, Any]) -> None: if any(not criterion["evidence_refs"] for criterion in criteria) or not artifact["evidence_refs"] or not artifact["checked"]: raise ContractError("verdict.v2 PASS requires criterion evidence plus nonempty evidence_refs and checked") if artifact["not_checked"]: - raise ContractError("verdict.v2 PASS cannot contain not_checked items") + raise ContractError( + f"verdict.v2 PASS cannot contain not_checked items: {NOT_CHECKED_HOMES}" + ) def artifact_bytes(draft: dict[str, Any]) -> tuple[dict[str, Any], bytes]: diff --git a/skills/validate/SKILL.md b/skills/validate/SKILL.md index 98365d02d..03cc9c8a9 100644 --- a/skills/validate/SKILL.md +++ b/skills/validate/SKILL.md @@ -77,10 +77,53 @@ fresh context produced the PASS (`e9b6cdb8...37b9`). If a mutating check is genuinely required by acceptance, run it against a disposable copy or a committed subject, never the judged working tree. +## Scope disclosure + +`not_checked` has exactly one meaning: **in-scope acceptance surface this +validation did not verify**. PASS asserts that the whole declared acceptance +surface was verified, so a PASS carries no `not_checked` entries; the helper +refuses one and records a `validate.integrity` finding. + +That rule never pays for deleting an honest caveat, because every kind of scope +limit has a home that survives inside a PASS: + +| Scope limit | Home | Example | +|---|---|---| +| A criterion proven by a bounded check | `criteria[].reason` on that criterion | "proven by the unit suite; the full integration matrix was not replayed" | +| A declared non-goal or out-of-scope area | the intent source's non-goals, optionally restated as an evidence-backed boundary criterion in `criteria` | "`cli/**` is a declared non-goal; the diff proves it untouched" | +| Residual risk or judgment caveat | the caller-facing report | "the migration path is untested against pre-3.0 stores" | +| Acceptance that genuinely went unverified | `not_checked`, and the result is `NOT_PROVEN` rather than PASS | "criterion 3 needs hardware this context cannot reach" | + +Emptying `not_checked` to obtain PASS is a contract violation, not a +workaround. If acceptance really went unverified, the honest result is +`NOT_PROVEN`. If the entry was never acceptance in the first place, it belongs +in one of the other homes, where it stays visible in the stored artifact +instead of being deleted. + +## Helper commands + +The helper ships beside this file. Invoke it through this skill's own +directory rather than a checkout-relative path: `$SKILL_DIR` is the directory +containing this `SKILL.md` — `skills/validate/` in a repository checkout, +`.agents/skills/validate/` in an installed runtime. + +| Command | Required | Optional | +|---|---|---| +| `manifest` | `--root `, `--include ` (repeatable, at least one) | `--exclude ` (repeatable), `--base-manifest `, `--git-metadata-json `, `--output ` | +| `verify-manifest` | `--root `, `--manifest ` | `--base-manifest ` | +| `snapshot-intent` | `--source ` (`-` reads stdin) | `--workspace `, `--intent-dir ` | +| `digest` | `` positional | none | +| `store-verdict` | `--draft`, `--intent-source`, `--subject-manifest`, `--author-context-id`, `--validator-context-id`, `--freshness-source `, `--freshness-attester-id`, `--scope-result ` | `--workspace `, `--verdict-dir ` | + +```sh +python3 "$SKILL_DIR/scripts/validate.py" manifest \ + --root . --include skills/validate --exclude '**/*.log' --output manifest.json +``` + ## Workflow -1. Recompute and compare `subject-manifest.v1` using - `python3 skills/validate/scripts/validate.py manifest`. The helper uses only +1. Recompute and compare `subject-manifest.v1` with the `manifest` command + above (`--root` plus at least one `--include`). The helper uses only filesystem content; Git commit/tree IDs are optional metadata. Derive the manifest at the start of validation and re-derive it at the end; any mismatch between the two is subject mutation and returns `NOT_PROVEN`. @@ -95,15 +138,19 @@ committed subject, never the judged working tree. claims, not evidence: re-execute the claimed proofs that bear on acceptance (see the freshness rules below for when a digest-bound receipt suffices). Judge every acceptance criterion and record criterion-level results, - findings, evidence references, `checked`, and `not_checked`. + findings, evidence references, `checked`, and any acceptance surface that + went unverified in `not_checked` (see Scope disclosure). 5. Choose exactly one semantic result: `PASS`, `FAIL`, or `NOT_PROVEN`. Return it with criterion results, findings, evidence references, `checked`, `not_checked`, the acceptance and subject identities, distinct author and validator context IDs, and the freshness attestation. PASS requires distinct identities, explicit freshness, nonempty checked scope, top-level evidence, - and evidence for every criterion. + evidence for every criterion, and an empty `not_checked`; route bounded + proofs, declared non-goals, and residual risk to the homes named in Scope + disclosure rather than deleting them or downgrading a proven result. 6. Only when the caller requests machine-readable evidence or a declared - downstream consumer requires it, persist canonical `verdict.v2` with + downstream consumer requires it, persist canonical `verdict.v2` with the + helper's `store-verdict --draft --intent-source --subject-manifest --author-context-id --validator-context-id --freshness-source diff --git a/skills/validate/scripts/test_validate.py b/skills/validate/scripts/test_validate.py index b94494f9d..4069779fa 100755 --- a/skills/validate/scripts/test_validate.py +++ b/skills/validate/scripts/test_validate.py @@ -278,6 +278,68 @@ def test_runtime_facts_override_model_authored_digests(self): self.assertEqual(artifact["subject_manifest_digest"], manifest["canonical_manifest_digest"]) self.assertEqual(artifact["verdict"], "PASS") + def test_honest_scoped_pass_round_trips_through_documented_homes(self): + """An honest draft with declared non-goals is representable as PASS. + + Both drafts below carry the same honest content. Draft A parks the + declared non-goals in ``not_checked``, which is reserved for unverified + in-scope acceptance: the result is NOT_PROVEN and the finding names + where each caveat belongs. Draft B moves the same caveats into the + documented homes and stores PASS with every caveat still readable in + the persisted artifact. Nothing is deleted to earn the PASS. + """ + bounded = "proven by the unit suite; the full integration matrix was not replayed" + boundary = "declared non-goal; the diff proves cli/** untouched" + + with tempfile.TemporaryDirectory() as raw: + draft_a = self.draft() + draft_a["not_checked"] = [ + "cli/** (declared non-goal)", + "Windows runners (declared non-goal)", + ] + artifact_a, _path, _existed = self.store_bound(draft_a, Path(raw)) + self.assertEqual(artifact_a["verdict"], "NOT_PROVEN") + summary = artifact_a["findings"][-1]["summary"] + self.assertIn("PASS cannot contain not_checked items", summary) + for home in ("criteria[].reason", "non-goal", "report"): + self.assertIn(home, summary) + self.assert_schema_valid(artifact_a) + + with tempfile.TemporaryDirectory() as raw: + draft_b = self.draft() + draft_b["criteria"][0]["reason"] = bounded + draft_b["criteria"].append( + { + "id": "non-goal:cli-untouched", + "result": "PASS", + "evidence_refs": ["git-diff:cli"], + "reason": boundary, + } + ) + draft_b["evidence_refs"] = ["e1", "git-diff:cli"] + draft_b["not_checked"] = [] + artifact_b, path, _existed = self.store_bound(draft_b, Path(raw)) + self.assertEqual(artifact_b["verdict"], "PASS") + self.assert_schema_valid(artifact_b) + # Round-trip: the caveats survive in the persisted PASS artifact. + stored = json.loads(path.read_text(encoding="utf-8")) + reasons = [criterion.get("reason") for criterion in stored["criteria"]] + self.assertIn(bounded, reasons) + self.assertIn(boundary, reasons) + self.assertEqual(stored["not_checked"], []) + self.assertEqual(stored["verdict"], "PASS") + + def test_criteria_field_error_names_the_allowed_set(self): + with tempfile.TemporaryDirectory() as raw: + draft = self.draft() + draft["criteria"][0]["confidence"] = "high" + with self.assertRaisesRegex( + tool.ContractError, + r"unknown confidence.*allowed fields are \{id, result, evidence_refs, reason\}", + ): + self.store_bound(draft, Path(raw)) + self.assertEqual(list(Path(raw).iterdir()), []) + def test_runtime_scope_failure_forces_fail(self): with tempfile.TemporaryDirectory() as raw: artifact, _path, _existed = self.store_bound(self.draft(), Path(raw), scope="FAIL") diff --git a/skills/validate/scripts/validate.py b/skills/validate/scripts/validate.py index b43d08103..b077b382b 100755 --- a/skills/validate/scripts/validate.py +++ b/skills/validate/scripts/validate.py @@ -22,6 +22,33 @@ HEX64 = set("0123456789abcdef") +# ``not_checked`` names the *in-scope acceptance surface a validator did not +# verify*. PASS asserts that the whole declared acceptance surface was +# verified, so a PASS carries no ``not_checked`` entries by construction. +# +# That rule only pays for honest disclosure if every kind of scope limit has a +# home that survives inside a PASS. Each does, so nothing is ever deleted to +# earn a PASS: +# +# * a bounded proof of a criterion -> ``criteria[].reason`` +# * a declared non-goal -> the intent source's non-goals, and +# optionally an evidence-backed boundary +# criterion in ``criteria`` +# * residual risk -> the caller-facing report +# +# ``not_checked`` stays reserved for its one meaning: acceptance that genuinely +# went unverified, which is NOT_PROVEN and not PASS. +NOT_CHECKED_HOMES = ( + "not_checked lists unverified in-scope acceptance surface, so a PASS has none by " + "construction; record a bounded proof of a criterion in criteria[].reason, a declared " + "non-goal in the intent source's non-goals (optionally as an evidence-backed boundary " + "criterion), and residual risk in the report; keep a not_checked entry only when " + "acceptance genuinely went unverified, which is NOT_PROVEN" +) + +CRITERION_KEYS = ("id", "result", "evidence_refs", "reason") +CRITERION_REQUIRED = ("id", "result", "evidence_refs") + class ContractError(ValueError): pass @@ -263,7 +290,7 @@ def enforce_identity(draft: dict[str, Any]) -> dict[str, Any]: if not isinstance(freshness, dict) or freshness.get("source") not in ("runtime", "caller") or not freshness.get("attester_identity"): problems.append("freshness attestation is missing or invalid") if draft.get("verdict") == "PASS" and (draft.get("not_checked") or []): - problems.append("PASS cannot contain not_checked items") + problems.append(f"PASS cannot contain not_checked items: {NOT_CHECKED_HOMES}") criteria = draft.get("criteria") if draft.get("verdict") == "PASS" and ( not isinstance(criteria, list) @@ -312,6 +339,20 @@ def require_string_list(value: Any, field: str, *, nonempty: bool = False) -> No raise ContractError(f"verdict.v2 {field} entries must be nonempty strings") +def criterion_fields_error(index: int, *, missing: list[str], unknown: list[str]) -> str: + """Return an actionable criteria-shape message naming the allowed field set.""" + detail: list[str] = [] + if missing: + detail.append(f"missing {', '.join(missing)}") + if unknown: + detail.append(f"unknown {', '.join(unknown)}") + problem = "; ".join(detail) if detail else "not an object" + return ( + f"verdict.v2 criteria[{index}] has invalid fields ({problem}); allowed fields are " + f"{{{', '.join(CRITERION_KEYS)}}}, of which {', '.join(CRITERION_REQUIRED)} are required" + ) + + def validate_verdict_v2(artifact: dict[str, Any]) -> None: """Enforce the complete bundled verdict.v2 contract before persistence.""" missing = sorted(VERDICT_KEYS - artifact.keys()) @@ -345,9 +386,14 @@ def validate_verdict_v2(artifact: dict[str, Any]) -> None: if not isinstance(criteria, list) or not criteria: raise ContractError("verdict.v2 criteria must be a nonempty array") for index, criterion in enumerate(criteria): - allowed = {"id", "result", "evidence_refs", "reason"} - if not isinstance(criterion, dict) or not {"id", "result", "evidence_refs"}.issubset(criterion) or not set(criterion).issubset(allowed): - raise ContractError(f"verdict.v2 criteria[{index}] has invalid fields") + if not isinstance(criterion, dict): + raise ContractError(criterion_fields_error(index, missing=[], unknown=[])) + missing_keys = [key for key in CRITERION_REQUIRED if key not in criterion] + unknown_keys = sorted(set(criterion) - set(CRITERION_KEYS)) + if missing_keys or unknown_keys: + raise ContractError( + criterion_fields_error(index, missing=missing_keys, unknown=unknown_keys) + ) if not isinstance(criterion["id"], str) or not criterion["id"]: raise ContractError(f"verdict.v2 criteria[{index}].id must be nonempty") if criterion["result"] not in {"PASS", "FAIL", "NOT_PROVEN"}: @@ -386,7 +432,9 @@ def validate_verdict_v2(artifact: dict[str, Any]) -> None: if any(not criterion["evidence_refs"] for criterion in criteria) or not artifact["evidence_refs"] or not artifact["checked"]: raise ContractError("verdict.v2 PASS requires criterion evidence plus nonempty evidence_refs and checked") if artifact["not_checked"]: - raise ContractError("verdict.v2 PASS cannot contain not_checked items") + raise ContractError( + f"verdict.v2 PASS cannot contain not_checked items: {NOT_CHECKED_HOMES}" + ) def artifact_bytes(draft: dict[str, Any]) -> tuple[dict[str, Any], bytes]: