Skip to content

Commit d00a680

Browse files
xhlucaxhluca
committed
test: require explicit native modality status
1 parent 35f3a45 commit d00a680

8 files changed

Lines changed: 107 additions & 22 deletions

File tree

‎tests/native_corpus/loader.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -489,8 +489,11 @@ def _load_modalities(value: Any, fixture_id: str) -> Mapping[str, ModalitySpec]:
489489
unknown = set(document) - ALLOWED_MODALITIES
490490
if unknown:
491491
raise CorpusValidationError(f"{fixture_id}: unsupported modalities: {sorted(unknown)}")
492-
if "text" not in document:
493-
raise CorpusValidationError(f"{fixture_id}: text modality is required")
492+
missing = ALLOWED_MODALITIES - set(document)
493+
if missing:
494+
raise CorpusValidationError(
495+
f"{fixture_id}: every modality must have an explicit status; missing={sorted(missing)}"
496+
)
494497
result: dict[str, ModalitySpec] = {}
495498
for name, item in document.items():
496499
label = f"{fixture_id}.modalities.{name}"

‎tests/native_corpus/schema/provenance.schema.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,10 @@
8181
"modalities": {
8282
"type": "object",
8383
"additionalProperties": false,
84-
"required": ["text"],
84+
"required": [
85+
"audio", "compaction", "document", "readable_reasoning", "text",
86+
"tool_call", "tool_result", "tool_result_image", "user_image", "video"
87+
],
8588
"properties": {
8689
"audio": {"$ref": "#/$defs/modality"},
8790
"compaction": {"$ref": "#/$defs/modality"},

‎tests/native_corpus/v1/sources/claude/2.1.209/portable-rich/provenance.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@
5959
"fixture_present": true,
6060
"portable": "preserve"
6161
},
62+
"tool_result_image": {
63+
"attempted": false,
64+
"native_accepted": false,
65+
"fixture_present": false,
66+
"portable": "preserve"
67+
},
6268
"user_image": {
6369
"attempted": true,
6470
"native_accepted": true,
@@ -82,13 +88,21 @@
8288
"native_accepted": false,
8389
"fixture_present": false,
8490
"portable": "unsupported"
91+
},
92+
"compaction": {
93+
"attempted": false,
94+
"native_accepted": false,
95+
"fixture_present": false,
96+
"portable": "preserve"
8597
}
8698
},
8799
"observations": {
100+
"tool_result_image": "This bounded exact source capture exercised a native user image but did not ask a tool to return an image; the separate Claude adapter suite covers portable image blocks inside tool results.",
88101
"user_image": "The public stream-json input surface accepted the exact PNG as a native base64 image block; the model reported a blue triangle and nonce 7319.",
89102
"document": "The public stream-json input surface accepted the exact PDF as a native base64 document block; the model reported ORBIT_2048. The current source adapter retains its document type but not its bytes, so portability is explicitly lossy.",
90103
"audio": "A fresh exact-client stream-json turn containing the WAV as an audio block received the vendor's HTTP 400 unsupported-content-tag response before a successful model turn.",
91-
"video": "A fresh exact-client stream-json turn containing the MP4 as a video block received the vendor's HTTP 400 unsupported-content-tag response before a successful model turn."
104+
"video": "A fresh exact-client stream-json turn containing the MP4 as a video block received the vendor's HTTP 400 unsupported-content-tag response before a successful model turn.",
105+
"compaction": "The bounded exact media/tool capture did not trigger the public compaction control; separate parser/writer tests cover Claude compact_boundary and compact_summary records."
92106
},
93107
"sanitization": {
94108
"required": true,

‎tests/native_corpus/v1/sources/codex/0.144.4/portable-rich/provenance.json‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,20 @@
8888
"native_accepted": false,
8989
"fixture_present": false,
9090
"portable": "unsupported"
91+
},
92+
"compaction": {
93+
"attempted": false,
94+
"native_accepted": false,
95+
"fixture_present": false,
96+
"portable": "preserve"
9197
}
9298
},
9399
"observations": {
94100
"user_image": "The documented --image surface accepted the exact PNG and stored it as a native input_image data URL; both the user image and a native tool-result image retain the exact PNG digest.",
95101
"document": "The only pinned public attachment flag is --image. Passing the PDF exited successfully but projected it through the image-oriented surface; the model explicitly did not inspect its content, and no distinct document modality exists in the public protocol.",
96102
"audio": "Passing the WAV through the only public attachment flag exited successfully but projected it through the image-oriented surface; the model explicitly did not transcribe or interpret it, and the public protocol advertises only text and image input modalities.",
97-
"video": "Passing the MP4 through the only public attachment flag exited successfully but projected it through the image-oriented surface; the model explicitly could not inspect it, and the public protocol advertises only text and image input modalities."
103+
"video": "Passing the MP4 through the only public attachment flag exited successfully but projected it through the image-oriented surface; the model explicitly could not inspect it, and the public protocol advertises only text and image input modalities.",
104+
"compaction": "The bounded exact media/tool capture did not force automatic context compaction; separate parser/writer tests cover compacted and context_compacted rollout records."
98105
},
99106
"sanitization": {
100107
"required": true,

‎tests/native_corpus/v1/sources/devin/3000.6.7/portable-rich/provenance.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@
5959
"fixture_present": true,
6060
"portable": "preserve"
6161
},
62+
"tool_result_image": {
63+
"attempted": false,
64+
"native_accepted": false,
65+
"fixture_present": false,
66+
"portable": "lossy"
67+
},
6268
"user_image": {
6369
"attempted": true,
6470
"native_accepted": true,
@@ -82,17 +88,25 @@
8288
"native_accepted": true,
8389
"fixture_present": false,
8490
"portable": "lossy"
91+
},
92+
"compaction": {
93+
"attempted": false,
94+
"native_accepted": false,
95+
"fixture_present": false,
96+
"portable": "preserve"
8597
}
8698
},
8799
"observations": {
88100
"text": "The exact client created a new native session, retained the initial and recall turns, and recalled SM_CORPUS_7319 and COPPER_4821 after an exact ACP load of the same ID.",
89101
"readable_reasoning": "The native active chain contained three provider reasoning objects. Their private text was replaced schema-aware; the portable projection remains explicitly lossy.",
90102
"tool_call": "The vendor model issued three linked native read calls for timeline.py, CORPUS_NOTE.txt, and the missing-file probe through Devin's ACP file surface.",
91103
"tool_result": "All three native results remain linked. The missing-file result contains the observed failure text, but Devin's completed message node does not carry a distinct error boolean.",
104+
"tool_result_image": "The exact source used native user-image input but did not induce a tool result carrying an image; Devin tool-result images are explicitly projected as lossy native metadata by the adapter suite.",
92105
"user_image": "The exact ACP surface accepted the PNG and JPEG, persisted both as native base64 images, and the vendor model reported the blue triangle and nonce 7319.",
93106
"document": "ACP accepted the PDF embedded-resource block, but the native database retained only a binary-resource text marker. The model incorrectly reported 7319 rather than ORBIT_2048, so document interpretation and byte persistence are not claimed.",
94107
"audio": "ACP accepted the WAV despite advertising audio=false. The exact client retained only an [Audio content] marker and the model acknowledged it without inferring content; audio bytes are not portable from this store.",
95-
"video": "ACP accepted the MP4 embedded-resource block without a protocol error, but the native database retained only a binary-resource text marker and the model merely acknowledged the video; video bytes are not portable from this store."
108+
"video": "ACP accepted the MP4 embedded-resource block without a protocol error, but the native database retained only a binary-resource text marker and the model merely acknowledged the video; video bytes are not portable from this store.",
109+
"compaction": "The bounded exact source did not reach Devin's imported-summary representation; separate parser/writer regressions cover portable compaction and repeated rewrite stability."
96110
},
97111
"sanitization": {
98112
"required": true,

‎tests/native_corpus/v1/sources/kimi/0.38.0/portable-rich/provenance.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,12 @@
6767
"fixture_present": true,
6868
"portable": "preserve"
6969
},
70+
"tool_result_image": {
71+
"attempted": false,
72+
"native_accepted": false,
73+
"fixture_present": false,
74+
"portable": "preserve"
75+
},
7076
"user_image": {
7177
"attempted": true,
7278
"native_accepted": true,
@@ -90,10 +96,18 @@
9096
"native_accepted": false,
9197
"fixture_present": false,
9298
"portable": "unsupported"
99+
},
100+
"compaction": {
101+
"attempted": false,
102+
"native_accepted": false,
103+
"fixture_present": false,
104+
"portable": "preserve"
93105
}
94106
},
95107
"observations": {
96-
"user_image": "The public TUI staged the exact PNG and showed its 640x400 dimensions, then hung for more than six minutes before creating a session or model request; the hung attempt is not fabricated into this text-and-tool fixture."
108+
"user_image": "The public TUI staged the exact PNG and showed its 640x400 dimensions, then hung for more than six minutes before creating a session or model request; the hung attempt is not fabricated into this text-and-tool fixture.",
109+
"tool_result_image": "The bounded exact source did not induce an image-bearing tool result; the adapter's structured image-result mapping is covered mechanically outside this native capture.",
110+
"compaction": "The bounded exact OpenRouter trajectory did not trigger Kimi context compaction; the adapter's native summary record is covered by parser/writer tests."
97111
},
98112
"sanitization": {
99113
"required": true,

‎tests/native_corpus/v1/sources/qwen/0.22.1/portable-rich/provenance.json‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@
5353
"fixture_present": true,
5454
"portable": "preserve"
5555
},
56+
"tool_result_image": {
57+
"attempted": false,
58+
"native_accepted": false,
59+
"fixture_present": false,
60+
"portable": "preserve"
61+
},
5662
"user_image": {
5763
"attempted": true,
5864
"native_accepted": false,
@@ -76,10 +82,25 @@
7682
"native_accepted": false,
7783
"fixture_present": false,
7884
"portable": "unsupported"
85+
},
86+
"compaction": {
87+
"attempted": false,
88+
"native_accepted": false,
89+
"fixture_present": false,
90+
"portable": "unsupported"
91+
},
92+
"readable_reasoning": {
93+
"attempted": false,
94+
"native_accepted": false,
95+
"fixture_present": false,
96+
"portable": "unsupported"
7997
}
8098
},
8199
"observations": {
82-
"user_image": "The public @file surface emitted a native unsupported-image rejection; no image event or bytes entered the session."
100+
"user_image": "The public @file surface emitted a native unsupported-image rejection; no image event or bytes entered the session.",
101+
"tool_result_image": "The bounded exact source did not induce an image-bearing tool result; the adapter's generic structured-result image mapping is covered mechanically outside this native capture.",
102+
"compaction": "The pinned Qwen session schema and public capture surface exposed no portable compaction record for this scenario.",
103+
"readable_reasoning": "The pinned Qwen session schema exposed no readable reasoning record for this scenario."
83104
},
84105
"sanitization": {
85106
"required": true,

‎tests/test_native_corpus_loader.py‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import pytest
99
from native_corpus.loader import (
10+
ALLOWED_MODALITIES,
1011
EXPECTED_FORMATS,
1112
ArtifactSpec,
1213
CorpusValidationError,
@@ -62,6 +63,27 @@ def _toy_corpus(root: Path) -> Path:
6263
mode=0o600,
6364
role="transcript",
6465
)
66+
modalities = {
67+
name: {
68+
"attempted": False,
69+
"native_accepted": False,
70+
"fixture_present": False,
71+
"portable": "unsupported",
72+
}
73+
for name in sorted(ALLOWED_MODALITIES)
74+
}
75+
modalities["text"] = {
76+
"attempted": True,
77+
"native_accepted": True,
78+
"fixture_present": True,
79+
"portable": "preserve",
80+
}
81+
modalities["audio"] = {
82+
"attempted": True,
83+
"native_accepted": False,
84+
"fixture_present": False,
85+
"portable": "unsupported",
86+
}
6587
provenance = {
6688
"schema_version": 1,
6789
"fixture_id": f"{format_name}-1.0-portable-rich",
@@ -98,20 +120,7 @@ def _toy_corpus(root: Path) -> Path:
98120
"role": artifact.role,
99121
}
100122
],
101-
"modalities": {
102-
"text": {
103-
"attempted": True,
104-
"native_accepted": True,
105-
"fixture_present": True,
106-
"portable": "preserve",
107-
},
108-
"audio": {
109-
"attempted": True,
110-
"native_accepted": False,
111-
"fixture_present": False,
112-
"portable": "unsupported",
113-
},
114-
},
123+
"modalities": modalities,
115124
"sanitization": {
116125
"required": True,
117126
"sanitizer": f"scripts/native_corpus/sanitize_{format_name}.py",

0 commit comments

Comments
 (0)