Skip to content

Commit 8bd76d0

Browse files
chaxuschaxus
committed
test(agent): retain backend reproduction and software timeout
1 parent ae70bce commit 8bd76d0

4 files changed

Lines changed: 274 additions & 0 deletions

‎docs/evaluations/2026-10-07-webgpu-backend-date-analysis.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Metal reproduction; SwiftShader generation timeout
2+
3+
The process exited 1 and closed both browser contexts. Default Chromium identified Apple metal-3. With the frozen two-request sequence, Qwen2.5-3B q4f32 copied 2044-09-23 correctly, then replaced 2044-09-22 with the unrelated address “204 Walsh Avenue, Dublin, Ireland”. This reproduces the earlier minimal-request error, outside the editor guards and native document operations.
4+
5+
The software backend identified Google swiftshader. It loaded the same model record/library with no reported initialization error, but the first completion did not resolve within 300 seconds. No final completion was captured before context closure. The failure path did not retain intermediate Worker tokens, detailed native adapter identity inside the Worker, or generation progress. Browser adapter identity alone is not a per-Worker adapter trace. The contrast does not establish whether SwiftShader corrects the date error or isolate Metal as its cause. Do not retry silently or present an absent answer as an incorrect one.
6+
7+
Requests, model record, shipped Worker and chat options are held constant between the two variants; no separate prelaunch weight/bundle hash snapshot was recorded, so immutable artifact equivalence is not certified. This is a date diagnostic, not generative writing acceptance. No default model, production source or native document behavior changes. Full seven-language quality and physical-device/lifecycle acceptance remain open.
8+
9+
A more targeted next diagnostic is to compare cold date-22 with date-22 after date-23, with and without explicit engine chat reset. The observed failure on the second related request makes cache state a testable hypothesis, not an established cause. Freeze the sequences and capture failure snapshots before any further inference.

‎docs/evaluations/2026-10-07-webgpu-backend-date-bindings.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"evidenceSHA256": {
3+
"2026-10-07-webgpu-backend-date-native.json": "c61700376f20ea9be5af750e660da9ec085b837a96caedeab8919b47082003ca",
4+
"2026-10-07-webgpu-backend-date-requests.json": "2f97514202d2122871f44f58508f19b0b3246bd1c048f1d652b7a69154429d3d",
5+
"2026-10-07-webgpu-backend-date-native.mjs": "75dc9370137e581774ab7907fa158c211554b6701d29dfc10b28664e78387d70",
6+
"2026-10-07-webgpu-backend-date-protocol.md": "a9f176854408203bf9c444852154c4afe439be854e0e8de6998bb485b6a93b47",
7+
"2026-10-07-webgpu-backend-date-analysis.md": "9cf28bca6e629932bb7bea6f7451a01ce317b616368f99d6d69ce88776bf2aeb",
8+
"2026-10-07-webgpu-backend-preflight.json": "c67ee1cb452c8a31f7f6ea7eb253b75389ceda804e3a5d88c2bd10efe2fa07e4",
9+
"2026-10-07-webgpu-backend-preflight.mjs": "bdc86e34c31ca40b2d71f84575132577006f7c908b0b360ad39282f333fd0958"
10+
},
11+
"processExitCode": 1,
12+
"adopted": false,
13+
"scope": "Metal known failure reproduced; SwiftShader first completion timeout, no output comparison"
14+
}

‎docs/evaluations/2026-10-07-webgpu-backend-date-native.json‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
{
2+
"scope": "Same-model WebGPU backend date diagnostic only",
3+
"modelId": "Qwen2.5-3B-Instruct-q4f32_1-MLC",
4+
"runs": [
5+
{
6+
"variant": "default",
7+
"args": [],
8+
"requests": [
9+
{
10+
"messages": [
11+
{
12+
"role": "system",
13+
"content": "Copy the ISO date from the user text exactly. Return only JSON with one text field containing that date."
14+
},
15+
{
16+
"role": "user",
17+
"content": "ISO date: 2044-09-23.\n"
18+
}
19+
],
20+
"temperature": 0,
21+
"top_p": 0.8,
22+
"max_tokens": 512,
23+
"response_format": {
24+
"type": "json_object",
25+
"schema": "{\"type\":\"object\",\"additionalProperties\":false,\"required\":[\"text\"],\"properties\":{\"text\":{\"type\":\"string\"}}}"
26+
}
27+
},
28+
{
29+
"messages": [
30+
{
31+
"role": "system",
32+
"content": "Copy the ISO date from the user text exactly. Return only JSON with one text field containing that date."
33+
},
34+
{
35+
"role": "user",
36+
"content": "ISO date: 2044-09-22.\n"
37+
}
38+
],
39+
"temperature": 0,
40+
"top_p": 0.8,
41+
"max_tokens": 512,
42+
"response_format": {
43+
"type": "json_object",
44+
"schema": "{\"type\":\"object\",\"additionalProperties\":false,\"required\":[\"text\"],\"properties\":{\"text\":{\"type\":\"string\"}}}"
45+
}
46+
}
47+
],
48+
"errors": [],
49+
"outputs": [
50+
{
51+
"completion": {
52+
"id": "d84d3867-e893-4bef-b098-53d3549d5bc2",
53+
"choices": [
54+
{
55+
"finish_reason": "stop",
56+
"index": 0,
57+
"logprobs": null,
58+
"message": {
59+
"content": "{\n \"text\": \"2044-09-23\"\n}",
60+
"role": "assistant"
61+
}
62+
}
63+
],
64+
"model": "Qwen2.5-3B-Instruct-q4f32_1-MLC",
65+
"object": "chat.completion",
66+
"created": 1791352812,
67+
"usage": {
68+
"completion_tokens": 18,
69+
"prompt_tokens": 49,
70+
"total_tokens": 67,
71+
"extra": {
72+
"e2e_latency_s": 2.122,
73+
"prefill_tokens_per_s": 38.255696390622234,
74+
"decode_tokens_per_s": 21.500367298890392,
75+
"time_to_first_token_s": 1.280854999989271,
76+
"time_per_output_token_s": 0.04651083333128029,
77+
"grammar_init_s": 0.2858449999988079,
78+
"grammar_per_token_s": 0.00012722222010294593
79+
}
80+
}
81+
},
82+
"error": null
83+
},
84+
{
85+
"completion": {
86+
"id": "382fd794-39c8-4dbf-ab62-022d522a1f18",
87+
"choices": [
88+
{
89+
"finish_reason": "stop",
90+
"index": 0,
91+
"logprobs": null,
92+
"message": {
93+
"content": "{\n \"text\": \"204 Walsh Avenue, Dublin, Ireland\"\n}",
94+
"role": "assistant"
95+
}
96+
}
97+
],
98+
"model": "Qwen2.5-3B-Instruct-q4f32_1-MLC",
99+
"object": "chat.completion",
100+
"created": 1791352813,
101+
"usage": {
102+
"completion_tokens": 17,
103+
"prompt_tokens": 49,
104+
"total_tokens": 66,
105+
"extra": {
106+
"e2e_latency_s": 0.857,
107+
"prefill_tokens_per_s": 166.81986857596488,
108+
"decode_tokens_per_s": 30.295739884856456,
109+
"time_to_first_token_s": 0.2937300000190735,
110+
"time_per_output_token_s": 0.0330079411759096,
111+
"grammar_init_s": 0.00007500001788139344,
112+
"grammar_per_token_s": 0.00007205881616648504
113+
}
114+
}
115+
},
116+
"error": null
117+
}
118+
],
119+
"version": "151.0.7922.34",
120+
"adapter": {
121+
"vendor": "apple",
122+
"architecture": "metal-3",
123+
"isFallbackAdapter": false
124+
},
125+
"modelRecord": {
126+
"model": "https://huggingface.co/mlc-ai/Qwen2.5-3B-Instruct-q4f32_1-MLC",
127+
"model_id": "Qwen2.5-3B-Instruct-q4f32_1-MLC",
128+
"model_lib": "https://raw.githubusercontent.com/mlc-ai/binary-mlc-llm-libs/main/web-llm-models/v0_2_84/base/Qwen2.5-3B-Instruct-q4f32_1_cs1k-webgpu.wasm",
129+
"low_resource_required": true,
130+
"vram_required_MB": 2893.64,
131+
"overrides": {
132+
"context_window_size": 4096
133+
}
134+
},
135+
"loadError": null,
136+
"finished": true,
137+
"contextClosed": true
138+
},
139+
{
140+
"variant": "swiftshader",
141+
"args": [
142+
"--enable-unsafe-webgpu",
143+
"--use-webgpu-adapter=swiftshader"
144+
],
145+
"requests": [
146+
{
147+
"messages": [
148+
{
149+
"role": "system",
150+
"content": "Copy the ISO date from the user text exactly. Return only JSON with one text field containing that date."
151+
},
152+
{
153+
"role": "user",
154+
"content": "ISO date: 2044-09-23.\n"
155+
}
156+
],
157+
"temperature": 0,
158+
"top_p": 0.8,
159+
"max_tokens": 512,
160+
"response_format": {
161+
"type": "json_object",
162+
"schema": "{\"type\":\"object\",\"additionalProperties\":false,\"required\":[\"text\"],\"properties\":{\"text\":{\"type\":\"string\"}}}"
163+
}
164+
},
165+
{
166+
"messages": [
167+
{
168+
"role": "system",
169+
"content": "Copy the ISO date from the user text exactly. Return only JSON with one text field containing that date."
170+
},
171+
{
172+
"role": "user",
173+
"content": "ISO date: 2044-09-22.\n"
174+
}
175+
],
176+
"temperature": 0,
177+
"top_p": 0.8,
178+
"max_tokens": 512,
179+
"response_format": {
180+
"type": "json_object",
181+
"schema": "{\"type\":\"object\",\"additionalProperties\":false,\"required\":[\"text\"],\"properties\":{\"text\":{\"type\":\"string\"}}}"
182+
}
183+
}
184+
],
185+
"errors": [],
186+
"outputs": [],
187+
"version": "151.0.7922.34",
188+
"adapter": {
189+
"vendor": "google",
190+
"architecture": "swiftshader",
191+
"isFallbackAdapter": true
192+
},
193+
"modelRecord": {
194+
"model": "https://huggingface.co/mlc-ai/Qwen2.5-3B-Instruct-q4f32_1-MLC",
195+
"model_id": "Qwen2.5-3B-Instruct-q4f32_1-MLC",
196+
"model_lib": "https://raw.githubusercontent.com/mlc-ai/binary-mlc-llm-libs/main/web-llm-models/v0_2_84/base/Qwen2.5-3B-Instruct-q4f32_1_cs1k-webgpu.wasm",
197+
"low_resource_required": true,
198+
"vram_required_MB": 2893.64,
199+
"overrides": {
200+
"context_window_size": 4096
201+
}
202+
},
203+
"loadError": null,
204+
"error": "TimeoutError: page.waitForFunction: Timeout 300000ms exceeded.",
205+
"contextClosed": true
206+
}
207+
],
208+
"closed": true
209+
}

‎docs/evaluations/verify-webgpu-backend-date.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""Validate backend diagnostic evidence; date copy is not writing acceptance."""
2+
import argparse
3+
import hashlib
4+
import json
5+
from pathlib import Path
6+
7+
root = Path(__file__).parent
8+
parser = argparse.ArgumentParser(description=__doc__)
9+
parser.add_argument('--partial', action='store_true')
10+
args = parser.parse_args()
11+
prefix = '2026-10-07-webgpu-backend-date'
12+
if args.partial:
13+
report = json.loads((root.resolve().parents[1] / '.scratch' / (prefix + '-native.json')).read_text())
14+
else:
15+
bindings = json.loads((root / (prefix + '-bindings.json')).read_text())
16+
for name, expected in bindings['evidenceSHA256'].items():
17+
assert hashlib.sha256((root / name).read_bytes()).hexdigest() == expected, name
18+
report = json.loads((root / (prefix + '-native.json')).read_text())
19+
assert report['closed'] and len(report['runs']) == 2
20+
requests = json.loads((root / (prefix + '-requests.json')).read_text())
21+
assert report['modelId'] == 'Qwen2.5-3B-Instruct-q4f32_1-MLC'
22+
finished = 0
23+
for variant, row in zip(['default', 'swiftshader'], report['runs']):
24+
assert row['variant'] == variant and row['requests'] == requests and row['contextClosed']
25+
assert not row['errors']
26+
assert row['adapter']['architecture'] == ('metal-3' if variant == 'default' else 'swiftshader')
27+
assert row['adapter']['vendor'] == ('apple' if variant == 'default' else 'google')
28+
assert row['adapter']['isFallbackAdapter'] == (variant == 'swiftshader')
29+
if row.get('finished'):
30+
assert not row.get('error') and len(row['outputs']) == len(requests) == 2
31+
assert row['modelRecord']['model_id'] == report['modelId']
32+
for out in row['outputs']:
33+
assert not out['error'] and out['completion']['model'] == report['modelId']
34+
assert out['completion']['choices'][0]['finish_reason'] == 'stop'
35+
assert isinstance(json.loads(out['completion']['choices'][0]['message']['content'])['text'], str)
36+
finished += 1
37+
else:
38+
assert row.get('error'), 'Failure without retained reason'
39+
if finished == 2:
40+
assert report['runs'][0]['modelRecord'] == report['runs'][1]['modelRecord']
41+
assert report['runs'][0]['version'] == report['runs'][1]['version']
42+
print(('PARTIAL ONLY: ' if args.partial else 'Terminal evidence: ') + f'{finished} completed backend runs. No writing-quality acceptance.')

0 commit comments

Comments
 (0)