Skip to content

Commit 7e4c87f

Browse files
chaxuschaxus
committed
test(agent): freeze current-build Q4 and Q6 native contrast
1 parent 4d2814b commit 7e4c87f

1 file changed

Lines changed: 209 additions & 0 deletions

File tree

‎docs/evaluations/2026-10-07-qwen3-instruct2507-q6-native.mjs‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
import { chromium } from '@playwright/test';
2+
import fs from 'node:fs/promises';
3+
import { createReadStream } from 'node:fs';
4+
import { createHash } from 'node:crypto';
5+
const q4=JSON.parse(await fs.readFile('docs/evaluations/2026-10-07-qwen3-instruct2507-artifact.json','utf8'));
6+
const q6=JSON.parse(await fs.readFile('docs/evaluations/2026-10-07-qwen3-instruct2507-q6-artifact.json','utf8'));
7+
const artifacts=[{variant:'Q4_K_M',file:q4.file,bytes:q4.expectedBytes,sha256:q4.expectedSHA256},{variant:'Q6_K',...q6.files[0]}];
8+
const verifiedArtifacts=[];
9+
for(const item of artifacts){
10+
const path='.scratch/node_modules/ai-models/'+item.file;const bytes=(await fs.stat(path)).size;const hash=createHash('sha256');
11+
for await(const chunk of createReadStream(path))hash.update(chunk);const sha256=hash.digest('hex');
12+
if(bytes!==item.bytes||sha256!==item.sha256)throw Error('Pinned candidate integrity mismatch; no load');
13+
verifiedArtifacts.push({variant:item.variant,file:item.file,bytes,sha256});
14+
}
15+
const prompts = JSON.parse(await fs.readFile('docs/evaluations/2026-10-07-qwen3-instruct2507-seven-language-cases.json', 'utf8')).filter(x=>['zh-CN-summarize','es-rewrite','ja-translate','de-rewrite'].includes(x.id));
16+
const report = {
17+
runtime: '64c1899',
18+
build: 'core1791357432/vendor-a048b43b3e2b',
19+
protocol: '2026-10-07-qwen3-instruct2507-q6-protocol.md',
20+
model: 'Qwen3-4B-Instruct-2507 Q4/Q6 pinned producer revision',
21+
verifiedArtifacts,
22+
cases: [],
23+
};
24+
const browser = await chromium.launch({ channel: 'chromium' });
25+
report.browserVersion = browser.version();
26+
try {
27+
for (const prompt of prompts)
28+
for (const variant of ['Q4_K_M','Q6_K']) {
29+
const row = { prompt, variant, errors: [], console: [], crashed: false };
30+
report.cases.push(row);
31+
const context = await browser.newContext({ serviceWorkers: 'block', viewport: { width: 1280, height: 900 } });
32+
await context.addInitScript(() => {
33+
if (window === top) {
34+
localStorage.setItem('agent-panel-provider', 'wllama');
35+
localStorage.removeItem('agent-panel-gguf-url');
36+
}
37+
});
38+
try {
39+
const page = await context.newPage();
40+
page.on('pageerror', (e) => row.errors.push(e.message));
41+
page.on('crash', () => {
42+
row.crashed = true;
43+
});
44+
page.on('console', (message) => {
45+
if (['error', 'warning'].includes(message.type()) && row.console.length < 100)
46+
row.console.push({ type: message.type(), text: message.text().slice(0, 2000) });
47+
});
48+
await page.goto('http://127.0.0.1:5193/editor?new=docx&agent=1&locale=zh-CN');
49+
await page.waitForFunction(
50+
() => {
51+
const e = document.querySelector('#app iframe')?.contentWindow.editor;
52+
return e?.isDocumentLoadComplete && e?.isLoadFullApi;
53+
},
54+
null,
55+
{ timeout: 90000 },
56+
);
57+
await page.evaluate(async()=>{
58+
const {Wllama}=await import('/assets/client-D5_UYdz1.js');const original=Wllama.prototype.loadModel;window.__loads=[];
59+
Wllama.prototype.loadModel=async function(files,options){const probe={requested:{n_threads:options.n_threads,n_ctx:options.n_ctx,n_gpu_layers:options.n_gpu_layers,reasoning:options.reasoning}};window.__loads.push(probe);const result=await original.call(this,files,options);probe.actualThreads=this.getNumThreads();probe.multithread=this.isMultithread();return result;};
60+
});
61+
await page.evaluate(() => window.__toggleAgentPanel());
62+
await page.locator('.agent-panel').waitFor();
63+
await page.locator('.agent-panel-settings-toggle').click();
64+
await page
65+
.locator('.agent-panel-gguf-files')
66+
.setInputFiles('.scratch/node_modules/ai-models/'+artifacts.find(x=>x.variant===variant).file);
67+
await page.locator('.agent-panel-gguf-load').click();
68+
await page.waitForFunction(
69+
() => !document.querySelector('.agent-panel-gguf-load').hasAttribute('disabled'),
70+
null,
71+
{ timeout: 180000 },
72+
);
73+
row.loads = await page.evaluate(()=>window.__loads);
74+
row.runtimeEnvironment=await page.evaluate(()=>({crossOriginIsolated,hardwareConcurrency:navigator.hardwareConcurrency,sharedArrayBuffer:typeof SharedArrayBuffer,runtimeResources:performance.getEntriesByType('resource').map(e=>e.name).filter(n=>/wllama|wasm|client-D5_UYdz1/.test(n))}));
75+
row.modelNote = await page.locator('.agent-panel-note').textContent();
76+
row.modelStatus = await page.locator('.agent-model-status').textContent();
77+
if(!row.modelNote.includes('模型已加载'))throw Error('Native startup readiness failed; no inference allowed');
78+
await page.locator('.agent-panel-settings-toggle').click();
79+
await page.evaluate((text) => {
80+
const api = document.querySelector('#app iframe').contentWindow.Asc.editor;
81+
api.pluginMethod_InputText(text);
82+
}, prompt.source);
83+
await page.waitForFunction(source =>
84+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText() === source + '\r\n',
85+
prompt.source, { timeout: 90000 });
86+
await page.evaluate(() => document.querySelector('#app iframe').contentWindow.Asc.editor.asc_EditSelectAll());
87+
row.selected = await page.evaluate(() =>
88+
document.querySelector('#app iframe').contentWindow.Asc.editor.pluginMethod_GetSelectedText(),
89+
);
90+
if (row.selected !== prompt.source + '\r\n') throw Error('Native source selection mismatch; no inference allowed');
91+
await page.locator('.agent-writing-task').selectOption(prompt.task);
92+
if(prompt.task==='translate') await page.locator('.agent-writing-language').selectOption(prompt.targetLanguage);
93+
row.documentBefore = await page.evaluate(() =>
94+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText(),
95+
);
96+
await page.evaluate(
97+
async () => {
98+
const { Wllama } = await import('/assets/client-D5_UYdz1.js');
99+
const adjust = (request) => {
100+
return ({...request, seed:42, temperature:0.7, top_p:0.8, top_k:20});
101+
};
102+
const count = Wllama.prototype.countChatTokens;
103+
window.__counts = [];
104+
Wllama.prototype.countChatTokens = function (request) {
105+
request = adjust(request);
106+
const { abortSignal: _abortSignal, ...body } = request;
107+
window.__counts.push(JSON.parse(JSON.stringify(body)));
108+
return count.call(this, request);
109+
};
110+
const original = Wllama.prototype.createChatCompletion;
111+
window.__streamProbe = [];
112+
Wllama.prototype.createChatCompletion = function (request) {
113+
const originalMessages = structuredClone(request.messages);
114+
request = adjust(request);
115+
const { abortSignal: _abortSignal, ...rest } = request;
116+
const probe = { originalMessages, request: JSON.parse(JSON.stringify(rest)), chunks: [] };
117+
window.__streamProbe.push(probe);
118+
return Promise.resolve(original.call(this, request)).then((result) => {
119+
if (!result?.[Symbol.asyncIterator]) {
120+
probe.completion = result;
121+
return result;
122+
}
123+
return (async function* () {
124+
try {
125+
for await (const chunk of result) {
126+
probe.chunks.push(JSON.parse(JSON.stringify(chunk)));
127+
yield chunk;
128+
}
129+
probe.done = true;
130+
} catch (e) {
131+
probe.error = String(e);
132+
throw e;
133+
}
134+
})();
135+
});
136+
};
137+
},
138+
139+
);
140+
await page.locator('.cui-input').fill(prompt.instruction);
141+
const start = Date.now();
142+
await page.locator('.cui-input').focus();
143+
await page.locator('.cui-input').press('Enter');
144+
await page.locator('.cui-msg-user').filter({ hasText: prompt.instruction }).last().waitFor({ timeout: 10000 });
145+
await page.waitForFunction(() => !document.querySelector('.cui-input').disabled, null, { timeout: 240000 });
146+
row.durationMs = Date.now() - start;
147+
row.toolMessages = await page.locator('.cui-activity > div, .cui-activity summary > div').allTextContents();
148+
row.previewCount = await page.locator('.agent-plan-preview').count();
149+
row.chatErrors = await page.locator('.cui-msg-error').allTextContents();
150+
row.documentAfter = await page.evaluate(() =>
151+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText(),
152+
);
153+
row.documentUnchanged = row.documentBefore === row.documentAfter;
154+
if (!row.documentUnchanged) {
155+
await page.evaluate(() => document.querySelector('#app iframe').contentWindow.Asc.editor.Undo());
156+
row.afterUndo = await page.evaluate(() =>
157+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText(),
158+
);
159+
await page.evaluate(() => document.querySelector('#app iframe').contentWindow.Asc.editor.Redo());
160+
row.afterRedo = await page.evaluate(() =>
161+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText(),
162+
);
163+
}
164+
row.sdk = await page.evaluate(() => window.__streamProbe);
165+
row.counts = await page.evaluate(() => window.__counts);
166+
if (row.sdk.length !== 1 || row.sdk[0].request.temperature !== 0.7) throw Error('Unexpected bounded CPU request');
167+
if (JSON.stringify(row.counts.at(-1)) !== JSON.stringify(row.sdk[0].request))
168+
throw Error('Count/completion mismatch');
169+
row.finished = true;
170+
} catch (e) {
171+
row.error = String(e);
172+
const page = context.pages()[0];
173+
if (page && !page.isClosed()) {
174+
row.failureSnapshot=await page.evaluate(()=>({loads:window.__loads,sdk:window.__streamProbe,counts:window.__counts,documentText:document.querySelector('#app iframe')?.contentWindow?.Asc?.editor?.WordControl?.m_oLogicDocument?.GetText?.(),chatErrors:[...document.querySelectorAll('.cui-msg-error')].map(e=>e.textContent)})).catch(e=>({error:String(e)}));
175+
row.failureSnapshot = await page.evaluate(()=>({loads:window.__loads,sdk:window.__streamProbe,counts:window.__counts,documentText:document.querySelector('#app iframe')?.contentWindow?.Asc?.editor?.WordControl?.m_oLogicDocument?.GetText?.(),chatErrors:[...document.querySelectorAll('.cui-msg-error')].map(e=>e.textContent)})).catch(e=>({error:String(e)}));
176+
row.failureNote = await page
177+
.locator('.agent-panel-note')
178+
.textContent()
179+
.catch(() => null);
180+
row.failureStatus = await page
181+
.locator('.agent-model-status')
182+
.textContent()
183+
.catch(() => null);
184+
}
185+
} finally {
186+
await context.close();
187+
row.contextClosed = true;
188+
await fs.writeFile('.scratch/2026-10-07-qwen3-instruct2507-q6-native.json', JSON.stringify(report, null, 2));
189+
console.log(
190+
JSON.stringify({
191+
variant: row.variant,
192+
source: row.prompt.source,
193+
body: row.documentAfter,
194+
errors: row.chatErrors,
195+
error: row.error,
196+
finished: row.finished,
197+
closed: row.contextClosed,
198+
}),
199+
);
200+
}
201+
if (!row.modelStatus) throw Error('Initial model setup failed; stop duplicate loading attempts');
202+
}
203+
} finally {
204+
await browser.close();
205+
report.browserClosed = true;
206+
await fs.writeFile('.scratch/2026-10-07-qwen3-instruct2507-q6-native.json', JSON.stringify(report, null, 2));
207+
}
208+
209+
if (report.cases.some((row) => !row.finished || row.error || !row.contextClosed)) process.exitCode = 1;

0 commit comments

Comments
 (0)