Skip to content

Commit c544737

Browse files
chaxuschaxus
committed
test(agent): freeze original writing policy placement contrast
1 parent ef97ba8 commit c544737

2 files changed

Lines changed: 221 additions & 0 deletions

File tree

‎docs/evaluations/2026-10-07-qwen3-instruct2507-policy-placement-native.mjs‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
import { chromium } from '@playwright/test';
2+
import fs from 'node:fs/promises';
3+
import { createReadStream } from 'node:fs';
4+
import { createHash } from 'node:crypto';
5+
const modelPath = '.scratch/node_modules/ai-models/Qwen3-4B-Instruct-2507-Q4_K_M.gguf';
6+
const modelBytes = (await fs.stat(modelPath)).size;
7+
if (modelBytes !== 2497280448) throw Error('Pinned model size mismatch; do not load');
8+
const hash = createHash('sha256');
9+
for await (const chunk of createReadStream(modelPath)) hash.update(chunk);
10+
const modelSHA256 = hash.digest('hex');
11+
if (modelSHA256 !== '8cdb57cbb880d313736a9bc4e3d3d2485f145b5e19cf33783746e753e82641fc')
12+
throw Error('Pinned model hash mismatch; do not load');
13+
const prompts = JSON.parse(await fs.readFile('docs/evaluations/2026-10-07-qwen3-instruct2507-seven-language-cases.json', 'utf8')).filter(row=>['zh-CN-summarize','en-translate','ja-translate'].includes(row.id));
14+
const report = {
15+
runtime: '9a0e3aa',
16+
build: 'core1791345713/vendorb6864850e7b3',
17+
protocol: '2026-10-07-qwen3-instruct2507-policy-placement-protocol.md',
18+
model: 'Qwen3-4B-Instruct-2507-Q4_K_M.gguf',
19+
modelBytes,
20+
modelSHA256,
21+
cases: [],
22+
};
23+
const browser = await chromium.launch({ channel: 'chromium' });
24+
report.browserVersion = browser.version();
25+
try {
26+
for (const prompt of prompts)
27+
for (const variant of ['product','policy-system']) {
28+
const row = { prompt, variant, errors: [], console: [], crashed: false };
29+
report.cases.push(row);
30+
const context = await browser.newContext({ serviceWorkers: 'block', viewport: { width: 1280, height: 900 } });
31+
await context.addInitScript(() => {
32+
if (window === top) {
33+
localStorage.setItem('agent-panel-provider', 'wllama');
34+
localStorage.removeItem('agent-panel-gguf-url');
35+
}
36+
});
37+
try {
38+
const page = await context.newPage();
39+
page.on('pageerror', (e) => row.errors.push(e.message));
40+
page.on('crash', () => {
41+
row.crashed = true;
42+
});
43+
page.on('console', (message) => {
44+
if (['error', 'warning'].includes(message.type()) && row.console.length < 100)
45+
row.console.push({ type: message.type(), text: message.text().slice(0, 2000) });
46+
});
47+
await page.goto('http://127.0.0.1:5193/editor?new=docx&agent=1&locale=zh-CN');
48+
await page.waitForFunction(
49+
() => {
50+
const e = document.querySelector('#app iframe')?.contentWindow.editor;
51+
return e?.isDocumentLoadComplete && e?.isLoadFullApi;
52+
},
53+
null,
54+
{ timeout: 90000 },
55+
);
56+
await page.evaluate(async()=>{
57+
const {Wllama}=await import('/assets/client-D5_UYdz1.js');const original=Wllama.prototype.loadModel;window.__loads=[];
58+
Wllama.prototype.loadModel=async function(files,options){const probe={requested:{n_threads:options.n_threads,n_ctx:options.n_ctx,n_gpu_layers:options.n_gpu_layers,reasoning:options.reasoning}};window.__loads.push(probe);const result=await original.call(this,files,options);probe.actualThreads=this.getNumThreads();probe.multithread=this.isMultithread();return result;};
59+
});
60+
await page.evaluate(() => window.__toggleAgentPanel());
61+
await page.locator('.agent-panel').waitFor();
62+
await page.locator('.agent-panel-settings-toggle').click();
63+
await page
64+
.locator('.agent-panel-gguf-files')
65+
.setInputFiles('.scratch/node_modules/ai-models/Qwen3-4B-Instruct-2507-Q4_K_M.gguf');
66+
await page.locator('.agent-panel-gguf-load').click();
67+
await page.waitForFunction(
68+
() => document.querySelector('.agent-panel-note')?.textContent?.includes('模型已加载'),
69+
null,
70+
{ timeout: 180000 },
71+
);
72+
row.loads = await page.evaluate(()=>window.__loads);
73+
row.modelNote = await page.locator('.agent-panel-note').textContent();
74+
row.modelStatus = await page.locator('.agent-model-status').textContent();
75+
await page.locator('.agent-panel-settings-toggle').click();
76+
await page.evaluate((text) => {
77+
const api = document.querySelector('#app iframe').contentWindow.Asc.editor;
78+
api.pluginMethod_InputText(text);
79+
}, prompt.source);
80+
await page.waitForFunction(source =>
81+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText() === source + '\r\n',
82+
prompt.source, { timeout: 90000 });
83+
await page.evaluate(() => document.querySelector('#app iframe').contentWindow.Asc.editor.asc_EditSelectAll());
84+
row.selected = await page.evaluate(() =>
85+
document.querySelector('#app iframe').contentWindow.Asc.editor.pluginMethod_GetSelectedText(),
86+
);
87+
if (row.selected !== prompt.source + '\r\n') throw Error('Native source selection mismatch; no inference allowed');
88+
await page.locator('.agent-writing-task').selectOption(prompt.task);
89+
if(prompt.task==='translate') await page.locator('.agent-writing-language').selectOption(prompt.targetLanguage);
90+
row.documentBefore = await page.evaluate(() =>
91+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText(),
92+
);
93+
await page.evaluate(
94+
async variant => {
95+
const { Wllama } = await import('/assets/client-D5_UYdz1.js');
96+
const adjust = (request) => {
97+
const adjusted={...request,seed:42,temperature:0.7,top_p:0.8,top_k:20};
98+
if(variant==='policy-system'){
99+
const users=request.messages.filter(m=>m.role==='user');
100+
if(users.length!==1)throw Error('Expected one writing user message');
101+
const content=users[0].content;const split=content.lastIndexOf('\n');
102+
if(split<0)throw Error('Missing original task JSON boundary');
103+
const taskJSON=content.slice(split+1);JSON.parse(taskJSON);
104+
adjusted.messages=[{role:'system',content:content.slice(0,split)},{role:'user',content:taskJSON}];
105+
}
106+
return adjusted;
107+
};
108+
const count = Wllama.prototype.countChatTokens;
109+
window.__counts = [];
110+
Wllama.prototype.countChatTokens = function (request) {
111+
request = adjust(request);
112+
const { abortSignal: _abortSignal, ...body } = request;
113+
window.__counts.push(JSON.parse(JSON.stringify(body)));
114+
return count.call(this, request);
115+
};
116+
const original = Wllama.prototype.createChatCompletion;
117+
window.__streamProbe = [];
118+
Wllama.prototype.createChatCompletion = function (request) {
119+
const originalMessages = structuredClone(request.messages);
120+
request = adjust(request);
121+
const { abortSignal: _abortSignal, ...rest } = request;
122+
const probe = { originalMessages, request: JSON.parse(JSON.stringify(rest)), chunks: [] };
123+
window.__streamProbe.push(probe);
124+
return Promise.resolve(original.call(this, request)).then((result) => {
125+
if (!result?.[Symbol.asyncIterator]) {
126+
probe.completion = result;
127+
return result;
128+
}
129+
return (async function* () {
130+
try {
131+
for await (const chunk of result) {
132+
probe.chunks.push(JSON.parse(JSON.stringify(chunk)));
133+
yield chunk;
134+
}
135+
probe.done = true;
136+
} catch (e) {
137+
probe.error = String(e);
138+
throw e;
139+
}
140+
})();
141+
});
142+
};
143+
},
144+
variant,
145+
);
146+
await page.locator('.cui-input').fill(prompt.instruction);
147+
const start = Date.now();
148+
await page.locator('.cui-input').focus();
149+
await page.locator('.cui-input').press('Enter');
150+
await page.locator('.cui-msg-user').filter({ hasText: prompt.instruction }).last().waitFor({ timeout: 10000 });
151+
await page.waitForFunction(() => !document.querySelector('.cui-input').disabled, null, { timeout: 240000 });
152+
row.durationMs = Date.now() - start;
153+
row.toolMessages = await page.locator('.cui-activity > div, .cui-activity summary > div').allTextContents();
154+
row.previewCount = await page.locator('.agent-plan-preview').count();
155+
row.chatErrors = await page.locator('.cui-msg-error').allTextContents();
156+
row.documentAfter = await page.evaluate(() =>
157+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText(),
158+
);
159+
row.documentUnchanged = row.documentBefore === row.documentAfter;
160+
if (!row.documentUnchanged) {
161+
await page.evaluate(() => document.querySelector('#app iframe').contentWindow.Asc.editor.Undo());
162+
row.afterUndo = await page.evaluate(() =>
163+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText(),
164+
);
165+
await page.evaluate(() => document.querySelector('#app iframe').contentWindow.Asc.editor.Redo());
166+
row.afterRedo = await page.evaluate(() =>
167+
document.querySelector('#app iframe').contentWindow.Asc.editor.WordControl.m_oLogicDocument.GetText(),
168+
);
169+
}
170+
row.sdk = await page.evaluate(() => window.__streamProbe);
171+
row.counts = await page.evaluate(() => window.__counts);
172+
if (row.sdk.length !== 1 || row.sdk[0].request.temperature !== 0.7) throw Error('Unexpected bounded CPU request');
173+
if (JSON.stringify(row.counts.at(-1)) !== JSON.stringify(row.sdk[0].request))
174+
throw Error('Count/completion mismatch');
175+
row.finished = true;
176+
} catch (e) {
177+
row.error = String(e);
178+
const page = context.pages()[0];
179+
if (page && !page.isClosed()) {
180+
row.failureSnapshot = await page.evaluate(()=>({loads:window.__loads,sdk:window.__streamProbe,counts:window.__counts,documentText:document.querySelector('#app iframe')?.contentWindow?.Asc?.editor?.WordControl?.m_oLogicDocument?.GetText?.(),chatErrors:[...document.querySelectorAll('.cui-msg-error')].map(e=>e.textContent)})).catch(e=>({error:String(e)}));
181+
row.failureNote = await page
182+
.locator('.agent-panel-note')
183+
.textContent()
184+
.catch(() => null);
185+
row.failureStatus = await page
186+
.locator('.agent-model-status')
187+
.textContent()
188+
.catch(() => null);
189+
}
190+
} finally {
191+
await context.close();
192+
row.contextClosed = true;
193+
await fs.writeFile('.scratch/2026-10-07-qwen3-instruct2507-policy-placement-native.json', JSON.stringify(report, null, 2));
194+
console.log(
195+
JSON.stringify({
196+
variant: row.variant,
197+
source: row.prompt.source,
198+
body: row.documentAfter,
199+
errors: row.chatErrors,
200+
error: row.error,
201+
finished: row.finished,
202+
closed: row.contextClosed,
203+
}),
204+
);
205+
}
206+
if (!row.modelStatus) throw Error('Initial model setup failed; stop duplicate loading attempts');
207+
}
208+
} finally {
209+
await browser.close();
210+
report.browserClosed = true;
211+
await fs.writeFile('.scratch/2026-10-07-qwen3-instruct2507-policy-placement-native.json', JSON.stringify(report, null, 2));
212+
}
213+
214+
if (report.cases.some((row) => !row.finished || row.error || !row.contextClosed)) process.exitCode = 1;

‎docs/evaluations/2026-10-07-qwen3-instruct2507-policy-placement-protocol.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Frozen original-policy placement diagnosis
2+
3+
After the original seven-language process is confirmed terminal, run three known failures (Chinese summary, English-to-Japanese translation and Japanese-to-Korean translation) with product then policy-system framing, fresh context/model per row. The exact task JSON, original writing preamble wording, selected source, pinned GGUF, matched browser CPU runtime, schema, temperature0.7/top_p0.8/top_k20/seed42/max512 remain unchanged. The candidate uses the original user-message preamble verbatim as the system content and the original final JSON line verbatim as the user message, replacing the generic chat system message. No new writing rules or examples are added. Apply the same transformation to counting and completion.
4+
5+
This changes role placement and removes generic chat framing together, so it does not isolate one system sentence. It is a known-source diagnosis, not untouched transfer or a quality acceptance screen. Record original and actual messages, raw output, exact source, actual threads, refusal/native application/history, failures and closure. Preserve the entire product result and do not tune or retry after observing candidate outputs. Do not start concurrently with the original model process. No production prompt/API/model/default changes occur.
6+
7+
Check Chinese summary for source language, one shorter sentence, all actors/date/quantity/conditions/current negatives; translations for every source fact, unchanged name spelling and ISO dates, correct target language and no new claims. Even improvements require a separately frozen unused-source seven-language transfer and device/latency/native lifecycle acceptance before adoption. A refusal is not task completion; extractive-only output is not a replacement for generative writing.

0 commit comments

Comments
 (0)