-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathchatEngine.js
More file actions
2991 lines (2783 loc) · 160 KB
/
chatEngine.js
File metadata and controls
2991 lines (2783 loc) · 160 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
'use strict';
const EventEmitter = require('events');
const path = require('path');
const fs = require('fs');
const os = require('os');
const { pathToFileURL } = require('url');
const { parseToolCalls, repairToolCalls, stripToolCallText } = require('./tools/toolParser');
const { visionServer } = require('./visionServer');
const { detectFamily, detectFamilyFromArch, detectParamSize } = require('./modelDetection');
const { getModelProfile } = require('./modelProfiles');
/** Base max chars of each tool result injected into chat history.
* Actual cap is computed at runtime proportional to the model's context size
* (Rule 9: no hardcoded context numbers) so it works for any model from 2K to 128K context.
*/
const BASE_TOOL_RESULT_INJECT_CHARS = 32000;
// PL2: Per-tool-type multipliers (fraction of context-based cap).
// Browser tools need more because snapshots contain both element refs and page text.
// File/web tools need less because the model can re-read or re-fetch.
const TOOL_INJECT_MULTIPLIERS = {
browser_snapshot: 1.5, browser_navigate: 1.5, browser_click: 1.5,
browser_type: 1.5, browser_screenshot: 0.5,
read_file: 0.5, fetch_webpage: 0.5, web_search: 0.25,
};
// Pre-compiled regex patterns for the streaming tool call filter.
// These are tested on line boundaries only — never on every character.
const RE_FENCE_HEADER = /^```\s*(\w*)\r?\n/;
// Tightened: only match tool-call schema, not arbitrary JSON with "tool" or "name" keys
const RE_TOOL_KEY = /"tool"\s*:\s*"[a-zA-Z0-9_]+"/; // {"tool":"write_file"} or {"tool":"vscode_askQuestions"} — requires string value
const RE_NAME_KEY = /"name"\s*:\s*"[a-zA-Z0-9_]+(?:_[a-zA-Z0-9_]+)*"/; // {"name":"read_file"} or mixed-case
const RE_PARAMS_KEY = /"(?:params|parameters|arguments)"\s*:\s*\{/; // {"params":{ — strong tool-call signal
const RE_FILE_WRITE_TOOLS = /write_file|create_file|append_to_file/;
const RE_CONTENT_START = /"content"\s*:\s*"$/;
const RE_FILE_PATH = /"(?:filePath|path)"\s*:\s*"([^"]*)"/;
const RE_TOOL_OR_SYSTEM_INJECT = /^\[(?:Tool Results|System)\]/i;
/** Web-facing tools — if these share a batch with workspace navigation tools, drop the latter (structural conflict rule). */
const WEB_TOOL_BATCH = new Set(['web_search', 'fetch_webpage', 'http_request']);
const WORKSPACE_NAV_TOOLS = new Set(['list_directory', 'get_project_structure']);
function filterWebWorkspaceToolConflict(calls) {
if (!calls?.length) return calls;
const hasWeb = calls.some((c) => WEB_TOOL_BATCH.has(c.tool));
const hasWs = calls.some((c) => WORKSPACE_NAV_TOOLS.has(c.tool));
if (!hasWeb || !hasWs) return calls;
const dropped = calls.filter((c) => WORKSPACE_NAV_TOOLS.has(c.tool));
const kept = calls.filter((c) => !WORKSPACE_NAV_TOOLS.has(c.tool));
console.log(
`[ChatEngine] Same-batch conflict: dropped workspace tool(s) ${dropped.map((d) => d.tool).join(', ')} because web tool(s) are present`,
);
return kept;
}
/** Hard floor for context window (aligned with llama.cpp 256-token alignment). */
const MIN_CONTEXT_FLOOR = 2048;
/** When requireMinContextForGpu is true, prefer at least this before accepting GPU offload. */
const MIN_CONTEXT_WHEN_GPU_REQUIRED = 4096;
/**
* Normalize settings / IPC payload for model load. Matches settingsManager defaults when fields are missing.
* @param {object} raw
* @returns {{ gpuPreference: 'auto'|'cpu', gpuLayers: number, contextSize: number, requireMinContextForGpu: boolean }}
*/
/**
* Absolute floor for the computed hardware context cap when GGUF metadata is
* missing (so we cannot compute a KV-derived cap). Equal to the llama.cpp-aligned
* minimum — used only as a last resort; the normal path computes from architecture.
*/
const CONTEXT_MAX_FALLBACK_NO_GGUF_FLOOR = 2048;
/**
* Estimate KV-cache bytes per token from GGUF architecture metadata.
* Transformer-standard, architecture-agnostic:
* KV per token = n_layer * n_head_kv * (key_length + value_length) * bytes_per_element
* Assumes fp16 KV cache (2 bytes/element), the llama.cpp default.
*
* GGUF metadata shape (llama.cpp convention):
* architectureMetadata.block_count → n_layer
* architectureMetadata.attention.head_count_kv → n_head_kv
* architectureMetadata.attention.head_count → n_head (fallback)
* architectureMetadata.attention.key_length → head_dim_k
* architectureMetadata.attention.value_length → head_dim_v
* architectureMetadata.embedding_length → embedding dim (fallback)
*/
function estimateKvBytesPerToken(am, kvCacheType) {
if (!am) return null;
const nLayer = am.block_count;
if (!nLayer) return null;
const att = am.attention || {};
const nHeadKv = att.head_count_kv || att.head_count;
if (!nHeadKv) return null;
let keyLen = att.key_length;
let valLen = att.value_length;
// Fallback: derive head_dim from embedding_length / head_count if per-head lengths missing
if ((!keyLen || !valLen) && am.embedding_length && att.head_count) {
const headDim = am.embedding_length / att.head_count;
if (Number.isFinite(headDim) && headDim > 0) {
if (!keyLen) keyLen = headDim;
if (!valLen) valLen = headDim;
}
}
if (!keyLen || !valLen) return null;
// Bytes per element depends on KV cache quantization type:
// f16 = 2 bytes/element (no compression)
// q8_0 = 1 byte/element (2x compression)
// q4_0 = 0.5 bytes/element (4x compression)
const bytesPerElement = kvCacheType === 'q3_0' ? 0.375
: kvCacheType === 'q4_0' ? 0.5
: kvCacheType === 'q4_1' ? 0.5625
: kvCacheType === 'q5_0' ? 0.625
: kvCacheType === 'q5_1' ? 0.6875
: kvCacheType === 'q8_0' ? 1
: 2; // f16 or default
return nLayer * nHeadKv * (keyLen + valLen) * bytesPerElement;
}
function buildEngineLoadSettings(raw = {}) {
const gpuPreference = raw.gpuPreference === 'cpu' ? 'cpu' : 'auto';
const gpuLayers = typeof raw.gpuLayers === 'number' ? raw.gpuLayers : -1;
const ctx = Number(raw.contextSize);
// 0 = auto — use model train cap (and VRAM) as upper bound, not a fixed 16k default
const contextSize = !Number.isFinite(ctx) || ctx < 0 ? 0 : Math.floor(ctx);
return {
gpuPreference,
gpuLayers,
contextSize,
requireMinContextForGpu: !!raw.requireMinContextForGpu,
gpuConstrainedContext: raw.gpuConstrainedContext !== false, // default true
kvCacheType: raw.kvCacheType || 'f16',
enableThinking: raw.enableThinking !== false, // default true
};
}
// Agent identity prompt — primacy-ordered: identity → format → examples → rubric → rules
const SYSTEM_PROMPT = `You are guIDE, an AI agent with tools for file editing, browsing, terminal commands, and more.
## When to use tools vs. prose
- If text alone fully addresses the user's request, respond with prose.
- When the user asks for action you cannot accomplish through text alone, use the appropriate tool.
- You have real tools — never say "I cannot," "I don't have access," or "I'm unable to" when action is needed. Use the tools instead.
## Tool-call format
Output a fenced JSON block with "tool" and "params" keys:
\`\`\`json
{"tool":"<TOOL_NAME>","params":{<PARAMETERS>}}
\`\`\`
Never output raw JSON, braces, or backticks outside a fenced block. If not calling a tool, write normal prose with no JSON syntax. After a tool call, wait for the result before continuing.
## Tool-call patterns
These are FORMAT patterns — substitute your own values for ANGLE_BRACKETS.
List directory:
\`\`\`json
{"tool":"list_directory","params":{"path":"<DIRECTORY_PATH>"}}
\`\`\`
Read file:
\`\`\`json
{"tool":"read_file","params":{"filePath":"<FILE_PATH>"}}
\`\`\`
Create file:
\`\`\`json
{"tool":"write_file","params":{"filePath":"<FILE_PATH>","content":"<FILE_CONTENT>"}}
\`\`\`
Edit file (exact string replacement):
\`\`\`json
{"tool":"edit_file","params":{"filePath":"<FILE_PATH>","old_string":"<EXACT_OLD_TEXT>","new_string":"<NEW_TEXT>"}}
\`\`\`
Append to file:
\`\`\`json
{"tool":"append_to_file","params":{"filePath":"<FILE_PATH>","content":"<TEXT_TO_APPEND>"}}
\`\`\`
Web search (always follow with fetch_webpage):
\`\`\`json
{"tool":"web_search","params":{"query":"<SEARCH_QUERY>"}}
\`\`\`
\`\`\`json
{"tool":"fetch_webpage","params":{"url":"<TOP_RESULT_URL>"}}
\`\`\`
Browser (always snapshot before click/type/fill — refs expire after each snapshot):
\`\`\`json
{"tool":"browser_navigate","params":{"url":"<TARGET_URL>"}}
\`\`\`
\`\`\`json
{"tool":"browser_snapshot","params":{}}
\`\`\`
\`\`\`json
{"tool":"browser_click","params":{"ref":"<REF_FROM_SNAPSHOT>","element":"<DESCRIPTION>"}}
\`\`\`
Shell command:
\`\`\`json
{"tool":"run_command","params":{"command":"<SHELL_COMMAND>"}}
\`\`\`
Todo list:
\`\`\`json
{"tool":"write_todos","params":{"items":["<STEP_1>","<STEP_2>"]}}
\`\`\`
\`\`\`json
{"tool":"update_todo","params":{"id":"<TODO_ID>","status":"completed"}}
\`\`\`
Ask user:
\`\`\`json
{"tool":"ask_question","params":{"question":"<YOUR_QUESTION>","options":[{"label":"<OPTION>","description":"<DESC>"}]}}
\`\`\`
## Available tools
File: read_file, write_file, edit_file, append_to_file, delete_file, rename_file, copy_file, get_file_info
Search: list_directory, find_files, search_codebase, grep_search, search_in_file, replace_in_files
Browser: browser_navigate, browser_snapshot, browser_click, browser_type, browser_fill_form, browser_select_option, browser_screenshot, browser_get_content, browser_evaluate, browser_scroll, browser_wait, browser_wait_for, browser_back, browser_press_key, browser_hover, browser_drag, browser_tabs, browser_handle_dialog, browser_console_messages, browser_file_upload, browser_resize, browser_get_url, browser_get_links, browser_close
Terminal: run_command, get_project_structure, create_directory, analyze_error, install_packages
Web: web_search, fetch_webpage, http_request, check_port
Git: git_status, git_commit, git_diff, git_log, git_branch, git_stash, git_reset
Memory: save_memory, get_memory, list_memories
Undo: undo_edit, list_undoable
Planning: write_todos, update_todo, ask_question
Scratchpad: write_scratchpad, read_scratchpad
Other: open_file_in_editor, generate_image, diff_files, save_rule, list_rules
## Decision rubric
Internet topics (live info, products, news, docs): web_search → fetch_webpage. Do not use file tools for internet topics.
Project files (code in workspace): read_file (known path), grep_search/search_codebase (unknown location), list_directory (folder contents). Do not use web_search for project files.
File changes: edit_file for existing files, write_file for new files, append_to_file for appending. Never paste file contents into chat as a substitute.
Shell/OS: run_command. Does not control a browser — never use curl/wget as a substitute for browser tools.
Browser: browser_navigate → browser_snapshot → browser_click/type/fill. Snapshot before every interaction — refs expire.
Planning: write_todos for multi-step work, ask_question when you need clarification or a user decision.
Batching: Multiple independent tool calls may go in one response.
## Vision
Images are automatically captioned. When you receive an image description, that is the image content — you have seen it. Do not call read_file on image files.
## Operating rules
- Do not repeat the same tool call with the same arguments. If a call fails, adjust arguments; if it still fails after one retry, ask the user.
- After a tool returns, use its result and continue. Do not re-ask for information the tool provided.
- Ground web answers in fetched page content, not training memory.
- If output is truncated, continue from the interruption point. Do not restart.
- If stuck or uncertain, call ask_question. Never guess — guessing causes errors.
- When the user provides information via ask_question, it is part of your context. Use it immediately.
- Ground every claim in evidence. Agreement without evidence is dishonest.
- Never emit [Tool Results] or [System: Tool Results] blocks. The system injects real tool results after you call a tool. Only emit tool calls and prose — never fabricate tool output.`;
class ChatEngine extends EventEmitter {
constructor() {
super();
console.log('[ChatEngine] constructor START');
this.isReady = false;
this.isLoading = false;
this.modelInfo = null;
this.currentModelPath = null;
this.gpuPreference = 'auto';
this._projectPath = null;
this._llama = null;
this._model = null;
this._context = null;
this._sequence = null;
this._chat = null;
this._chatHistory = [];
this._lastEvaluation = null;
this._abortController = null;
this._pendingUserMessage = null; // injected by user interrupt during tool loop
this._recentlyWrittenFiles = new Map(); // filePath → content written in current chat() call
this._sessionId = 0; // increments on resetSession to detect stale tool results
console.log('[ChatEngine] constructor DONE');
}
/**
* @param {string} modelPath
* @param {object} [rawLoadSettings] — from settingsManager.get() (gpuPreference, gpuLayers, contextSize, requireMinContextForGpu)
*/
async initialize(modelPath, rawLoadSettings) {
// If a load is already in progress, wait for it to finish, then proceed with this new request.
// This prevents "Already loading a model" errors when the user switches models quickly.
if (this._loadPromise) {
console.log('[ChatEngine] Model load already in progress — queuing new request');
try { await this._loadPromise; } catch (_) { /* previous load failed — proceed */ }
}
// Build a new promise for this load so subsequent requests can await it
this._loadPromise = this._doInitialize(modelPath, rawLoadSettings);
try {
return await this._loadPromise;
} finally {
this._loadPromise = null;
}
}
async _doInitialize(modelPath, rawLoadSettings) {
this.isLoading = true;
this.emit('status', { state: 'loading', message: 'Loading model...' });
try {
const llamaCppPath = this._getNodeLlamaCppPath();
const { getLlama, LlamaChat, readGgufFileInfo } = await import(pathToFileURL(llamaCppPath).href);
const s = buildEngineLoadSettings(rawLoadSettings || {});
this.gpuPreference = s.gpuPreference;
if (this._model) await this._dispose();
let trainMaxContext = null;
let totalLayersFromGguf = null;
let ggufArchMeta = null;
let ggufArchString = null; // metadata.general.architecture (e.g. "chatglm", "qwen35", "gemma3")
let ggufChatTemplate = null; // tokenizer.chat_template — Jinja template string from GGUF metadata
try {
const gguf = await readGgufFileInfo(modelPath, { readTensorInfo: false, logWarnings: false });
const am = gguf.architectureMetadata;
ggufArchMeta = am || null;
if (gguf.metadata?.general?.architecture) ggufArchString = gguf.metadata.general.architecture;
if (am && typeof am.context_length === 'number') trainMaxContext = am.context_length;
if (am && typeof am.block_count === 'number') totalLayersFromGguf = am.block_count;
// Tier 1: Extract chat template from GGUF metadata for auto-detection.
// tokenizer.chat_template is a Jinja2 string embedded in the GGUF file.
// It's the authoritative source for what the model's chat format supports.
// Models with thinking capability include `enable_thinking` as a Jinja variable.
if (gguf.metadata?.tokenizer?.chat_template) {
ggufChatTemplate = gguf.metadata.tokenizer.chat_template;
}
} catch (e) {
console.warn(`[ChatEngine] readGgufFileInfo: ${e.message}`);
}
// Initialize llama runtime early so we can query VRAM state before context sizing.
this._llama = await getLlama({
gpu: s.gpuPreference === 'cpu' ? false : 'auto',
});
const modelStats = fs.statSync(modelPath);
// ─── Hardware-aware context ceiling computation ───
// Compute the maximum context window that the user's hardware can realistically support,
// derived from GGUF architecture + available memory. No hardcoded ceilings.
const kvBytesPerToken = estimateKvBytesPerToken(ggufArchMeta, s.kvCacheType);
let hardwareCap = null;
let kvSourceMem = 'none';
let vramTotal = 0;
let vramFree = 0;
// Diagnostic: log raw GGUF architecture metadata for KV estimate verification
if (ggufArchMeta) {
const att = ggufArchMeta.attention || {};
console.log(`[ChatEngine] GGUF arch metadata: block_count=${ggufArchMeta.block_count}, head_count_kv=${att.head_count_kv}, head_count=${att.head_count}, key_length=${att.key_length}, value_length=${att.value_length}, embedding_length=${ggufArchMeta.embedding_length}, kvCacheType=${s.kvCacheType}, kvBytesPerToken=${kvBytesPerToken}`);
}
if (kvBytesPerToken) {
// Compute hwCap from both VRAM and RAM, pick whichever gives larger context.
// Previously we preferred VRAM whenever possible, but for small models that
// nearly fill VRAM, this gave tiny contexts (e.g. 12K) when RAM would give 32K+.
let vramHwCap = null;
let ramHwCap = null;
if (s.gpuPreference !== 'cpu') {
try {
const vramState = await this._llama.getVramState();
vramTotal = vramState?.total || 0;
vramFree = vramState?.free || 0;
} catch (e) { console.warn('[ChatEngine] VRAM state query failed:', e.message); }
}
// VRAM path: free VRAM minus model weights, half reserved for KV
if (vramFree > modelStats.size) {
const vramAvail = vramFree - modelStats.size;
const vramKvBudget = vramAvail / 2;
vramHwCap = Math.max(MIN_CONTEXT_FLOOR, Math.floor(vramKvBudget / kvBytesPerToken));
}
// RAM path. useMmap=true (line 445) means model weights are
// memory-mapped and shared with the OS page cache, NOT consuming
// anonymous RAM upfront. Deriving from os.freemem() and subtracting
// modelStats.size double-counts on every OS with mmap and collapses
// to MIN_CONTEXT_FLOOR when free memory is tight at load time.
// Use os.totalmem() (deterministic upper bound, independent of
// transient memory state) minus the model file size minus a fixed
// runtime overhead for activations, the JS/Electron heap, sub-agent
// context, and OS slack. A precise computation would need gpuLayers,
// which is computed AFTER hardwareCap (chicken-and-egg) — fixed
// constant is the right tradeoff here.
const RAM_RUNTIME_OVERHEAD = 1.5 * 1024 * 1024 * 1024;
const ramAvail = Math.max(0, os.totalmem() - modelStats.size - RAM_RUNTIME_OVERHEAD);
const ramKvBudget = ramAvail / 2;
ramHwCap = Math.max(MIN_CONTEXT_FLOOR, Math.floor(ramKvBudget / kvBytesPerToken));
// Pick whichever source gives the larger context
if (vramHwCap != null && vramHwCap >= ramHwCap) {
hardwareCap = vramHwCap;
kvSourceMem = 'vram';
} else {
hardwareCap = ramHwCap;
kvSourceMem = 'ram';
}
console.log(`[ChatEngine] Memory diagnostic: vramTotal=${(vramTotal/1e9).toFixed(2)}GB, vramFree=${(vramFree/1e9).toFixed(2)}GB, freeRam=${(os.freemem()/1e9).toFixed(2)}GB, modelSize=${(modelStats.size/1e9).toFixed(2)}GB, vramHwCap=${vramHwCap}, ramHwCap=${ramHwCap}, source=${kvSourceMem}, hwCap=${hardwareCap}`);
}
const testMaxCtx = parseInt(process.env.TEST_MAX_CONTEXT, 10) || 0;
let desiredMax;
if (testMaxCtx > 0) {
desiredMax = testMaxCtx;
} else if (s.contextSize <= 0) {
// P4: Auto sizing uses only hardware cap and train cap (no hardcoded ceiling).
// Per RULES.md Rule 9: never hardcode context numbers — compute from actual resources.
// The hardware cap already accounts for VRAM/RAM headroom, so it self-limits to safe sizes.
if (hardwareCap != null && trainMaxContext != null) {
desiredMax = Math.min(hardwareCap, trainMaxContext);
} else if (hardwareCap != null) {
desiredMax = hardwareCap;
} else if (trainMaxContext != null) {
desiredMax = trainMaxContext;
} else {
desiredMax = CONTEXT_MAX_FALLBACK_NO_GGUF_FLOOR;
}
} else {
desiredMax = s.contextSize;
if (trainMaxContext != null) desiredMax = Math.min(desiredMax, trainMaxContext);
desiredMax = Math.max(MIN_CONTEXT_FLOOR, desiredMax);
}
const minBase = s.requireMinContextForGpu ? MIN_CONTEXT_WHEN_GPU_REQUIRED : MIN_CONTEXT_FLOOR;
const contextMin = Math.min(minBase, desiredMax);
console.log(
`[ChatEngine] Context sizing: train=${trainMaxContext}, hwCap=${hardwareCap} (source=${kvSourceMem}, kvBytesPerToken=${kvBytesPerToken}), user=${s.contextSize <= 0 ? 'auto' : s.contextSize}, test=${testMaxCtx || 'none'} → desiredMax=${desiredMax}`,
);
// Pre-flight validation: catch undefined variables before they cause runtime errors
const _preflightVars = { kvBytesPerToken, hardwareCap, kvSourceMem, vramTotal, vramFree, trainMaxContext, totalLayersFromGguf, ggufArchMeta, desiredMax, contextMin, modelStats };
for (const [name, val] of Object.entries(_preflightVars)) {
if (val === undefined) {
throw new Error(`Internal error: ${name} is undefined at loadModel. This is a bug — please report.`);
}
}
const loadModelOpts = {
modelPath,
defaultContextFlashAttention: s.gpuPreference !== 'cpu',
ignoreMemorySafetyChecks: true,
useMmap: true,
// Lock model pages in RAM to prevent OS from swapping them to disk (causes stalls)
useMlock: os.totalmem() > modelStats.size * 2,
onLoadProgress: (p) => {
this.emit('status', { state: 'loading', message: `Loading model... ${Math.round(p * 100)}%`, progress: p });
},
};
// KV cache quantization: resolve BEFORE loadModel so we can adjust fitContext.
// Default 'f16' matches llama.cpp upstream and enables the fastest fused flash-attention
// kernel path on consumer NVIDIA GPUs (this is what LM Studio / llama-server use). Lower
// precision options (q8_0, q4_0, q4_1, q5_0, q5_1, q3_0) are user-selectable for trading
// VRAM headroom against generation speed. 'currentQuant' lets node-llama-cpp match the
// model's weight quantization (legacy behaviour, retained for explicit selection).
const ALLOWED_KV_TYPES = new Set(['currentQuant', 'q3_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'q8_0', 'f16']);
const rawKvType = rawLoadSettings.kvCacheType || 'f16';
const kvCacheType = ALLOWED_KV_TYPES.has(rawKvType) ? rawKvType : undefined;
if (s.gpuPreference === 'cpu') {
loadModelOpts.gpuLayers = 0;
} else if (s.gpuLayers >= 0) {
loadModelOpts.gpuLayers = s.gpuLayers;
} else {
// Compute GPU layers iteratively.
// llama.cpp allocates the KV cache on the same device as each model layer,
// so the GPU-proportional portion of the KV cache must fit in VRAM alongside
// the model weights. This creates a circular dependency: gpuLayers depends on
// kvOnGpu which depends on gpuLayers. We solve it by iterating until stable.
//
// CRITICAL: We reserve VRAM for MIN_VIABLE_CONTEXT (4096 tokens) of KV cache,
// NOT the full desiredMax. Using desiredMax (e.g. 112K tokens = 14GB KV) when
// VRAM is only 3.45GB causes the iteration to oscillate between 0 and max layers
// and never converge. After model load, the actual context size is computed from
// remaining VRAM — this reservation just ensures we don't starve the KV cache.
const totalLayers = totalLayersFromGguf || 32;
const bytesPerLayer = modelStats.size / totalLayers;
const MIN_VIABLE_CONTEXT = 4096;
const kvBytesForMinCtx = (kvBytesPerToken || 0) * MIN_VIABLE_CONTEXT;
const vramOverhead = 512 * 1024 * 1024; // 512MB for activations/buffers
let computedGpuLayers = totalLayers; // start optimistic
for (let i = 0; i < 10; i++) {
const kvOnGpu = kvBytesForMinCtx * (computedGpuLayers / totalLayers);
const availableForModel = vramFree - kvOnGpu - vramOverhead;
const newGpuLayers = availableForModel > 0
? Math.max(0, Math.min(Math.floor(availableForModel / bytesPerLayer), totalLayers))
: 0;
if (newGpuLayers === computedGpuLayers) break; // converged
computedGpuLayers = newGpuLayers;
}
const kvOnGpuFinal = kvBytesForMinCtx * (computedGpuLayers / totalLayers);
console.log(`[ChatEngine] GPU layer computation: vramFree=${(vramFree/1e9).toFixed(2)}GB, kvMinCtx=${(kvBytesForMinCtx/1e6).toFixed(1)}MB, kvOnGpu=${(kvOnGpuFinal/1e6).toFixed(1)}MB, overhead=0.50GB, bytesPerLayer=${(bytesPerLayer/1e6).toFixed(1)}MB, gpuLayers=${computedGpuLayers}/${totalLayers}`);
loadModelOpts.gpuLayers = computedGpuLayers;
}
this._model = await this._llama.loadModel(loadModelOpts);
// P1: Use node-llama-cpp's cpuMathCores (probed by llama.cpp from actual CPU topology)
// instead of os.cpus().length / 2 which was wrong on:
// - AMD Ryzen without SMT (formula halved usable cores)
// - Apple Silicon (no hyperthreading — formula halved usable cores)
// - Intel with E-cores (formula misclassified E-cores)
// Falls back to os.cpus().length if cpuMathCores is missing or zero.
const cpuMathCores = (typeof this._llama.cpuMathCores === 'number' && this._llama.cpuMathCores > 0)
? this._llama.cpuMathCores
: os.cpus().length;
const threadCount = Math.max(1, cpuMathCores);
// P2: Adaptive batchSize from actual VRAM headroom after model load.
// Larger batchSize = faster prompt processing (prefill). Output tok/s unchanged.
// Measured free VRAM determines safe upper bound. CPU stays conservative.
let vramFreeAfterModel = vramFree;
if (s.gpuPreference !== 'cpu') {
try {
const vs = await this._llama.getVramState();
vramFreeAfterModel = vs?.free || vramFree;
} catch (e) { console.warn('[ChatEngine] VRAM query (batchSize) failed:', e.message); }
}
let batchSize;
if (s.gpuPreference === 'cpu') batchSize = 512;
else if (vramFreeAfterModel < 2 * 1024 * 1024 * 1024) batchSize = 1024;
else if (vramFreeAfterModel < 4 * 1024 * 1024 * 1024) batchSize = 2048;
else batchSize = 4096;
console.log(`[ChatEngine] Perf: cpuMathCores=${cpuMathCores} (threads=${threadCount}), vramFreeAfterModel=${(vramFreeAfterModel/1e9).toFixed(2)}GB, batchSize=${batchSize}`);
// P5: SWA models (Gemma, Mistral with sliding window) — set swaFullCache to enable
// prefix reuse on multi-turn chats. Without this, multi-turn re-evaluates entire history.
const swaSize = this._model?.fileInsights?.swaSize || 0;
const swaFullCache = swaSize > 0;
if (swaFullCache) console.log(`[ChatEngine] P5: SWA detected (swaSize=${swaSize}) — swaFullCache enabled for prefix reuse`);
// Compute exact context size after model loading.
// node-llama-cpp's createContext with { min, max } uses f16 for KV estimation,
// which inflates 4x when using q4_0, causing all retries to fail and collapse
// to the minimum (2048). We bypass this by computing the exact size ourselves
// using actual post-load VRAM measurements and q4_0-aware KV estimates,
// then passing a single number to createContext.
const actualGpuLayers = this._model.gpuLayers || 0;
const totalLayersForCtx = totalLayersFromGguf || 32;
const gpuLayerRatio = totalLayersForCtx > 0 ? actualGpuLayers / totalLayersForCtx : 0;
let computedCtxSize = desiredMax;
if (kvBytesPerToken > 0 && gpuLayerRatio > 0 && s.gpuPreference !== 'cpu') {
// vramFreeAfterModel was already measured above (P2 batchSize computation)
// KV on GPU = kvBytesPerToken * contextSize * gpuLayerRatio
// Available for KV = vramFreeAfterModel - buffer
// contextSize = available / (kvBytesPerToken * gpuLayerRatio)
const vramBuffer = 256 * 1024 * 1024; // 256MB safety buffer
const maxCtxFromVram = Math.floor((vramFreeAfterModel - vramBuffer) / (kvBytesPerToken * gpuLayerRatio));
computedCtxSize = Math.max(MIN_CONTEXT_FLOOR, Math.min(maxCtxFromVram, desiredMax));
console.log(`[ChatEngine] Context size computation: vramFreeAfterModel=${(vramFreeAfterModel/1e9).toFixed(2)}GB, kvBpt=${kvBytesPerToken}, gpuRatio=${gpuLayerRatio.toFixed(2)}, maxCtxFromVram=${maxCtxFromVram}, computedCtxSize=${computedCtxSize}`);
// Fix D: GPU-constrained context cap. When only a small fraction of layers
// fit on GPU (gpuRatio < 0.3), the formula above allows massive context because
// only the GPU fraction of KV needs VRAM — but the remaining KV lives in system
// RAM, and every generation step traverses CPU layers with that massive KV cache,
// causing significant slowdown. Cap context to what fits in VRAM for the full KV
// (not just the GPU fraction) to keep generation fast. Toggleable via setting.
if (s.gpuConstrainedContext && gpuLayerRatio < 0.3 && kvBytesPerToken > 0) {
const vramBoundedCtx = Math.floor((vramFreeAfterModel - vramBuffer) / kvBytesPerToken);
if (vramBoundedCtx > 0 && vramBoundedCtx < computedCtxSize) {
console.log(`[ChatEngine] Fix D: GPU-constrained context cap: ${vramBoundedCtx} (from ${computedCtxSize}) — gpuRatio=${gpuLayerRatio.toFixed(2)}, only ${actualGpuLayers}/${totalLayersForCtx} layers on GPU`);
computedCtxSize = Math.max(MIN_CONTEXT_FLOOR, vramBoundedCtx);
}
}
}
// Build createContext options. Only attach experimentalKvCacheKeyType / ValueType when
// the user has selected a non-f16 KV type. Passing 'f16' explicitly is supported by
// node-llama-cpp but bypasses the upstream-default code path; omitting the override lets
// llama.cpp pick the fastest fused flash-attention kernel for the model + GPU combination.
const baseCtxOpts = {
contextSize: computedCtxSize,
flashAttention: s.gpuPreference !== 'cpu',
ignoreMemorySafetyChecks: true,
batchSize,
threads: { ideal: threadCount, min: Math.max(1, threadCount - 1) },
swaFullCache,
};
if (kvCacheType && kvCacheType !== 'f16') {
baseCtxOpts.experimentalKvCacheKeyType = kvCacheType;
baseCtxOpts.experimentalKvCacheValueType = kvCacheType;
}
// Create context with computed single-number size (bypasses f16-based fitting)
try {
this._context = await this._model.createContext(baseCtxOpts);
} catch (ctxErr) {
// Fallback: if a quantised KV type isn't actually applied by llama.cpp for this model,
// the real KV size is f16 (up to 4x larger). Recompute with f16 estimate and retry.
// vramFreeAfterModel was already measured above; reuse it.
if (kvBytesPerToken > 0 && gpuLayerRatio > 0 && s.gpuPreference !== 'cpu') {
const kvBytesF16 = kvBytesPerToken * 4;
const vramBuffer = 256 * 1024 * 1024;
const maxCtxF16 = Math.floor((vramFreeAfterModel - vramBuffer) / (kvBytesF16 * gpuLayerRatio));
const fallbackCtxSize = Math.max(MIN_CONTEXT_FLOOR, Math.min(maxCtxF16, desiredMax));
console.warn(`[ChatEngine] Context creation at ${computedCtxSize} failed (${ctxErr.message}), retrying with f16 estimate: ${fallbackCtxSize}`);
const fallbackOpts = { ...baseCtxOpts, contextSize: fallbackCtxSize };
this._context = await this._model.createContext(fallbackOpts);
} else {
throw ctxErr;
}
}
// Diagnostic: verify context creation
const actualCtxSize = this._context?.contextSize;
console.log(`[ChatEngine] Context created: ctx=${actualCtxSize}, gpuLayers=${actualGpuLayers}, flashAttn=${s.gpuPreference !== 'cpu'}, batchSize=${batchSize}, threads=${threadCount}, swaFullCache=${swaFullCache}, kvCacheType=${kvCacheType || 'default'}, requestedSize=${computedCtxSize}`);
// Graceful context degradation: log exact reason and suggest action
const ctxDegradation = actualCtxSize < desiredMax * 0.8;
if (ctxDegradation) {
const ratio = actualCtxSize / desiredMax;
const reason = ratio < 0.1
? `Context collapsed to ${actualCtxSize} (${(ratio * 100).toFixed(1)}% of desired ${desiredMax}). Likely cause: GPU layers consumed too much VRAM, leaving none for KV cache.`
: `Context reduced to ${actualCtxSize} (${(ratio * 100).toFixed(1)}% of desired ${desiredMax}). GPU layers (${actualGpuLayers}) may be consuming VRAM needed for KV cache.`;
const suggestion = actualGpuLayers > 0
? `Try: reduce GPU layers in settings, or set a smaller context size manually.`
: `Try: increase context size in settings, or switch to a smaller model.`;
console.warn(`[ChatEngine] Context degradation: ${reason} ${suggestion}`);
}
this._sequence = this._context.getSequence();
const actualCtx = this._context.contextSize || 0;
// Estimate parameter count from GGUF metadata or filename (e.g. "Qwen3.5-4B" → 4B)
let paramCount = ggufArchMeta?.totalParameterCount || null;
if (!paramCount) {
const sizeMatch = path.basename(modelPath).match(/(\d+(?:\.\d+)?)\s*[Bb]/);
if (sizeMatch) paramCount = Math.round(parseFloat(sizeMatch[1]) * 1e9);
}
this.modelInfo = {
path: modelPath,
name: path.basename(modelPath),
size: modelStats.size,
parameterCount: paramCount,
contextSize: actualCtx,
contextSizeRequested: s.contextSize <= 0 ? 'auto' : s.contextSize,
contextSizeCap: desiredMax,
contextTrainMax: trainMaxContext,
contextHardwareCap: hardwareCap,
kvBytesPerToken: kvBytesPerToken,
kvMemSource: kvSourceMem,
totalLayers: totalLayersFromGguf != null ? totalLayersFromGguf : undefined,
gpuLayers: this._model.gpuLayers || 0,
gpuMode: s.gpuPreference === 'cpu' ? false : 'auto',
};
// ─── Three-Tier Model Profile Resolution ───
// Tier 1: GGUF metadata auto-detection (what the file tells us about itself)
// Tier 2: Vendor profile overrides (curated settings from model vendor documentation)
// Tier 3: BASE_DEFAULTS (neutral fallback when neither source provides info)
//
// This system is future-proof: a brand-new model with an unknown architecture
// string and unknown filename will still work correctly because Tier 1 detects
// capabilities directly from the GGUF file's embedded chat template, and Tier 3
// provides neutral defaults. Vendor profiles (Tier 2) are optional enhancements
// that optimize sampling for families with documented vendor recommendations.
// Tier 1: Auto-detect thinking support from the GGUF chat template.
// The Jinja chat template is embedded in every GGUF file under tokenizer.chat_template.
// If it contains `enable_thinking` as a variable, the model supports thinking mode.
// This is future-proof: any model that adds thinking will include this variable.
// No manual profile entry is required for thinking to work on new models.
const _templateSupportsThinking = ggufChatTemplate
? /enable_thinking/.test(ggufChatTemplate)
: false;
console.log(`[ChatEngine] Tier 1: chat_template auto-detect — templatePresent=${!!ggufChatTemplate}, supportsThinking=${_templateSupportsThinking}`);
// Tier 2: Family detection — GGUF architecture metadata (primary) then filename (fallback).
// GGUF metadata.general.architecture is authoritative (e.g. "chatglm", "qwen35", "gemma3").
// Filename detection is kept as fallback for corrupted/missing metadata.
const _archFamily = detectFamilyFromArch(ggufArchString);
const _fileFamily = detectFamily(modelPath);
const _detectedFamily = _archFamily || _fileFamily;
const _detectedSizeB = paramCount ? paramCount / 1e9 : detectParamSize(modelPath);
this._modelProfile = getModelProfile(_detectedFamily, _detectedSizeB);
// Merge Tier 1 auto-detection into the profile.
// If the chat template says the model supports thinking but the vendor profile
// says mode='none' (e.g. an old profile for a non-thinking variant), the template
// wins — it's the ground truth from the actual model file.
// If the vendor profile says mode='budget' but the template has no enable_thinking,
// the vendor profile wins — some models think without the template variable.
const _profileThinkMode = this._modelProfile?.thinkTokens?.mode;
if (_templateSupportsThinking && _profileThinkMode === 'none') {
// Template says thinking is supported but profile says none — template wins
this._modelProfile.thinkTokens.mode = 'budget';
this._modelProfile.thinkTokens.budget = this._modelProfile.thinkTokens.budget || 2048;
console.log(`[ChatEngine] Tier 1 override: chat_template supports thinking but profile said 'none' — upgraded to 'budget'`);
} else if (_templateSupportsThinking && _profileThinkMode !== 'budget' && _profileThinkMode !== 'strip') {
// Unknown model with template thinking support — activate budget mode
this._modelProfile.thinkTokens.mode = 'budget';
this._modelProfile.thinkTokens.budget = this._modelProfile.thinkTokens.budget || 2048;
console.log(`[ChatEngine] Tier 1 auto-detect: chat_template supports thinking — set thinkTokens.mode='budget'`);
}
// Pass enable_thinking to chat template kwargs.
// When the model supports thinking (Tier 1 or Tier 2) and the user hasn't disabled
// thinking in settings, pass enable_thinking=true. This is silently ignored by
// models whose templates don't reference the kwarg.
const _resolvedThinkMode = this._modelProfile?.thinkTokens?.mode;
const _chatTemplateKwargs = {};
if (s.enableThinking && _resolvedThinkMode !== 'none') {
_chatTemplateKwargs.enable_thinking = true;
} else if (_resolvedThinkMode === 'none' || !s.enableThinking) {
_chatTemplateKwargs.enable_thinking = false;
}
this._chat = new LlamaChat({ contextSequence: this._sequence, chatTemplateKwargs: _chatTemplateKwargs });
console.log(`[ChatEngine] chatTemplateKwargs=${JSON.stringify(_chatTemplateKwargs)}, resolvedThinkMode=${_resolvedThinkMode}, enableThinking=${s.enableThinking}, templateSupportsThinking=${_templateSupportsThinking}`);
this._chatHistory = [{ type: 'system', text: SYSTEM_PROMPT }];
this._lastEvaluation = null;
this.modelInfo.family = _detectedFamily;
this.modelInfo.sizeB = _detectedSizeB;
this.modelInfo.tier = this._modelProfile._meta.tier;
console.log(`[ChatEngine] P3: Model profile resolved — family=${_detectedFamily} (arch=${ggufArchString || 'n/a'} → ${_archFamily || 'fallback'}, file=${_fileFamily}), sizeB=${_detectedSizeB.toFixed(2)}, tier=${this._modelProfile._meta.tier}, sampling=${JSON.stringify(this._modelProfile.sampling)}`);
this.currentModelPath = modelPath;
this.isReady = true;
this.isLoading = false;
// Check vision availability — do NOT auto-start. Vision starts on-demand when an image needs captioning.
try {
const visionCheck = visionServer.checkAvailability(modelPath);
if (visionCheck.available) {
console.log(`[ChatEngine] Vision: mmproj found at ${visionCheck.mmprojPath} — vision available (will start on first image)`);
this.modelInfo.visionAvailable = true;
} else {
console.log(`[ChatEngine] Vision: ${visionCheck.reason} — vision unavailable for this model`);
this.modelInfo.visionAvailable = false;
}
} catch (visionErr) {
console.error(`[ChatEngine] Vision check error: ${visionErr.message}`);
this.modelInfo.visionAvailable = false;
}
console.log(
`[ChatEngine] Model ready: ctx=${actualCtx} (${s.contextSize <= 0 ? 'auto' : `fixed ${s.contextSize}`}, cap ${desiredMax}${trainMaxContext != null ? `, train ${trainMaxContext}` : ''}), gpuLayers=${this.modelInfo.gpuLayers}`,
);
this.emit('status', { state: 'ready', message: `Model ready: ${this.modelInfo.name}`, modelInfo: this.modelInfo });
return this.modelInfo;
} catch (err) {
this.isLoading = false;
this.isReady = false;
// Detect "unknown model architecture" failures from llama.cpp and surface
// an actionable message instead of the raw C++ error. Matches messages like:
// "unknown model architecture: 'gemma4'"
// "unknown architecture 'qwen3'"
// The bundled llama.cpp build only knows the architectures present at the
// node-llama-cpp release time. New model families (e.g. gemma4 released
// after our build) cannot be loaded until the runtime is updated.
const archMatch = /unknown\s+(?:model\s+)?architecture[:\s]*['"]?([A-Za-z0-9_.-]+)['"]?/i.exec(err.message || '');
let userMessage = err.message;
if (archMatch) {
const arch = archMatch[1];
userMessage =
`Model architecture "${arch}" is not supported by the bundled llama.cpp runtime. ` +
`Update guIDE to a newer release, or rebuild the runtime with ` +
`'npx -n node-llama-cpp source download --release latest'. ` +
`(Original error: ${err.message})`;
console.error(`[ChatEngine] Unsupported architecture detected: ${arch}`);
}
this.emit('status', { state: 'error', message: userMessage });
throw err;
}
}
// Redact passwords/credentials from text before storing in chat history
_redactCredentials(text) {
console.log(`[ChatEngine] _redactCredentials: input=${text?.length || 0} chars`);
if (!text || typeof text !== 'string') return text;
const redacted = text
.replace(/(?:password|passwd|pwd|secret|token|api[_-]?key)\s*[:=]\s*["']?([^\s"'`,;}\]]{3,})/gi,
(m, val) => m.replace(val, '[REDACTED]'))
.replace(/(?:password|passwd|pwd|secret|token|api[_-]?key)["']\s*:\s*["']([^"']{3,})["']/gi,
(m, val) => m.replace(val, '[REDACTED]'));
console.log(`[ChatEngine] _redactCredentials: redacted=${redacted.length} chars`);
return redacted;
}
async chat(userMessage, options = {}) {
if (!this.isReady || !this._chat) throw new Error('Model not ready');
const { onToken, onComplete, onContextUsage, onToolCall, onStreamEvent, systemPrompt, functions, toolPrompt, compactToolPrompt, executeToolFn, guideInstructionsPath } = options;
// Inject attachment content into user message
const attachments = Array.isArray(options.attachments) ? options.attachments : [];
let effectiveUserMessage = String(userMessage ?? '');
this._recentlyWrittenFiles.clear(); // reset per chat() call
this._toolRoundCount = 0;
console.log(`[ChatEngine] ═══ USER MESSAGE ═══ "${String(userMessage)}"`);
if (attachments.length > 0) {
console.log(`[ChatEngine] Attachments: ${attachments.length} (${attachments.map(a => `${a.name||'?'} ${a.mimeType||a.type||'?'}`).join(', ')})`);
}
console.log(`[ChatEngine] Processing ${attachments.length} attachment(s)`);
if (attachments.length > 0) {
const textParts = [];
for (let ai = 0; ai < attachments.length; ai++) {
const a = attachments[ai];
console.log(`[ChatEngine] Attachment #${ai}: name=${a?.name || '?'}, mime=${a?.mimeType || a?.type || '?'}, hasData=${!!a?.data}`);
if (!a?.data) {
console.warn(`[ChatEngine] Attachment #${ai} has no data — skipping`);
continue;
}
const mime = (a.mimeType || a.type || '').toLowerCase();
if (mime.startsWith('image/')) {
console.log(`[ChatEngine] Attachment #${ai} is image, visionAvailable=${!!this.modelInfo?.visionAvailable}`);
// Vision captioning: captionImage() handles load-on-demand internally
// (starts server if needed, captions, then unloads to free VRAM)
let captioned = false;
if (this.modelInfo?.visionAvailable) {
console.log(`[ChatEngine] Calling visionServer.captionImage for attachment #${ai}`);
try {
const caption = await visionServer.captionImage(
a.data, mime,
'Describe this image in detail. List all visible text, UI elements, content, and any information shown.'
);
if (caption) {
textParts.push(`[VISION ANALYSIS — You have already seen this image via your vision system. This is what you observed:\n${caption}\nEND VISION ANALYSIS]`);
console.log(`[ChatEngine] Image attachment #${ai} captioned: ${caption.length} chars`);
captioned = true;
} else {
console.warn(`[ChatEngine] visionServer.captionImage returned empty caption for attachment #${ai}`);
}
} catch (visionErr) {
console.error(`[ChatEngine] Vision caption for attachment #${ai} failed: ${visionErr.message}`);
}
} else {
console.warn(`[ChatEngine] Vision not available for attachment #${ai} — skipping caption`);
}
if (!captioned) {
// Vision unavailable — tell the model explicitly that the image cannot be processed.
// Do NOT inject raw attachment metadata like "[Attached context] filename.png"
// because the model echoes it verbatim as its response instead of answering the user.
console.log(`[ChatEngine] Attachment #${ai} not captioned — injecting fallback message`);
textParts.push(`[Image attachment "${a.name || 'image'}" could not be processed by vision engine. Tell the user you cannot see the image and ask them to describe its contents.]`);
}
continue;
}
try {
const decoded = Buffer.from(a.data, 'base64').toString('utf8');
textParts.push(`[Attached file: ${a.name || 'file'}]\n${decoded}`);
console.log(`[ChatEngine] Attachment #${ai} decoded as text: ${decoded.length} chars`);
} catch (e) {
console.warn(`[ChatEngine] Attachment #${ai} decode failed: ${e.message}`);
}
}
if (textParts.length > 0) {
effectiveUserMessage = effectiveUserMessage + '\n\n' + textParts.join('\n\n---\n\n');
console.log(`[ChatEngine] Final effectiveUserMessage length: ${effectiveUserMessage.length} chars`);
} else {
console.warn(`[ChatEngine] No text parts produced from ${attachments.length} attachment(s)`);
}
}
// ─── Context Assembly Pipeline (6-layer ordered injection) ───
// Same architecture as Windsurf Cascade: Rules → Memories → Editor → RAG → Tools → History
// Each layer is appended in order so the model sees them in priority sequence.
const contextTokens = this._context?.contextSize || 8192;
let basePrompt = systemPrompt || SYSTEM_PROMPT;
// Layer 1: System prompt (identity, behavior rules, tool calling format)
// (already set above as basePrompt)
// Thinking: Qwen 3.5 models are reasoning models that emit thinking natively via the
// chat wrapper segment handling. Do NOT force thinking via prompt instructions - it
// interferes with the native segment API and can cause premature EOG stops.
// The raw-text detection in _sfProcessChunk still routes any thinking content to
// llm-thinking-token events when the model does produce it.
// Layer 2: Project rules & environment context (date, OS, project path, guide instructions)
const now = new Date();
const dateStr = now.toISOString().split('T')[0];
const timeStr = now.toTimeString().split(' ')[0];
const platform = `${os.type()} ${os.release()} (${os.platform()})`;
basePrompt += `\n\nCurrent date: ${dateStr}\nCurrent time: ${timeStr}\nOperating system: ${platform}`;
if (this._projectPath) {
basePrompt += `\nProject directory: ${this._projectPath}\nAll file tools operate relative to this directory.`;
}
// Project instructions file (AGENTS.md / guide rules)
if (guideInstructionsPath) {
try {
const fs = require('fs');
const instrPath = path.isAbsolute(guideInstructionsPath)
? guideInstructionsPath
: path.join(this._projectPath || process.cwd(), guideInstructionsPath);
if (fs.existsSync(instrPath)) {
const guideContent = fs.readFileSync(instrPath, 'utf-8').trim();
if (guideContent) {
basePrompt += `\n\n## Project Instructions (from ${guideInstructionsPath})\n${guideContent}`;
}
}
} catch (e) { console.warn('[ChatEngine] Guide instructions load failed:', e.message); }
}
// Custom instructions from settings — user-defined behavior overrides
const customInstructions = options.customInstructions;
if (customInstructions && customInstructions.trim()) {
basePrompt += `\n\n## Custom Instructions\n${customInstructions.trim()}`;
}
// Layer 3: Editor context (active file, cursor position, recent saves, diagnostics)
// This is the "real-time action context" layer — model knows what user is doing
if (this._ctx?.editorContext) {
const ec = this._ctx.editorContext;
const parts = [];
if (ec.activeFilePath) {
const rel = this._projectPath ? ec.activeFilePath.replace(this._projectPath.replace(/\\/g, '/'), '').replace(/^\//, '') : ec.activeFilePath;
let ctxLine = `Active file: ${rel}`;
if (ec.cursorLine) ctxLine += ` (cursor at line ${ec.cursorLine})`;
parts.push(ctxLine);
}
if (ec.recentSaves?.length > 0) {
const saveNames = ec.recentSaves
.map(s => this._projectPath ? s.filePath.replace(this._projectPath.replace(/\\/g, '/'), '').replace(/^\//, '') : s.filePath)
.filter(Boolean);
if (saveNames.length > 0) parts.push(`Recently saved: ${saveNames.join(', ')}`);
}
// Include global diagnostics summary if there are errors
const diagStore = this._ctx.editorDiagnostics;
if (diagStore) {
let totalErrors = 0, totalWarnings = 0, errorFiles = [];
for (const [fp, d] of Object.entries(diagStore)) {
if (d.errors > 0) {
totalErrors += d.errors;
const rel = this._projectPath ? fp.replace(this._projectPath.replace(/\\/g, '/').toLowerCase(), '').replace(/^\//, '') : fp;
errorFiles.push(`${rel} (${d.errors} errors)`);
}
totalWarnings += d.warnings || 0;
}
if (totalErrors > 0) {
parts.push(`Diagnostics: ${totalErrors} errors in ${errorFiles.slice(0, 5).join(', ')}`);
}
}
if (parts.length > 0) {
basePrompt += `\n\n## Editor Context\n${parts.join('\n')}`;
}
}
// Mode overrides: inject behavioral instructions before tool prompt
if (options.planMode) {
basePrompt += '\n\n## PLAN MODE ACTIVE\nBefore modifying or creating any files, you MUST:\n1. Use read_file, list_directory, grep_search, and git_status to fully understand scope.\n2. Write a complete, numbered implementation plan to a file called "GUIDE_PLAN.md" using write_file.\n3. STOP after writing the plan file. Do NOT modify source files in this turn.\n4. The user will review the plan and say "proceed" when ready for execution.\nIn Plan Mode: read-only tools and write_file (for GUIDE_PLAN.md only) are permitted. Do NOT run commands, modify source code, or install packages.';
console.log('[ChatEngine] Plan mode — model will write GUIDE_PLAN.md before executing');
}
if (options.askOnly) {
basePrompt += '\n\n## ASK MODE ACTIVE\nYou are in Q&A mode. Answer the user\'s question directly in text. Do NOT call any tools or make any file or system changes.';
console.log('[ChatEngine] Ask mode — tool prompt suppressed, model responds conversationally');
}
// Layer 4: Tool prompt (available tools and their descriptions)
// Hoist useCompact and effectiveToolPrompt so they're accessible in the
// generation-space trimming section (line ~1746) regardless of which
// branch sets them. Without this, ReferenceError when toolPrompt is absent.
let useCompact = false;
let effectiveToolPrompt = '';
if (options.askOnly) {
// Ask mode: no tools available — just set the base prompt with mode instruction
this._chatHistory[0].text = basePrompt;
} else if (toolPrompt) {
// Estimate token cost of tool prompt (~3.5 chars/token for English)
const toolPromptTokens = Math.ceil(toolPrompt.length / 3.5);
const toolPct = Math.round((toolPromptTokens / contextTokens) * 100);
// Emit warning to UI when tool prompt consumes too much context
if (toolPct > 50 && onStreamEvent) {
onStreamEvent('generation-warning', {
message: `Tool prompt uses ${toolPct}% of context (${toolPromptTokens.toLocaleString()}/${contextTokens.toLocaleString()} tokens)`,
suggestion: 'Responses may be limited. Try a smaller model, reduce context in settings, or start a new session.',
});
}
// Use compact prompt when tool prompt would consume >40% of context
// (not just when ctx<8192 — a 13K tool prompt in 14K ctx is equally disastrous)
// Decision is based on CONTEXT SIZE, not model parameters — small models in 2026
// have huge context windows (128K+), so model size doesn't determine prompt style.
useCompact = compactToolPrompt && (contextTokens < 8192 || toolPct > 40);
effectiveToolPrompt = useCompact ? compactToolPrompt : toolPrompt;
// If even the compact prompt is too large, progressively trim it
if (useCompact && typeof effectiveToolPrompt === 'string') {
const compactPct = Math.round((Math.ceil(effectiveToolPrompt.length / 3.5) / contextTokens) * 100);
if (compactPct > 50) {
// Strip everything after the first 8 tool descriptions — keep format header + core tools
const lines = effectiveToolPrompt.split('\n');
const toolLineIdx = [];
lines.forEach((l, i) => { if (l.startsWith('- **')) toolLineIdx.push(i); });
if (toolLineIdx.length > 8) {
effectiveToolPrompt = lines.slice(0, toolLineIdx[8]).join('\n') + '\n…and more tools available\n';
}
}
}
this._chatHistory[0].text = basePrompt + '\n\n' + effectiveToolPrompt;
const finalPct = Math.round((Math.ceil(effectiveToolPrompt.length / 3.5) / contextTokens) * 100);
console.log(`[ChatEngine] Tool prompt injected (${effectiveToolPrompt.length} chars${useCompact ? ', compact' : ''}, ctx=${contextTokens}, finalPct=${finalPct}%, compactAvailable=${!!compactToolPrompt})`);
} else if (functions && Object.keys(functions).length > 0) {
this._chatHistory[0].text = basePrompt + this._buildToolPrompt(functions);
console.log(`[ChatEngine] Functions provided (fallback): ${Object.keys(functions).length} tools`);
} else if (systemPrompt) {
this._chatHistory[0].text = systemPrompt;
} else {