Preserve sync progress and refresh synonyms with collision-free context keys

Author: ktappdev

README.md

@@ -52,7 +52,7 @@ Remote LLM endpoints must use HTTPS; HTTP is accepted only on loopback.
 
 **Synonyms:** Each node (file or directory) gets 5–12 LLM-generated synonyms via batched API calls to an OpenAI-compatible endpoint. For example, `Skeletons.tsx → ["skeletons", "loading", "placeholder", "animation"]`. Names are smart-batched (up to 60 per request, 10 parallel by default). The prompt includes extracted symbols and, for JS/TS files, ESM imports to generate contextual terms. Use `--offline` to guarantee that no LLM requests are made.
 
-**Bootstrap diff:** On subsequent runs, `watch` diffs the filesystem against the existing snapshot using file modification times and sizes. Timestamp-preserving edits with a different size are detected; edits preserving both size and timestamp require a full rebuild. Deleted files are removed, new files are added, modified files are re-extracted. No LLM calls for existing synonyms — run `ctxt sync` to fill gaps. Synonym keys use `parentDir/basename` (e.g., `webhook/route.ts`) instead of bare basenames to prevent duplicate filenames from overwriting each other in the synonym map.
+**Bootstrap diff:** On subsequent runs, `watch` diffs the filesystem against the existing snapshot using file modification times and sizes. Timestamp-preserving edits with a different size are detected; edits preserving both size and timestamp require a full rebuild. Deleted files are removed, new files are added, modified files are re-extracted. Watch bootstrap reuses cached synonyms; run `ctxt sync` to fill gaps and refresh changed symbol/import context. Synonym cache keys use the full project-relative path (e.g., `billing/routes/index.ts`) so identically named files in separate features remain distinct.
 
 ### Search mechanics
 
@@ -105,13 +105,30 @@ On first run, creates a starter `.ctxt/ctx_config.toml` and pauses so you can ed
 
 ### `ctxt sync`
 
-Targeted synonym generation — only generates synonyms for names that are missing or below `synonyms_min`. Works on the existing index without rebuilding.
+Targeted synonym generation — generates synonyms for indexed names that are missing, below `synonyms_min`, or have changed symbols/imports. It refreshes symbol extraction from disk before comparing context fingerprints. Works on the existing index without rebuilding the file tree.
 
 ```bash
 ctxt sync .
+ctxt sync . --path command_sync.go
+ctxt sync . --path docs --force
 ```
 
-Use after `init` to fill synonym gaps, or after adding new files to catch names that were missed.
+`--path` selects an indexed file or directory; `--force` regenerates selected
+entries even if their fingerprints are current. Use `--force` after changing
+models or generation instructions. Body-only edits that leave symbols/imports
+unchanged also require `--force`.
+
+Successful LLM responses are saved even if another batch fails. The command
+still exits with an error; rerun without `--force` to request only missing,
+short, or stale entries. `--batch-size` explicitly controls batch size when set.
+
+The cache automatically migrates to version 2 on the next write. Unique legacy
+entries are retained; ambiguous parent/file keys are dropped rather than shared
+between unrelated files. Legacy entries have no context fingerprint, so their
+first online `init` or `sync` refreshes them. Use `--offline` with `init` to migrate
+without LLM requests. Keep the CLI upgraded when sharing these cache files.
+
+Use `init` or restart `watch` after adding files, then run `sync` to fill gaps.
 
 ### `ctxt watch`
 
@@ -292,7 +309,7 @@ search-hints → load .ctxt/ctx_index.json (or query live memory) → score toke
 ## File formats
 
 - **`.ctxt/ctx_index.json`** — schema version, root path, timestamp, and indexed tree
-- **`.ctxt/ctx_cache.json`** — directory or parent/file key → synonyms for reuse
+- **`.ctxt/ctx_cache.json`** — versioned project-relative synonyms and symbol/import context fingerprints
 - **`.ctxt/ctx_config.toml`** — config-driven defaults
 - **`.ctxt/ctx_runtime.json`** — live watch discovery for memory search
 - **Bench/eval case files** — v2 format with categories (path-intent, symbol-lookup, concept-synonym, exact-file, narrow-scope, vague-intent); v1 bare array format is backward compatible

cache.go

@@ -6,27 +6,65 @@ import (
 	"os"
 )
 
+type synonymCacheState struct {
+	Version      int               `json:"cache_version"`
+	Entries      SynonymResponse   `json:"entries"`
+	Fingerprints map[string]string `json:"fingerprints,omitempty"`
+}
+
 func LoadSynonymCache(path string) (SynonymResponse, error) {
+	state, err := loadSynonymCacheState(path)
+	return state.Entries, err
+}
+
+func loadSynonymCacheState(path string) (synonymCacheState, error) {
+	state := synonymCacheState{Entries: make(SynonymResponse), Fingerprints: make(map[string]string)}
 	if path == "" {
-		return make(SynonymResponse), nil
+		return state, nil
+	}
+	data, err := os.ReadFile(path)
+	if os.IsNotExist(err) {
+		return state, nil
 	}
-	bytes, err := os.ReadFile(path)
 	if err != nil {
-		if os.IsNotExist(err) {
-			return make(SynonymResponse), nil
+		return state, fmt.Errorf("read synonym cache: %w", err)
+	}
+	var fields map[string]json.RawMessage
+	if err := json.Unmarshal(data, &fields); err != nil {
+		return state, fmt.Errorf("parse synonym cache: %w", err)
+	}
+	if version, ok := fields["cache_version"]; ok && len(version) > 0 && version[0] != '[' {
+		if err := json.Unmarshal(data, &state); err != nil {
+			return state, fmt.Errorf("parse synonym cache: %w", err)
 		}
-		return nil, fmt.Errorf("read synonym cache: %w", err)
+		if state.Version != 2 {
+			return state, fmt.Errorf("unsupported synonym cache version %d", state.Version)
+		}
+	} else if err := json.Unmarshal(data, &state.Entries); err != nil {
+		return state, fmt.Errorf("parse synonym cache: %w", err)
 	}
-
-	cache := make(SynonymResponse)
-	if err := json.Unmarshal(bytes, &cache); err != nil {
-		return nil, fmt.Errorf("parse synonym cache: %w", err)
+	if state.Entries == nil {
+		state.Entries = make(SynonymResponse)
 	}
+	if state.Fingerprints == nil {
+		state.Fingerprints = make(map[string]string)
+	}
+	for name, syns := range state.Entries {
+		state.Entries[name] = sanitizeSynonyms(syns, 16)
+	}
+	return state, nil
+}
 
-	for name, syns := range cache {
-		cache[name] = sanitizeSynonyms(syns, 16)
+func saveSynonymCacheState(path string, state synonymCacheState) error {
+	if path == "" {
+		return nil
+	}
+	state.Version = 2
+	data, err := json.MarshalIndent(state, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal synonym cache: %w", err)
 	}
-	return cache, nil
+	return writeFileAtomic(path, append(data, '\n'), 0o600)
 }
 
 func SaveSynonymCache(path string, cache SynonymResponse) error {

cache_migration.go

@@ -0,0 +1,57 @@
+package contexting
+
+import "path/filepath"
+
+// migrateSynonymCache preserves only legacy entries that identify one node.
+// Ambiguous suffixes must not carry one file's meaning into another file.
+func migrateSynonymCache(tree *Node, state synonymCacheState) synonymCacheState {
+	counts := make(map[string]int)
+	walkTree(tree, func(node *Node) {
+		if node == tree {
+			return
+		}
+		counts[legacySynonymKey(node)]++
+	})
+	entries := make(SynonymResponse)
+	walkTree(tree, func(node *Node) {
+		if node == tree {
+			return
+		}
+		rel, err := filepath.Rel(tree.FullPath, node.FullPath)
+		if err != nil {
+			return
+		}
+		node.SynonymKey = filepath.ToSlash(rel)
+		if state.Version >= 2 {
+			return
+		}
+		legacy := legacySynonymKey(node)
+		if counts[legacy] == 1 {
+			if values, ok := state.Entries[legacy]; ok {
+				entries[node.SynonymKey] = values
+				return
+			}
+		}
+		// Support callers already supplying relative keys in the public map API.
+		if counts[node.SynonymKey] == 0 {
+			if values, ok := state.Entries[node.SynonymKey]; ok {
+				entries[node.SynonymKey] = values
+			}
+		}
+	})
+	if state.Version < 2 {
+		state.Entries = entries
+	}
+	state.Version = 2
+	if state.Fingerprints == nil {
+		state.Fingerprints = make(map[string]string)
+	}
+	return state
+}
+
+func legacySynonymKey(node *Node) string {
+	if node.Type == "file" {
+		return pathSuffix(node.FullPath)
+	}
+	return filepath.Base(node.FullPath)
+}

command_init.go

@@ -107,7 +107,7 @@ Examples:
 			}
 			llmEndpoint, llmModel, llmKey, llmTemp, llmMaxTokens, llmProvider := resolveLLMConfig(flags, cfg.LLM)
 			LogInfof("LLM: provider=%s model=%s endpoint=%s api_key=%s", llmProvider, llmModel, endpointForLog(llmEndpoint), maskAPIKey(llmKey))
-			cache, err := LoadSynonymCache(cachePath)
+			cacheState, err := loadSynonymCacheState(cachePath)
 			if err != nil {
 				return err
 			}
@@ -118,22 +118,24 @@ Examples:
 			defer stop()
 
 			result, err := BuildIndex(BuildOptions{
-				Ctx:              ctx,
-				RootPath:         rootPath,
-				IgnoredPaths:     ignored,
-				APIKey:           llmKey,
-				Model:            llmModel,
-				BatchSize:        flags.BatchSize,
-				SynonymsPerName:  flags.SynonymsPerName,
-				SynonymsMin:      flags.SynonymsMin,
-				SynonymsMax:      flags.SynonymsMax,
-				SynonymCache:     cache,
-				MaxBatchSize:     cfg.Watch.MaxBatchSize,
-				Endpoint:         llmEndpoint,
-				Temperature:      llmTemp,
-				MaxTokens:        llmMaxTokens,
-				ParallelRequests: cfg.LLM.ParallelRequests,
-				Verbose:          flags.Verbose,
+				Ctx:                 ctx,
+				RootPath:            rootPath,
+				IgnoredPaths:        ignored,
+				APIKey:              llmKey,
+				Model:               llmModel,
+				BatchSize:           flags.BatchSize,
+				SynonymsPerName:     flags.SynonymsPerName,
+				SynonymsMin:         flags.SynonymsMin,
+				SynonymsMax:         flags.SynonymsMax,
+				SynonymCache:        cacheState.Entries,
+				CacheVersion:        cacheState.Version,
+				SynonymFingerprints: cacheState.Fingerprints,
+				MaxBatchSize:        cfg.Watch.MaxBatchSize,
+				Endpoint:            llmEndpoint,
+				Temperature:         llmTemp,
+				MaxTokens:           llmMaxTokens,
+				ParallelRequests:    cfg.LLM.ParallelRequests,
+				Verbose:             flags.Verbose,
 			})
 			if err != nil {
 				return err
@@ -143,7 +145,7 @@ Examples:
 			}
 
 			emitSynonymWarning(result.SynonymError)
-			if err := SaveSynonymCache(cachePath, result.SynonymCache); err != nil {
+			if err := saveSynonymCacheState(cachePath, synonymCacheState{Version: 2, Entries: result.SynonymCache, Fingerprints: result.SynonymFingerprints}); err != nil {
 				return err
 			}
 			if err := SaveContextIndex(outputPath, result.Index); err != nil {