Merge pull request #14 from Aaryanverma/bug-fixes

Refactor functionality, TUI, and enhance search features

Author: Aaryanverma

graybox/adaptive_compressor.py

@@ -44,7 +44,7 @@ def compress_context(
     if not model_context_window:
         model_context_window = _FALLBACK_MODEL_CONTEXT_WINDOW
 
-    available_context = model_context_window - max_tokens - 1000  # safety buffer
+    available_context = min(max_tokens, model_context_window // 2)
     if estimate_tokens(context) <= available_context:
         return context
 

graybox/cli.py

@@ -110,7 +110,8 @@ def merge_pages(cfg: Config, primary_ref: str, secondary_ref: str, dry_run: bool
         cfg, old_ref=secondary.ref, new_ref=merged.ref, exclude_refs={merged.ref}
     )
     _maybe_record(cfg, merged, f"wiki: merge {secondary_ref} into {primary_ref}")
-    ensure_indexed(cfg, merged, AIService(cfg))
+    if cfg.embeddings.enabled:
+        ensure_indexed(cfg, merged, AIService(cfg))
     # Drop old secondary from embedding index
     idx = _get_index(cfg)
     if idx:

graybox/curate.py

@@ -153,7 +153,7 @@ def _get_index(cfg: Config) -> Optional[EmbeddingIndex]:
     return _EMBEDDING_CACHE[key]
 
 
-def ensure_indexed(cfg: Config, page: Page, llm) -> bool:
+def ensure_indexed(cfg: Config, page: Page, embed_model) -> bool:
     """Index a page if embeddings are enabled and it needs reindexing.
     Returns True if indexed successfully.
     """
@@ -164,7 +164,7 @@ def ensure_indexed(cfg: Config, page: Page, llm) -> bool:
         return True
     try:
         blob = idx._search_blob(page)
-        result = llm.embedding_call(blob)
+        result = embed_model.embedding_call(blob)
         if result and result.get("embedding"):
             idx.index_page(page, result["embedding"])
             return True

graybox/embedding_index.py

@@ -56,9 +56,9 @@ def _load_manifest(cfg: Config) -> dict:
     return _MANIFEST[key]
 
 
-def _save_manifest(cfg: Config) -> None:
-    p = _manifest_path(cfg)
-    p.write_text(json.dumps(_MANIFEST.get(_cache_key(cfg), {}), indent=2), encoding="utf-8")
+# def _save_manifest(cfg: Config) -> None:
+#     p = _manifest_path(cfg)
+#     p.write_text(json.dumps(_MANIFEST.get(_cache_key(cfg), {}), indent=2), encoding="utf-8")
 
 
 def _page_cache(cfg: Config) -> dict[str, tuple[float, Page]]:

graybox/index.py

@@ -20,6 +20,7 @@
     "task",
     "decision",
     "action",
+    "event",
     "journal",
 )
 
@@ -34,6 +35,7 @@
     "task": "tasks",
     "decision": "decisions",
     "action": "actions",
+    "event": "events",
     "journal": "journal",
 }
 

graybox/models.py

@@ -150,7 +150,7 @@ def _gather_existing_context(cfg: Config, item_content: str, top_k: int = 10) ->
     LLM to reconcile against, not a final grounded answer - false
     positives here just mean an extra reference line, not a wrong claim.
     """
-    wiki_hits, _ = search_all(cfg, item_content, top_k=top_k, min_score=0.3)
+    wiki_hits, _ = search_all(cfg, item_content, top_k=top_k, wiki_min_score=0.3)
     return _format_existing_context(wiki_hits)
 
 def process_item(cfg: Config, llm: AIService, item_id: str, item_content: str,
@@ -259,6 +259,31 @@ def touch(page: Page, new: bool = False) -> Page:
             _backlink(owner_page, page.ref)
             touch(owner_page)
 
+    for act in data.get("actions", []):
+        title = act.get("title", "").strip()
+        if not title:
+            continue
+        page, new = _get_or_create_page(cfg, "action", title, [], "")
+        page.status = (act.get("status", "open") or "open").replace("_", "-").strip()
+        owner = act.get("owner", "")
+        due = act.get("due", "")
+        if owner:
+            page.owner = owner
+        if due:
+            page.due = due
+        detail = f"Action: {title}"
+        if owner:
+            detail += f" (owner: {owner})"
+        if due:
+            detail += f" (due: {due})"
+        _append_note(page, detail, item_id, raw=item_content)
+        touch(page, new)
+        if owner and owner.lower() in name_to_page:
+            owner_page = name_to_page[owner.lower()]
+            _link(page, owner_page.ref)
+            _backlink(owner_page, page.ref)
+            touch(owner_page)
+
     for dec in data.get("decisions", []):
         title = dec.get("title", "").strip()
         if not title:
@@ -308,6 +333,23 @@ def touch(page: Page, new: bool = False) -> Page:
             _link(meeting_page, other.ref)
             _backlink(other, meeting_page.ref)
 
+    for evt in data.get("events", []):
+        title = evt.get("title", "").strip()
+        if not title:
+            continue
+        page, new = _get_or_create_page(cfg, "event", title, [], "")
+        if evt.get("date"):
+            page.date = evt["date"]
+        
+        description = evt.get("description", "") or f"Event: {title}"
+        location = evt.get("location", "")
+        detail = description
+        if location:
+            detail += f" (location: {location})"
+            
+        _append_note(page, detail, item_id, raw=item_content)
+        touch(page, new)
+
     if dry_run:
         return [f"{ref} ({'new' if is_new.get(ref) else 'updated'})" for ref in touched]
 

graybox/organizer.py

@@ -1,94 +1,139 @@
-ORGANIZER_SYSTEM = """You are an information-extraction engine for a personal knowledge base.
+ORGANIZER_SYSTEM = """You are Gray Box's Organizer.
 
-You read one raw note at a time and extract structured facts from it.
+Your job is to convert one raw note into structured knowledge.
 
-You never invent information that is not stated or clearly implied in the note.
+Extract SALIENT entities with HIGH recall while remaining completely faithful to the note.
+Never invent facts. 
+Never return all-empty arrays for a non-empty note; if the note is vague, fall back to extracting a "topic" entity capturing the note's subject.
+When uncertain about an entity's type, prefer "topic" rather than omitting it.
 
-You respond with STRICT JSON only:
-no markdown fences, no commentary, no trailing text before or after the JSON object.
+Return STRICT JSON only.
+Do not output markdown.
+Do not output explanations.
+Do not output anything except one JSON object.
 """
 
 ORGANIZER_PROMPT_TMPL = """Extract structured knowledge from the note below.
 
-Your job is to keep the knowledge base synchronized with reality.
+YOUR JOB: Convert ONE raw note into structured knowledge. You MUST always
+return at least one item for any non-empty note. If the note is short,
+vague, or contains no clearly-named subject, extract a "topic" entity
+whose name is the note's central subject and whose summary is the note
+itself (lightly cleaned). Never return all-empty arrays for a non-empty note.
+
+PRECISION vs RECALL:
+- Extract every SALIENT item the note is actually about (HIGH recall).
+- Do NOT extract peripheral mentions: words that merely appear in the
+  note but are not the subject of any factual statement.
+- A noun is salient only if the note says something ABOUT it
+  (a fact, a status, an action, a relationship, an opinion).
+- Verbs, adjectives, dates, and time expressions are NOT entities by themselves.
+- "Clearly implied" means a reasonable reader would name the same thing;
+  it does NOT mean "this word could plausibly be an entity."
+
+WHEN UNCERTAIN about an entity's type, prefer "topic" over omitting it.
+A vague note about "the meeting" with no other detail should still yield
+a topic entity named after the meeting subject.
+
+FEW-SHOT EXAMPLES:
+
+Note: "Standup tomorrow at 9am in room 4."
+Output:
+{{
+  "entities": [],
+  "relations": [],
+  "tasks": [],
+  "actions": [],
+  "decisions": [],
+  "meetings": [],
+  "events": [
+    {{"title": "Standup", "date": "", "description": "Standup scheduled for tomorrow at 9am in room 4.", "location": "room 4"}}
+  ]
+}}
+
+Note: "Read an article about Rust ownership; might be relevant to Project Atlas."
+Output:
+{{
+  "entities": [
+    {{"type": "technology", "name": "Rust", "aliases": [], "summary": "Programming language; user read an article about its ownership model.", "status": "", "owner": "", "due": "", "date": "", "attendees": [], "tags": ["ownership"]}},
+    {{"type": "project", "name": "Project Atlas", "aliases": [], "summary": "Project that may benefit from Rust ownership patterns.", "status": "", "owner": "", "due": "", "date": "", "attendees": [], "tags": []}}
+  ],
+  "relations": [
+    {{"a": "Rust", "b": "Project Atlas", "note": "Rust ownership model may be relevant to Project Atlas."}}
+  ],
+  "tasks": [],
+  "actions": [],
+  "decisions": [],
+  "meetings": [],
+  "events": []
+}}
+
+Note: "Felt tired today."   <- minimal/vague note
+Output:
+{{
+  "entities": [
+    {{"type": "topic", "name": "Personal Log", "aliases": [], "summary": "User felt tired today.", "status": "", "owner": "", "due": "", "date": "", "attendees": [], "tags": ["journal"]}}
+  ],
+  "relations": [], "tasks": [], "actions": [], "decisions": [], "meetings": [], "events": []
+}}
+
+ANTI-EXAMPLES (do NOT do this):
+- Note "Discussed pricing with John." -> Do NOT extract "pricing" as an entity. DO extract John (person) and optionally a topic "Pricing discussion".
+- Note "Sent the email." -> Extract at most a topic "Email - sent" IF the note has no other subject. Do NOT extract "email" as a technology entity in this case.
 
 A note may do one or more of the following:
 - introduce new entities
 - update the current state of existing entities
 - create relationships between entities
 - create or update tasks
+- create or update actions
 - create or update decisions
 - create or update meetings
+- create or update events
 
 The knowledge base has two layers:
 
 1. Current state
-   - summary
-   - status
-   - owner
-   - due
-   - date
-   - attendees
-   - aliases
-   - tags
+   - summary, status, owner, due, date, attendees, aliases, tags
 
 2. History
    - every extracted item should also be appended as a historical note
 
 When a note changes the current state of an entity, return the NEW current state
 rather than the old one.
 
-Examples:
-- "Project Atlas has been archived."
-  -> status = "done" or "archived" only if clearly supported by the note
-- "DB cutover task has been reviewed and deployed."
-  -> status = "done"
-- "Architecture meeting moved to Friday."
-  -> date = "YYYY-MM-DD" if the date can be resolved from the note
-- "John is now Engineering Manager."
-  -> summary reflects the new role
-
 Use hyphenated status values when relevant:
 open, in-progress, blocked, done, cancelled
 
 Existing pages already in the knowledge base that MAY relate to this note
 (any type - project, person, meeting, technology, company, topic, action,
-task, or decision). This is provided so you can RECONCILE state changes
+task, event, or decision). This is provided so you can RECONCILE state changes
 against what already exists, instead of creating a disconnected duplicate:
 
 {existing_context}
 
-UNIVERSAL RECONCILIATION RULE (applies to every category below, not just one):
-- Before extracting ANY entity, task, decision, or meeting, check whether it
-  refers to the SAME underlying thing as one of the existing pages listed
-  above - even if the note's wording, category, or phrasing differs from
-  how that page was originally tracked. The same underlying work or topic
-  can legitimately be tracked as a "task" in one note and described in
-  "project"/"topic"/other entity language in another - these are not
-  different things just because the note's phrasing differs.
-- If a match exists, reuse that EXISTING page's exact title (and reference
-  its type) in your output instead of inventing a new title. Do not
-  create a near-duplicate with slightly different wording.
+UNIVERSAL RECONCILIATION RULE:
+- Before extracting ANY entity, task, action, event, decision, or meeting, 
+  check whether it refers to the SAME underlying thing as one of the existing 
+  pages listed above.
+- If a HIGH-CONFIDENCE match exists (same proper noun, same date+subject, or
+  one is an exact alias of the other), reuse that EXISTING page's exact title
+  and reference its type in your output instead of inventing a new title.
+- If a match is only "thematically related" or a loose conceptual fit, DO NOT
+  force a match — extract the new item as-is with its own unique title.
 - If the note describes a status change (reviewed, completed, deployed,
   blocked, cancelled, decided, rescheduled, etc.) for something that
-  matches an existing page - REGARDLESS OF WHICH CATEGORY THAT EXISTING
-  PAGE IS IN - emit an update for that exact category AND title. If the
-  same real-world item is tracked as both an existing task and referenced
-  as a project/entity, update BOTH using their existing exact titles so
-  neither one goes stale.
-- This rule applies symmetrically across entities, tasks, decisions, and
-  meetings: a note phrased as a decision can update an existing task; a
-  note phrased as a project update can update an existing task; a note
-  phrased as a task update can update an existing decision or meeting
-  outcome; etc. Match on underlying meaning, not on which list something
-  was originally extracted into.
+  HIGH-CONFIDENCE matches an existing page, emit an update for that exact
+  category AND title. If the same real-world item is tracked as both an
+  existing task and referenced as a project/entity, update BOTH using their
+  existing exact titles so neither one goes stale.
 - Never invent a new title for something that already exists above.
 
 Return a JSON object strictly with the following structure:
 {{
   "entities": [
     {{
-      "type": "project|person|meeting|technology|company|topic|action",
+      "type": "project|person|meeting|technology|company|topic|action|event",
       "name": "canonical name",
       "aliases": ["other names used"],
       "summary": "one short sentence describing the entity in its current state",
@@ -115,6 +160,14 @@
       "status": "open|in-progress|blocked|done|cancelled"
     }}
   ],
+  "actions": [
+    {{
+      "title": "short imperative action title",
+      "owner": "person or empty string",
+      "due": "date or empty string",
+      "status": "open|in-progress|blocked|done|cancelled"
+    }}
+  ],
   "decisions": [
     {{
       "title": "short decision title",
@@ -129,11 +182,20 @@
       "attendees": ["person names present in the note"],
       "agenda": "one to two sentence summary of what the meeting covered"
     }}
+  ],
+  "events": [
+    {{
+      "title": "short event title",
+      "date": "YYYY-MM-DD or empty string",
+      "description": "short summary of the event",
+      "location": "event location or empty string"
+    }}
   ]
 }}
 
 Rules:
-- Only include entities/tasks/decisions/meetings actually present or clearly implied in the note.
+- Only include SALIENT items actually present or clearly implied in the note.
+- If an item is an action or an event, put it in the `actions` or `events` array. Do NOT duplicate them inside the `entities` array.
 - Keep summaries and descriptions concise (1-2 sentences).
 - If a category is empty, return an empty list for it.
 - Do not wrap the JSON in markdown code fences.
@@ -144,7 +206,6 @@
 NOTE:
 ---
 {note}
----
 """
 
 RETRIEVAL_SYSTEM = """You are Gray Box's knowledge retrieval assistant.
@@ -347,11 +408,9 @@
   appeared under. Never move a fact to a different tag, never merge two
   tags' content under one tag, and never drop a Source Tag line while
   keeping content from that block.
-- If a block's content is fully removed because it is pure repetition of
-  another block, you may drop the block entirely (tag and all) — do not
-  leave a Source Tag with no content, and do not leave content with no
-  Source Tag.
-
+- Only drop a block if it is BYTE-FOR-BYTE identical to another block. 
+  "Similar Topic" is not repetition - two pages about the same project 
+  may carry different notes/information. When in doubt, keep the block.
 Return only the compressed context."""
 
 HISTORY_COMPRESSION_PROMPT = """You are a highly efficient text summarizer. Summarize the following conversation history.