From 42dbb9696f3d926c4263b10d936f75de825d946f Mon Sep 17 00:00:00 2001 From: Huanyi Xie <74809554+xiehuanyi@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:36:06 +0300 Subject: [PATCH 1/2] fix(skill): bind semantic merges to dispatched source ownership Validate every current-run chunk before cache writes and reconstruct incremental merge payloads from current dispatch, walked owned cache entries, and structural inputs. Foreign stubs and stale aggregates must not replace undispatched files. Fixes #3947 AI-Assisted-By: OpenAI Codex --- graphify/skill-agents.md | 76 ++-- graphify/skill-amp.md | 76 ++-- graphify/skill-claw.md | 74 ++-- graphify/skill-codex.md | 70 +-- graphify/skill-copilot.md | 74 ++-- graphify/skill-droid.md | 76 ++-- graphify/skill-kilo.md | 74 ++-- graphify/skill-kiro.md | 74 ++-- graphify/skill-opencode.md | 70 +-- graphify/skill-pi.md | 74 ++-- graphify/skill-trae.md | 76 ++-- graphify/skill-vscode.md | 78 ++-- graphify/skill-windows.md | 74 ++-- graphify/skill.md | 74 ++-- graphify/skill_merge.py | 179 ++++++++ .../agents/references/extraction-spec.md | 2 + graphify/skills/agents/references/update.md | 20 + .../skills/amp/references/extraction-spec.md | 2 + graphify/skills/amp/references/update.md | 20 + .../claude/references/extraction-spec.md | 2 + graphify/skills/claude/references/update.md | 20 + .../skills/claw/references/extraction-spec.md | 2 + graphify/skills/claw/references/update.md | 20 + .../codex/references/extraction-spec.md | 2 + graphify/skills/codex/references/update.md | 20 + .../copilot/references/extraction-spec.md | 2 + graphify/skills/copilot/references/update.md | 20 + .../droid/references/extraction-spec.md | 2 + graphify/skills/droid/references/update.md | 20 + .../skills/kilo/references/extraction-spec.md | 2 + graphify/skills/kilo/references/update.md | 20 + .../skills/kiro/references/extraction-spec.md | 2 + graphify/skills/kiro/references/update.md | 20 + .../opencode/references/extraction-spec.md | 2 + graphify/skills/opencode/references/update.md | 20 + .../skills/pi/references/extraction-spec.md | 2 + graphify/skills/pi/references/update.md | 20 + .../skills/trae/references/extraction-spec.md | 2 + graphify/skills/trae/references/update.md | 20 + .../vscode/references/extraction-spec.md | 2 + graphify/skills/vscode/references/update.md | 20 + .../windows/references/extraction-spec.md | 2 + graphify/skills/windows/references/update.md | 20 + tests/test_skill_merge.py | 397 ++++++++++++++++++ .../expected/graphify__skill-agents.md | 76 ++-- .../skillgen/expected/graphify__skill-amp.md | 76 ++-- .../skillgen/expected/graphify__skill-claw.md | 74 ++-- .../expected/graphify__skill-codex.md | 70 +-- .../expected/graphify__skill-copilot.md | 74 ++-- .../expected/graphify__skill-droid.md | 76 ++-- .../skillgen/expected/graphify__skill-kilo.md | 74 ++-- .../skillgen/expected/graphify__skill-kiro.md | 74 ++-- .../expected/graphify__skill-opencode.md | 70 +-- tools/skillgen/expected/graphify__skill-pi.md | 74 ++-- .../skillgen/expected/graphify__skill-trae.md | 76 ++-- .../expected/graphify__skill-vscode.md | 78 ++-- .../expected/graphify__skill-windows.md | 74 ++-- tools/skillgen/expected/graphify__skill.md | 74 ++-- ...ls__agents__references__extraction-spec.md | 2 + ...ify__skills__agents__references__update.md | 20 + ...kills__amp__references__extraction-spec.md | 2 + ...aphify__skills__amp__references__update.md | 20 + ...ls__claude__references__extraction-spec.md | 2 + ...ify__skills__claude__references__update.md | 20 + ...ills__claw__references__extraction-spec.md | 2 + ...phify__skills__claw__references__update.md | 20 + ...lls__codex__references__extraction-spec.md | 2 + ...hify__skills__codex__references__update.md | 20 + ...s__copilot__references__extraction-spec.md | 2 + ...fy__skills__copilot__references__update.md | 20 + ...lls__droid__references__extraction-spec.md | 2 + ...hify__skills__droid__references__update.md | 20 + ...ills__kilo__references__extraction-spec.md | 2 + ...phify__skills__kilo__references__update.md | 20 + ...ills__kiro__references__extraction-spec.md | 2 + ...phify__skills__kiro__references__update.md | 20 + ...__opencode__references__extraction-spec.md | 2 + ...y__skills__opencode__references__update.md | 20 + ...skills__pi__references__extraction-spec.md | 2 + ...raphify__skills__pi__references__update.md | 20 + ...ills__trae__references__extraction-spec.md | 2 + ...phify__skills__trae__references__update.md | 20 + ...ls__vscode__references__extraction-spec.md | 2 + ...ify__skills__vscode__references__update.md | 20 + ...s__windows__references__extraction-spec.md | 2 + ...fy__skills__windows__references__update.md | 20 + tools/skillgen/fragments/core/core.md | 68 +-- .../dispatch/agent-tool-disk-powershell.md | 6 +- .../fragments/dispatch/agent-tool-disk.md | 6 +- .../fragments/dispatch/codex-agenttask.md | 2 +- .../fragments/dispatch/manual-paste.md | 10 +- .../fragments/dispatch/opencode-mention.md | 2 +- .../fragments/dispatch/task-tool-disk-trae.md | 8 +- .../fragments/dispatch/task-tool-disk.md | 8 +- .../shared/extraction-spec-compact.md | 2 + .../references/shared/extraction-spec.md | 2 + .../fragments/references/shared/update.md | 20 + 97 files changed, 2479 insertions(+), 927 deletions(-) create mode 100644 graphify/skill_merge.py create mode 100644 tests/test_skill_merge.py diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 3f7fade819..d1a3d247b6 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** > Uses the `Task` tool for parallel subagent dispatch. @@ -267,13 +295,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -281,35 +305,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -318,9 +328,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -332,9 +343,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 3f7fade819..d1a3d247b6 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** > Uses the `Task` tool for parallel subagent dispatch. @@ -267,13 +295,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -281,35 +305,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -318,9 +328,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -332,9 +343,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 2eca2a295f..6e1bf71172 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index a126527fd7..56e6852a56 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message (Codex)** > **Codex platform:** Uses `spawn_agent` + `wait_agent` + `close_agent` instead of the Agent tool. @@ -273,7 +301,7 @@ After all agents are dispatched, collect results sequentially in memory: result = wait_agent(handle); close_agent(handle) # repeat per handle ``` -Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. Codex collects in memory, so there are no per-chunk files on disk; the disk-based success checks in Step B3 do not apply — a chunk that returns invalid JSON is the failure signal instead. +Parse each result as JSON. Codex collects in memory, then the controller writes each response as JSON data to the absolute CHUNK_PATH returned by B1 for that agent's FILE_LIST. Do not concatenate results yet: every chunk must pass the same B3 ownership validation before caching or merge. A malformed response stops the run; a missing result stays missing in the current dispatch. Subagent prompt template: @@ -281,35 +309,21 @@ See `references/extraction-spec.md` for the compact subagent prompt (rules, node **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -318,9 +332,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -332,9 +347,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 2eca2a295f..6e1bf71172 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index b1eeb4925a..8739340d08 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** > Uses the `Task` tool for parallel subagent dispatch. @@ -267,13 +295,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -281,35 +305,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -318,9 +328,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -332,9 +343,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 387563de43..1f313a3393 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 2eca2a295f..6e1bf71172 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 144378e65a..fc4438e4d7 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message (OpenCode)** > **OpenCode platform:** Uses `@mention` dispatch instead of the Agent tool. All mentions in a single message run in parallel. @@ -268,7 +296,7 @@ Dispatch one `@mention` per chunk — ALL in the same response: @agent Chunk 2 of TOTAL_CHUNKS: [next chunk] ``` -Wait for all agents to return. Parse each response as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. If the `@agent` path cannot write chunk files, fall back to the serial path that writes each `graphify-out/.graphify_chunk_NN.json` before merge. +Wait for all agents to return. Parse each response as JSON and write it as data to its assigned CHUNK_PATH from B1, retaining that assignment's FILE_LIST. If an agent cannot write files, the controller writes its inline response to the same path. Do not accumulate directly into a semantic aggregate: B3 validates each current-run chunk before caching or merge. Subagent prompt template: @@ -276,35 +304,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -313,9 +327,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -327,9 +342,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 2eca2a295f..6e1bf71172 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 36706e3d7c..eff6a1f0a3 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** > Uses the `Task` tool for parallel subagent dispatch. @@ -268,13 +296,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -282,35 +306,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -319,9 +329,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -333,9 +344,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 6191a4ca6d..7c9a94f54e 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch subagents and paste their responses** > **No automated subagent tool:** this host has no parallel Agent/Task API, so @@ -264,15 +292,7 @@ Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-2 For each chunk of uncached files (20-25 per chunk), give a subagent the extraction prompt below. When it returns, paste its JSON response and write it to that chunk's file so Step B3 can collect it: -```bash -# After pasting a subagent's JSON for chunk N, save it (replace N and PASTED_JSON): -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -cat > "${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" <<'CHUNK_JSON' -PASTED_JSON -CHUNK_JSON -``` - -Repeat for every chunk. Each chunk's JSON must land in its own `graphify-out/.graphify_chunk_NN.json` before Step B3 runs. +Use the host's file-writing tool to save each parsed JSON response as data to its assigned absolute CHUNK_PATH from B1. Copy the matching FILE_LIST into the extraction prompt. Do not paste model output into an executable shell/Python command or construct a numbered filename. Every current-run response must pass the shared B3 ownership check before caching or merge. Subagent prompt template: @@ -280,35 +300,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -317,9 +323,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -331,9 +338,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 456244a246..b8557f1da2 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -278,12 +278,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n '@ | & (Get-Content graphify-out\.graphify_python) - ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```powershell +@' +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding="utf-8-sig").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +'@ | & (Get-Content graphify-out\.graphify_python) - +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -300,11 +328,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```powershell -$PROJECT_ROOT = (Get-Location).Path # cwd — where Part C globs graphify-out\ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: $CHUNK_PATH = Join-Path $PROJECT_ROOT "graphify-out\.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -312,35 +336,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```powershell @' -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding="utf-8")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding="utf-8") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding="utf-8") +print(f'Merged current dispatch: {new["input_tokens"]:,} in / {new["output_tokens"]:,} out tokens') '@ | & (Get-Content graphify-out\.graphify_python) - ``` @@ -349,9 +359,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each @' import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -363,9 +374,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: @' import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill.md b/graphify/skill.md index 2eca2a295f..6e1bf71172 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/graphify/skill_merge.py b/graphify/skill_merge.py new file mode 100644 index 0000000000..5c77ae883c --- /dev/null +++ b/graphify/skill_merge.py @@ -0,0 +1,179 @@ +"""Controller-owned chunk provenance checks for generated skill runbooks.""" +from __future__ import annotations + +import json +import re +from collections.abc import Iterable +from copy import deepcopy +from pathlib import Path +from uuid import uuid4 + +from graphify.cache import scope_semantic_result +from graphify.paths import write_json_atomic + +_MANIFEST = ".graphify_dispatch.json" +_CHUNK_NAME = re.compile(r"\.graphify_chunk_([0-9a-f]{32})_([0-9]+)\.json\Z") + + +def prepare_semantic_dispatch( + chunks: Iterable[Iterable[str | Path]], + *, + root: str | Path, + output_dir: str | Path = "graphify-out", +) -> list[dict]: + """Record exact FILE_LIST inputs and allocate fresh run-owned chunk paths. + + The UUID identifies output artifacts only; it never enters graph node IDs. + Inline-return hosts also write each result to its assigned path before B3. + """ + output = Path(output_dir).resolve() + run_id = uuid4().hex + assignments: dict[str, list[str]] = {} + dispatch: list[dict] = [] + for index, files in enumerate(chunks, 1): + if isinstance(files, (str, bytes)): + raise ValueError("Each semantic chunk must contain a list of file paths.") + paths = [] + for source in files: + if not isinstance(source, (str, Path)) or not str(source).strip(): + raise ValueError("Semantic FILE_LIST entries must be non-empty file paths.") + paths.append(str(source)) + if not paths: + raise ValueError("A semantic chunk cannot have an empty FILE_LIST.") + name = f".graphify_chunk_{run_id}_{index:02d}.json" + assignments[name] = paths + dispatch.append({"path": str(output / name), "files": paths}) + output.mkdir(parents=True, exist_ok=True) + write_json_atomic(output / _MANIFEST, { + "version": 1, "run_id": run_id, "root": str(Path(root).resolve()), + "chunks": assignments, + }, ensure_ascii=False) + return dispatch + + +def assert_semantic_scope( + result: dict, + *, + root: str | Path, + allowed_source_files: Iterable[str | Path], +) -> None: + """Refuse a chunk's foreign provenance without modifying its input. + + Use the cache's walked-path identity rules instead of another path normalizer. + Cross-chunk edge endpoints are allowed; their source_file must belong to the + referring chunk, and no foreign node definition may be supplied as a stub. + """ + for bucket in ("nodes", "edges", "hyperedges"): + items = result.get(bucket, [] if bucket == "hyperedges" else None) + if not isinstance(items, list): + raise ValueError(f"Semantic chunk {bucket} must be a list.") + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("source_file"), str): + raise ValueError(f"Every semantic {bucket} item needs an originating source_file.") + if not item["source_file"].strip(): + raise ValueError(f"Every semantic {bucket} item needs a non-empty source_file.") + checked = deepcopy(result) + dropped_files, count = scope_semantic_result( + checked, root=Path(root), allowed_source_files=allowed_source_files, + ) + if count: + raise ValueError( + f"Refusing semantic chunk: {count} item(s) outside its FILE_LIST: " + f"{sorted(dropped_files)!r}. Re-extract before caching or merging." + ) + + +def collect_semantic_dispatch(output_dir: str | Path = "graphify-out") -> dict: + """Validate every current-run chunk before aggregating or saving any result. + + Only controller-planned basenames are read. Stale glob matches are ignored, + and a missing current output can never reuse an older run's chunk number. + """ + output = Path(output_dir) + if (output / _MANIFEST).is_symlink(): + raise ValueError("Controller dispatch manifest must not be a symlink.") + manifest = json.loads((output / _MANIFEST).read_text(encoding="utf-8")) + run_id = manifest.get("run_id") + assignments = manifest.get("chunks") + if (manifest.get("version") != 1 or not isinstance(run_id, str) + or not re.fullmatch(r"[0-9a-f]{32}", run_id) + or not isinstance(assignments, dict) or not isinstance(manifest.get("root"), str)): + raise ValueError("Invalid controller dispatch manifest; prepare a new semantic run.") + chunks: list[dict] = [] + missing: list[str] = [] + for name, files in assignments.items(): + match = _CHUNK_NAME.fullmatch(name) if isinstance(name, str) else None + if (match is None or match.group(1) != run_id or not isinstance(files, list) + or not files or not all(isinstance(f, str) and f.strip() for f in files)): + raise ValueError("Chunk does not belong to the controller's current dispatch run.") + chunk_path = output / name + if chunk_path.is_symlink(): + raise ValueError(f"Semantic chunk {name!r} must not be a symlink.") + if not chunk_path.is_file(): + missing.append(name) + continue + result = json.loads(chunk_path.read_text(encoding="utf-8")) + if not isinstance(result, dict): + raise ValueError(f"Semantic chunk {name!r} must be a JSON object.") + assert_semantic_scope(result, root=manifest["root"], allowed_source_files=files) + chunks.append(result) + if missing and len(missing) * 2 > len(assignments): + raise ValueError(f"More than half the current semantic chunks are missing: {missing!r}") + merged: dict = { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, "missing_chunks": missing, + "dispatched_source_files": [source for files in assignments.values() for source in files], + } + for chunk in chunks: + for bucket in ("nodes", "edges", "hyperedges"): + merged[bucket].extend(chunk.get(bucket, [])) + for key in ("input_tokens", "output_tokens"): + merged[key] += int(chunk.get(key, 0)) + return merged + + +def load_skill_update_extraction( + semantic_files: Iterable[str | Path], + *, + root: str | Path, + prompt_file: str | Path, + output_dir: str | Path = "graphify-out", +) -> dict: + """Bind a destructive update to owned inputs, never a stale aggregate. + + Semantic cache records are read again by walked source identity and prompt + fingerprint, not from an earlier cached/semantic/extract sidecar. The AST + sidecar is kept separate: structural cross-file records have different + ownership semantics and must not be scoped as semantic stubs. + """ + from graphify.cache import _semantic_source_matcher, check_semantic_cache + + files = [str(f) for f in semantic_files] + fresh = collect_semantic_dispatch(output_dir) + assert_semantic_scope(fresh, root=root, allowed_source_files=files) + planned = fresh["dispatched_source_files"] + assert_semantic_scope({"nodes": [{"source_file": f} for f in planned], "edges": []}, + root=root, allowed_source_files=files) + source_identity, _ = _semantic_source_matcher(Path(root)) + dispatched = {source_identity(f) for f in planned} + # B3 has already saved fresh results. Do not read them a second time as cache + # hits, or repeat their edges; failed dispatched chunks must not replay cache. + cache_files = [f for f in files if source_identity(f) not in dispatched] + ast = json.loads((Path(output_dir) / ".graphify_ast.json").read_text(encoding="utf-8")) + nodes, edges, hyperedges, _ = check_semantic_cache(cache_files, root=Path(root), + prompt_file=prompt_file) + cached = {"nodes": nodes, "edges": edges, "hyperedges": hyperedges} + assert_semantic_scope(cached, root=root, allowed_source_files=files) + seen = set() + merged_nodes = [] + for node in ast.get("nodes", []) + fresh["nodes"] + nodes: + if node["id"] not in seen: + seen.add(node["id"]) + merged_nodes.append(node) + return { + "nodes": merged_nodes, + "edges": ast.get("edges", []) + fresh["edges"] + edges, + "hyperedges": fresh["hyperedges"] + hyperedges, + "input_tokens": fresh["input_tokens"], + "output_tokens": fresh["output_tokens"], + } diff --git a/graphify/skills/agents/references/extraction-spec.md b/graphify/skills/agents/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/agents/references/extraction-spec.md +++ b/graphify/skills/agents/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/amp/references/extraction-spec.md b/graphify/skills/amp/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/amp/references/extraction-spec.md +++ b/graphify/skills/amp/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/claude/references/extraction-spec.md b/graphify/skills/claude/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/claude/references/extraction-spec.md +++ b/graphify/skills/claude/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/claw/references/extraction-spec.md b/graphify/skills/claw/references/extraction-spec.md index 4b278b28d3..b04a302c2a 100644 --- a/graphify/skills/claw/references/extraction-spec.md +++ b/graphify/skills/claw/references/extraction-spec.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/codex/references/extraction-spec.md b/graphify/skills/codex/references/extraction-spec.md index 4b278b28d3..b04a302c2a 100644 --- a/graphify/skills/codex/references/extraction-spec.md +++ b/graphify/skills/codex/references/extraction-spec.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/copilot/references/extraction-spec.md b/graphify/skills/copilot/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/copilot/references/extraction-spec.md +++ b/graphify/skills/copilot/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/droid/references/extraction-spec.md b/graphify/skills/droid/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/droid/references/extraction-spec.md +++ b/graphify/skills/droid/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/kilo/references/extraction-spec.md b/graphify/skills/kilo/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/kilo/references/extraction-spec.md +++ b/graphify/skills/kilo/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/kiro/references/extraction-spec.md b/graphify/skills/kiro/references/extraction-spec.md index 4b278b28d3..b04a302c2a 100644 --- a/graphify/skills/kiro/references/extraction-spec.md +++ b/graphify/skills/kiro/references/extraction-spec.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/opencode/references/extraction-spec.md b/graphify/skills/opencode/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/opencode/references/extraction-spec.md +++ b/graphify/skills/opencode/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/pi/references/extraction-spec.md b/graphify/skills/pi/references/extraction-spec.md index 4b278b28d3..b04a302c2a 100644 --- a/graphify/skills/pi/references/extraction-spec.md +++ b/graphify/skills/pi/references/extraction-spec.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/trae/references/extraction-spec.md b/graphify/skills/trae/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/trae/references/extraction-spec.md +++ b/graphify/skills/trae/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/vscode/references/extraction-spec.md b/graphify/skills/vscode/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/vscode/references/extraction-spec.md +++ b/graphify/skills/vscode/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/graphify/skills/windows/references/extraction-spec.md b/graphify/skills/windows/references/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/graphify/skills/windows/references/extraction-spec.md +++ b/graphify/skills/windows/references/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tests/test_skill_merge.py b/tests/test_skill_merge.py new file mode 100644 index 0000000000..1ef93ae9bf --- /dev/null +++ b/tests/test_skill_merge.py @@ -0,0 +1,397 @@ +"""Skill-side ownership checks before cache writes and destructive replacement.""" +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from graphify.build import build_merge +from graphify.skill_merge import ( + assert_semantic_scope, + collect_semantic_dispatch, + prepare_semantic_dispatch, +) + + +def node(identifier, source): + return {"id": identifier, "label": identifier, "file_type": "document", + "source_file": str(source), "_origin": "semantic"} + + +def write_chunk(assignment, data): + Path(assignment["path"]).write_text(json.dumps(data), encoding="utf-8") + + +def test_foreign_stub_growth_cannot_replace_23_undispatched_nodes(tmp_path): + own, foreign = tmp_path / "A.md", tmp_path / "B.md" + own.write_text("Changed A") + foreign.write_text("Unchanged B") + output = tmp_path / "graphify-out" + assignment = prepare_semantic_dispatch([[own]], root=tmp_path, output_dir=output)[0] + saved = {"nodes": [node(f"b_rich_{i}", "B.md") for i in range(23)] + + [node("a_old", "A.md")], "links": [], "graph": {}, + "directed": False, "multigraph": False} + graph_path = output / "graph.json" + graph_path.write_text(json.dumps(saved), encoding="utf-8") + cache_path = output / "cache-marker.json" + cache_path.write_text('{"unchanged": true}', encoding="utf-8") + graph_before, cache_before = graph_path.read_bytes(), cache_path.read_bytes() + incoming = {"nodes": [node(f"a_new_{i}", own) for i in range(35)] + + [node("b_stub", foreign)], "edges": []} + write_chunk(assignment, incoming) + + # The old unguarded path loses B even though its total grows 24 -> 36. + unguarded = build_merge([incoming], graph_path, root=tmp_path, dedup=False) + assert unguarded.number_of_nodes() == 36 + assert not any(f"b_rich_{i}" in unguarded for i in range(23)) + with pytest.raises(ValueError, match="outside its FILE_LIST"): + collected = collect_semantic_dispatch(output) + build_merge([collected], graph_path, root=tmp_path, dedup=False) + cache_path.write_text("should never execute", encoding="utf-8") + assert graph_path.read_bytes() == graph_before + assert cache_path.read_bytes() == cache_before + + +def test_sibling_dispatch_does_not_grant_another_chunks_file(tmp_path): + a, b = tmp_path / "A.md", tmp_path / "B.md" + a.touch() + b.touch() + output = tmp_path / "graphify-out" + assignments = prepare_semantic_dispatch([[a], [b]], root=tmp_path, output_dir=output) + write_chunk(assignments[0], {"nodes": [node("foreign_stub", b)], "edges": []}) + write_chunk(assignments[1], {"nodes": [node("b_real", b)], "edges": []}) + with pytest.raises(ValueError, match="outside its FILE_LIST"): + collect_semantic_dispatch(output) + + +def test_owned_chunks_keep_complete_cross_chunk_edge_references(tmp_path): + a, b = tmp_path / "A.md", tmp_path / "B.md" + a.touch() + b.touch() + output = tmp_path / "graphify-out" + assignments = prepare_semantic_dispatch([[a], [b]], root=tmp_path, output_dir=output) + edge = {"source": "a_complete", "target": "b_complete", "relation": "references", + "confidence": "EXTRACTED", "source_file": str(a)} + write_chunk(assignments[0], {"nodes": [node("a_complete", a)], "edges": [edge]}) + write_chunk(assignments[1], {"nodes": [node("b_complete", b)], "edges": []}) + result = collect_semantic_dispatch(output) + assert [n["id"] for n in result["nodes"]] == ["a_complete", "b_complete"] + assert result["edges"] == [edge] + + +@pytest.mark.parametrize("bucket", ["nodes", "edges", "hyperedges"]) +def test_all_semantic_buckets_require_chunk_owned_provenance(tmp_path, bucket): + a, b = tmp_path / "A.md", tmp_path / "B.md" + a.touch() + b.touch() + result = {"nodes": [node("a_complete", a)], "edges": [], "hyperedges": []} + result[bucket].append({"id": "foreign", "source_file": str(b)}) + original = copy.deepcopy(result) + with pytest.raises(ValueError, match="outside its FILE_LIST"): + assert_semantic_scope(result, root=tmp_path, allowed_source_files=[a]) + assert result == original # the guard does not silently trim the result + + +def test_scope_uses_the_cache_path_alias_rules(tmp_path): + source = tmp_path / "pkg" / "A.md" + source.parent.mkdir() + source.touch() + result = {"nodes": [node("a_complete", "pkg\\A.md")], "edges": []} + original = copy.deepcopy(result) + assert_semantic_scope(result, root=tmp_path, allowed_source_files=[source]) + assert result == original + + +def test_old_run_glob_matches_never_enter_current_aggregation(tmp_path): + source = tmp_path / "A.md" + source.touch() + output = tmp_path / "graphify-out" + old = prepare_semantic_dispatch([[source]], root=tmp_path, output_dir=output)[0] + Path(old["path"]).write_text("invalid stale JSON", encoding="utf-8") + current = prepare_semantic_dispatch([[source]], root=tmp_path, output_dir=output)[0] + assert current["path"] != old["path"] + write_chunk(current, {"nodes": [node("fresh", source)], "edges": []}) + assert [n["id"] for n in collect_semantic_dispatch(output)["nodes"]] == ["fresh"] + + +def test_missing_current_result_never_reuses_an_old_chunk_number(tmp_path): + source = tmp_path / "A.md" + source.touch() + output = tmp_path / "graphify-out" + old = prepare_semantic_dispatch([[source]], root=tmp_path, output_dir=output)[0] + write_chunk(old, {"nodes": [node("old", source)], "edges": []}) + prepare_semantic_dispatch([[source]], root=tmp_path, output_dir=output) + with pytest.raises(ValueError, match="current semantic chunks are missing"): + collect_semantic_dispatch(output) + + +@pytest.mark.parametrize("name", ["../escape.json", ".graphify_chunk_" + "0" * 32 + "_01.json"]) +def test_dispatch_manifest_cannot_read_unowned_paths(tmp_path, name): + source = tmp_path / "A.md" + source.touch() + output = tmp_path / "graphify-out" + prepare_semantic_dispatch([[source]], root=tmp_path, output_dir=output) + manifest_path = output / ".graphify_dispatch.json" + manifest = json.loads(manifest_path.read_text()) + manifest["chunks"] = {name: [str(source)]} + manifest_path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="current dispatch run"): + collect_semantic_dispatch(output) + + +def test_missing_provenance_is_not_guessed(tmp_path): + result = {"nodes": [{"id": "stub"}], "edges": []} + with pytest.raises(ValueError, match="originating source_file"): + assert_semantic_scope(result, root=tmp_path, allowed_source_files=[]) + + +def runbook_python(text, marker): + """Execute the actual generated runbook block, with no shell or agent calls.""" + import re + + section = text[text.index(marker):] + block = re.search(r'```bash\n.*?-c "\n(.*?)\n"\n```', section, re.DOTALL) + assert block is not None + return block.group(1).replace('\\"', '"').replace('IS_DIRECTED', 'False') + + +def test_generated_controller_refuses_before_aggregate_cache_and_update_writes(tmp_path, monkeypatch): + from tools.skillgen import gen + + arts = gen.render_all(gen.load_platforms(), only="claude") + core = next(a.content for a in arts if a.path == "graphify/skill.md") + update = next(a.content for a in arts if a.path.endswith("/update.md")) + monkeypatch.chdir(tmp_path) + output = tmp_path / "graphify-out" + output.mkdir() + a, b = tmp_path / "A.md", tmp_path / "B.md" + a.touch() + b.touch() + (output / ".graphify_root").write_text(str(tmp_path)) + (output / ".graphify_detect.json").write_text(json.dumps({"files": {"document": [str(a)]}})) + (output / ".graphify_uncached.txt").write_text(str(a)) + exec(runbook_python(core, "Before dispatch, persist"), {}) + manifest = json.loads((output / ".graphify_dispatch.json").read_text()) + name = next(iter(manifest["chunks"])) + write_chunk({"path": output / name}, {"nodes": [node("foreign_stub", b)], "edges": []}) + saved = {"nodes": [node(f"b_rich_{i}", "B.md") for i in range(23)] + + [node("a_old", "A.md")], "links": [], "graph": {}, + "directed": False, "multigraph": False} + paths = [output / "graph.json", output / ".graphify_semantic_new.json", + output / ".graphify_manifest.json", output / ".graphify_extract.json"] + paths[0].write_text(json.dumps(saved)) + paths[1].write_text('{"previous": "aggregate"}') + paths[2].write_text('{"previous": "manifest"}') + paths[3].write_text(json.dumps({"nodes": [node("foreign_stub", b)], "edges": []})) + before = {p: p.read_bytes() for p in paths} + (output / ".graphify_incremental.json").write_text(json.dumps({ + "new_files": {"document": [str(a)]}, "files": {"document": [str(a), str(b)]}, + })) + for text, marker in [(core, "**Step B3"), (core, "Save new results to cache."), + (update, "# Load new extraction and incremental state")]: + # For the update snippet the marker is inside the block, so select its preceding fence. + if text is update: + marker = "\nThen:\n" + with pytest.raises(ValueError, match="outside its FILE_LIST"): + exec(runbook_python(text, marker).replace("INPUT_PATH", str(tmp_path)), {}) + assert {p: p.read_bytes() for p in paths} == before + + +def test_generated_incremental_code_only_skips_old_dispatch(tmp_path, monkeypatch): + from tools.skillgen import gen + + update = next(a.content for a in gen.render_all(gen.load_platforms(), only="claude") + if a.path.endswith("/update.md")) + monkeypatch.chdir(tmp_path) + output = tmp_path / "graphify-out" + output.mkdir() + (tmp_path / "A.py").touch() + (output / "graph.json").write_text(json.dumps({ + "nodes": [node("a_old", "A.py")], "links": [], "graph": {}, + "directed": False, "multigraph": False, + })) + (output / ".graphify_extract.json").write_text(json.dumps({ + "nodes": [node("a_new", "A.py")], "edges": [], + })) + (output / ".graphify_incremental.json").write_text(json.dumps({ + "new_files": {"code": [str(tmp_path / "A.py")]}, + "files": {"code": [str(tmp_path / "A.py")]}, + })) + (output / ".graphify_dispatch.json").write_text("invalid stale manifest") + exec(runbook_python(update, "\nThen:\n").replace("INPUT_PATH", str(tmp_path)), {}) + merged = json.loads((output / ".graphify_extract.json").read_text()) + assert [n["id"] for n in merged["nodes"]] == ["a_new"] + + +def test_empty_dispatch_replaces_previous_run_without_old_chunk_replay(tmp_path): + output = tmp_path / "graphify-out" + source = tmp_path / "A.md" + source.touch() + old = prepare_semantic_dispatch([[source]], root=tmp_path, output_dir=output)[0] + write_chunk(old, {"nodes": [node("old", source)], "edges": []}) + assert prepare_semantic_dispatch([], root=tmp_path, output_dir=output) == [] + assert collect_semantic_dispatch(output)["nodes"] == [] + + +def test_missing_dispatch_manifest_is_a_failure(tmp_path): + with pytest.raises(FileNotFoundError): + collect_semantic_dispatch(tmp_path) + + +def test_generated_cached_assembly_reads_owned_chunks_not_old_aggregate(tmp_path, monkeypatch): + from tools.skillgen import gen + + core = next(a.content for a in gen.render_all(gen.load_platforms(), only="claude") + if a.path == "graphify/skill.md") + monkeypatch.chdir(tmp_path) + output = tmp_path / "graphify-out" + a, b = tmp_path / "A.md", tmp_path / "B.md" + a.touch() + b.touch() + assignment = prepare_semantic_dispatch([[a]], root=tmp_path, output_dir=output)[0] + write_chunk(assignment, {"nodes": [node("a_current", a)], "edges": []}) + (output / ".graphify_cached.json").write_text(json.dumps({ + "nodes": [node("b_cached", b)], "edges": [], + })) + (output / ".graphify_semantic_new.json").write_text(json.dumps({ + "nodes": [node("b_foreign_old_aggregate", b)], "edges": [], + })) + exec(runbook_python(core, "Merge cached + new results into"), {}) + merged = json.loads((output / ".graphify_semantic.json").read_text()) + assert [n["id"] for n in merged["nodes"]] == ["b_cached", "a_current"] + before = (output / ".graphify_semantic.json").read_bytes() + write_chunk(assignment, {"nodes": [node("b_foreign", b)], "edges": []}) + with pytest.raises(ValueError, match="outside its FILE_LIST"): + exec(runbook_python(core, "Merge cached + new results into"), {}) + assert (output / ".graphify_semantic.json").read_bytes() == before + + +def test_generated_update_reports_prefix_but_keeps_known_external_reference(tmp_path, monkeypatch, capsys): + from tools.skillgen import gen + + update = next(a.content for a in gen.render_all(gen.load_platforms(), only="claude") + if a.path.endswith("/update.md")) + monkeypatch.chdir(tmp_path) + output = tmp_path / "graphify-out" + a, b = tmp_path / "A.md", tmp_path / "B.md" + a.touch() + b.touch() + assignment = prepare_semantic_dispatch([[a]], root=tmp_path, output_dir=output)[0] + edges = [{"source": "a_complete", "target": target, "relation": "references", + "confidence": "EXTRACTED", "source_file": str(a)} + for target in ("b_complete", "b")] + fresh = {"nodes": [node("a_complete", a)], "edges": edges} + write_chunk(assignment, fresh) + (output / "graph.json").write_text(json.dumps({ + "nodes": [node("b_complete", "B.md")], "links": [], "graph": {}, + "directed": False, "multigraph": False, + })) + graph_before = (output / "graph.json").read_bytes() + (output / ".graphify_ast.json").write_text(json.dumps({"nodes": [], "edges": []})) + (output / ".graphify_extract.json").write_text(json.dumps(fresh)) + (output / ".graphify_incremental.json").write_text(json.dumps({ + "new_files": {"document": [str(a)]}, "files": {"document": [str(a), str(b)]}, + })) + exec(runbook_python(update, "\nThen:\n").replace("INPUT_PATH", str(tmp_path)), {}) + message = capsys.readouterr().out + assert "Unresolved incoming references before persistence:" in message + assert "dangling_endpoint_edges: 1" in message + merged = json.loads((output / ".graphify_extract.json").read_text()) + assert len(merged["edges"]) == 1 + assert {merged["edges"][0]["source"], merged["edges"][0]["target"]} == {"a_complete", "b_complete"} + assert (output / "graph.json").read_bytes() == graph_before + + +def test_generated_update_ignores_stale_payload_and_loads_only_owned_cache(tmp_path, monkeypatch): + from graphify.cache import save_semantic_cache + from tools.skillgen import gen + + update = next(a.content for a in gen.render_all(gen.load_platforms(), only="claude") + if a.path.endswith("/update.md")) + monkeypatch.chdir(tmp_path) + output = tmp_path / "graphify-out" + a, b, c = (tmp_path / name for name in ("A.md", "B.md", "C.md")) + for source in (a, b, c): + source.write_text(source.name) + spec = tmp_path / "spec.md" + spec.write_text("test prompt") + cached_edge = {"source": "c_cached", "target": "b_rich_0", "relation": "references", + "confidence": "EXTRACTED", "source_file": str(c)} + save_semantic_cache([node("c_cached", c)], [cached_edge], [], root=tmp_path, prompt_file=spec) + assignment = prepare_semantic_dispatch([[a]], root=tmp_path, output_dir=output)[0] + fresh = {"nodes": [node(f"a_new_{i}", a) for i in range(35)], "edges": []} + write_chunk(assignment, fresh) + (output / "graph.json").write_text(json.dumps({ + "nodes": [node(f"b_rich_{i}", "B.md") for i in range(23)] + [node("a_old", "A.md")], + "links": [], "graph": {}, "directed": False, "multigraph": False, + })) + graph_before = (output / "graph.json").read_bytes() + # Structural records stay outside the semantic ownership check. + (output / ".graphify_ast.json").write_text(json.dumps({ + "nodes": [node("x_structural", "X.py")], "edges": [], + })) + stale = {**fresh, "nodes": fresh["nodes"] + [node("b_foreign_stub", b)]} + for name in (".graphify_extract.json", ".graphify_semantic.json", + ".graphify_semantic_new.json", ".graphify_cached.json"): + (output / name).write_text(json.dumps(stale)) + (output / ".graphify_incremental.json").write_text(json.dumps({ + "new_files": {"document": [str(a), str(c)]}, + "files": {"document": [str(a), str(b), str(c)]}, + })) + exec(runbook_python(update, "\nThen:\n").replace("INPUT_PATH", str(tmp_path)) + .replace("SPEC_PATH", str(spec)), {}) + merged = json.loads((output / ".graphify_extract.json").read_text()) + ids = {n["id"] for n in merged["nodes"]} + assert all(f"b_rich_{i}" in ids for i in range(23)) + assert "b_foreign_stub" not in ids + assert {"a_new_0", "c_cached", "x_structural"} <= ids + assert any({e["source"], e["target"]} == {"c_cached", "b_rich_0"} for e in merged["edges"]) + assert (output / "graph.json").read_bytes() == graph_before + + +def test_update_cannot_reuse_a_valid_dispatch_for_unchanged_sources(tmp_path): + from graphify.skill_merge import load_skill_update_extraction + + a, b = tmp_path / "A.md", tmp_path / "B.md" + a.touch() + b.touch() + output = tmp_path / "graphify-out" + old = prepare_semantic_dispatch([[b]], root=tmp_path, output_dir=output)[0] + write_chunk(old, {"nodes": [node("b_old", b)], "edges": []}) + with pytest.raises(ValueError, match="outside its FILE_LIST"): + load_skill_update_extraction([a], root=tmp_path, prompt_file="missing-spec", output_dir=output) + + +def test_update_does_not_replay_just_saved_fresh_edges_as_cache_hits(tmp_path): + from graphify.cache import save_semantic_cache + from graphify.skill_merge import load_skill_update_extraction + + source = tmp_path / "A.md" + source.write_text("A") + output = tmp_path / "graphify-out" + assignment = prepare_semantic_dispatch([[source]], root=tmp_path, output_dir=output)[0] + edge = {"source": "a_one", "target": "a_two", "relation": "references", + "confidence": "EXTRACTED", "source_file": str(source)} + fresh = {"nodes": [node("a_one", source), node("a_two", source)], "edges": [edge]} + write_chunk(assignment, fresh) + (output / ".graphify_ast.json").write_text(json.dumps({"nodes": [], "edges": []})) + spec = tmp_path / "spec.md" + spec.write_text("prompt") + save_semantic_cache(fresh["nodes"], fresh["edges"], [], root=tmp_path, prompt_file=spec) + merged = load_skill_update_extraction([source], root=tmp_path, prompt_file=spec, output_dir=output) + assert merged["edges"] == [edge] + + +def test_empty_foreign_dispatch_plan_cannot_borrow_current_changed_scope(tmp_path): + from graphify.skill_merge import load_skill_update_extraction + + a, b = tmp_path / "A.md", tmp_path / "B.md" + a.touch() + b.touch() + output = tmp_path / "graphify-out" + old = prepare_semantic_dispatch([[b]], root=tmp_path, output_dir=output)[0] + write_chunk(old, {"nodes": [], "edges": []}) + with pytest.raises(ValueError, match="outside its FILE_LIST"): + load_skill_update_extraction([a], root=tmp_path, prompt_file="missing-spec", output_dir=output) diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 3f7fade819..d1a3d247b6 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** > Uses the `Task` tool for parallel subagent dispatch. @@ -267,13 +295,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -281,35 +305,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -318,9 +328,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -332,9 +343,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 3f7fade819..d1a3d247b6 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** > Uses the `Task` tool for parallel subagent dispatch. @@ -267,13 +295,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -281,35 +305,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -318,9 +328,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -332,9 +343,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 2eca2a295f..6e1bf71172 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index a126527fd7..56e6852a56 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message (Codex)** > **Codex platform:** Uses `spawn_agent` + `wait_agent` + `close_agent` instead of the Agent tool. @@ -273,7 +301,7 @@ After all agents are dispatched, collect results sequentially in memory: result = wait_agent(handle); close_agent(handle) # repeat per handle ``` -Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. Codex collects in memory, so there are no per-chunk files on disk; the disk-based success checks in Step B3 do not apply — a chunk that returns invalid JSON is the failure signal instead. +Parse each result as JSON. Codex collects in memory, then the controller writes each response as JSON data to the absolute CHUNK_PATH returned by B1 for that agent's FILE_LIST. Do not concatenate results yet: every chunk must pass the same B3 ownership validation before caching or merge. A malformed response stops the run; a missing result stays missing in the current dispatch. Subagent prompt template: @@ -281,35 +309,21 @@ See `references/extraction-spec.md` for the compact subagent prompt (rules, node **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -318,9 +332,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -332,9 +347,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 2eca2a295f..6e1bf71172 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index b1eeb4925a..8739340d08 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** > Uses the `Task` tool for parallel subagent dispatch. @@ -267,13 +295,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -281,35 +305,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -318,9 +328,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -332,9 +343,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 387563de43..1f313a3393 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 2eca2a295f..6e1bf71172 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 144378e65a..fc4438e4d7 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message (OpenCode)** > **OpenCode platform:** Uses `@mention` dispatch instead of the Agent tool. All mentions in a single message run in parallel. @@ -268,7 +296,7 @@ Dispatch one `@mention` per chunk — ALL in the same response: @agent Chunk 2 of TOTAL_CHUNKS: [next chunk] ``` -Wait for all agents to return. Parse each response as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. If the `@agent` path cannot write chunk files, fall back to the serial path that writes each `graphify-out/.graphify_chunk_NN.json` before merge. +Wait for all agents to return. Parse each response as JSON and write it as data to its assigned CHUNK_PATH from B1, retaining that assignment's FILE_LIST. If an agent cannot write files, the controller writes its inline response to the same path. Do not accumulate directly into a semantic aggregate: B3 validates each current-run chunk before caching or merge. Subagent prompt template: @@ -276,35 +304,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -313,9 +327,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -327,9 +342,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 2eca2a295f..6e1bf71172 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 36706e3d7c..eff6a1f0a3 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** > Uses the `Task` tool for parallel subagent dispatch. @@ -268,13 +296,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -282,35 +306,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -319,9 +329,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -333,9 +344,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 6191a4ca6d..7c9a94f54e 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch subagents and paste their responses** > **No automated subagent tool:** this host has no parallel Agent/Task API, so @@ -264,15 +292,7 @@ Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-2 For each chunk of uncached files (20-25 per chunk), give a subagent the extraction prompt below. When it returns, paste its JSON response and write it to that chunk's file so Step B3 can collect it: -```bash -# After pasting a subagent's JSON for chunk N, save it (replace N and PASTED_JSON): -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -cat > "${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" <<'CHUNK_JSON' -PASTED_JSON -CHUNK_JSON -``` - -Repeat for every chunk. Each chunk's JSON must land in its own `graphify-out/.graphify_chunk_NN.json` before Step B3 runs. +Use the host's file-writing tool to save each parsed JSON response as data to its assigned absolute CHUNK_PATH from B1. Copy the matching FILE_LIST into the extraction prompt. Do not paste model output into an executable shell/Python command or construct a numbered filename. Every current-run response must pass the shared B3 ownership check before caching or merge. Subagent prompt template: @@ -280,35 +300,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -317,9 +323,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -331,9 +338,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 456244a246..b8557f1da2 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -278,12 +278,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n '@ | & (Get-Content graphify-out\.graphify_python) - ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```powershell +@' +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding="utf-8-sig").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +'@ | & (Get-Content graphify-out\.graphify_python) - +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -300,11 +328,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```powershell -$PROJECT_ROOT = (Get-Location).Path # cwd — where Part C globs graphify-out\ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: $CHUNK_PATH = Join-Path $PROJECT_ROOT "graphify-out\.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -312,35 +336,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```powershell @' -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding="utf-8")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding="utf-8") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding="utf-8") +print(f'Merged current dispatch: {new["input_tokens"]:,} in / {new["output_tokens"]:,} out tokens') '@ | & (Get-Content graphify-out\.graphify_python) - ``` @@ -349,9 +359,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each @' import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -363,9 +374,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: @' import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 2eca2a295f..6e1bf71172 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -250,12 +250,40 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. @@ -272,11 +300,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: @@ -284,35 +308,21 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -321,9 +331,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -335,9 +346,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md index 4b278b28d3..b04a302c2a 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md index 4b278b28d3..b04a302c2a 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md index 4b278b28d3..b04a302c2a 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md index 4b278b28d3..b04a302c2a 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index c527a12563..628c15c90a 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -203,45 +203,59 @@ print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files n " ``` -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, prepare an empty dispatch in B1, skip B2, and run B3 to merge cached results without replaying an earlier run. **Step B1 - Split into chunks** Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Before dispatch, persist the exact FILE_LIST for every chunk using the controller helper below. This is data, not text supplied by an extraction agent. CHUNK_PATH comes from its returned `path`; FILE_LIST comes from the matching `files`. Never invent numbered paths or glob old chunk files. The run identifier belongs only to artifact filenames, never to entity IDs. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from collections import defaultdict +from pathlib import Path +from graphify.skill_merge import prepare_semantic_dispatch + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +images = set(detect.get('files', {}).get('image', [])) +files = Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() +groups = defaultdict(list) +chunks = [] +for source in files: + if source in images: + chunks.append([source]) + elif source: + groups[str(Path(source).parent)].append(source) +for directory in sorted(groups): + group = sorted(groups[directory]) + chunks.extend(group[start:start + 22] for start in range(0, len(group), 22)) +root = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip() +dispatch = prepare_semantic_dispatch(chunks, root=root) +print(json.dumps(dispatch, ensure_ascii=False)) +" +``` + @@DISPATCH@@ **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. Write inline responses to their assigned CHUNK_PATH as JSON data. After each call completes, copy real token counts from its `usage` field into that chunk's JSON; placeholder zeros are not usage evidence. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Validate **all current-run chunks before any cache or merge write**. A foreign `source_file`, missing provenance, malformed JSON, or an unowned artifact stops the run: re-extract the affected chunk instead of trimming its records or overwriting a valid graph. Missing outputs produce a warning; if more than half are missing, stop and re-run with a writable general-purpose agent. The controller reads only this run's manifest, so old chunk files never fill a missing result. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json, glob +import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +new = collect_semantic_dispatch() +for missing in new['missing_chunks']: + print(f'Warning: current chunk {missing} missing; re-run with a writable general-purpose agent.') +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(new, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged current dispatch: {new[\"input_tokens\"]:,} in / {new[\"output_tokens\"]:,} out tokens') " ``` @@ -250,9 +264,10 @@ Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache +from graphify.skill_merge import collect_semantic_dispatch from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') @@ -264,9 +279,10 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.skill_merge import collect_semantic_dispatch cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = collect_semantic_dispatch() all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) diff --git a/tools/skillgen/fragments/dispatch/agent-tool-disk-powershell.md b/tools/skillgen/fragments/dispatch/agent-tool-disk-powershell.md index 54f2f54e3d..53bcd3dcdc 100644 --- a/tools/skillgen/fragments/dispatch/agent-tool-disk-powershell.md +++ b/tools/skillgen/fragments/dispatch/agent-tool-disk-powershell.md @@ -14,11 +14,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```powershell -$PROJECT_ROOT = (Get-Location).Path # cwd — where Part C globs graphify-out\ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: $CHUNK_PATH = Join-Path $PROJECT_ROOT "graphify-out\.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: diff --git a/tools/skillgen/fragments/dispatch/agent-tool-disk.md b/tools/skillgen/fragments/dispatch/agent-tool-disk.md index 9e07ca5db3..53bcd3dcdc 100644 --- a/tools/skillgen/fragments/dispatch/agent-tool-disk.md +++ b/tools/skillgen/fragments/dispatch/agent-tool-disk.md @@ -14,11 +14,7 @@ All three in one message. Not three separate messages. Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: diff --git a/tools/skillgen/fragments/dispatch/codex-agenttask.md b/tools/skillgen/fragments/dispatch/codex-agenttask.md index 1fd2f6a1a7..0503eed2d0 100644 --- a/tools/skillgen/fragments/dispatch/codex-agenttask.md +++ b/tools/skillgen/fragments/dispatch/codex-agenttask.md @@ -15,7 +15,7 @@ After all agents are dispatched, collect results sequentially in memory: result = wait_agent(handle); close_agent(handle) # repeat per handle ``` -Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. Codex collects in memory, so there are no per-chunk files on disk; the disk-based success checks in Step B3 do not apply — a chunk that returns invalid JSON is the failure signal instead. +Parse each result as JSON. Codex collects in memory, then the controller writes each response as JSON data to the absolute CHUNK_PATH returned by B1 for that agent's FILE_LIST. Do not concatenate results yet: every chunk must pass the same B3 ownership validation before caching or merge. A malformed response stops the run; a missing result stays missing in the current dispatch. Subagent prompt template: diff --git a/tools/skillgen/fragments/dispatch/manual-paste.md b/tools/skillgen/fragments/dispatch/manual-paste.md index b4120cc301..b4ca2d031f 100644 --- a/tools/skillgen/fragments/dispatch/manual-paste.md +++ b/tools/skillgen/fragments/dispatch/manual-paste.md @@ -6,15 +6,7 @@ For each chunk of uncached files (20-25 per chunk), give a subagent the extraction prompt below. When it returns, paste its JSON response and write it to that chunk's file so Step B3 can collect it: -```bash -# After pasting a subagent's JSON for chunk N, save it (replace N and PASTED_JSON): -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -cat > "${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" <<'CHUNK_JSON' -PASTED_JSON -CHUNK_JSON -``` - -Repeat for every chunk. Each chunk's JSON must land in its own `graphify-out/.graphify_chunk_NN.json` before Step B3 runs. +Use the host's file-writing tool to save each parsed JSON response as data to its assigned absolute CHUNK_PATH from B1. Copy the matching FILE_LIST into the extraction prompt. Do not paste model output into an executable shell/Python command or construct a numbered filename. Every current-run response must pass the shared B3 ownership check before caching or merge. Subagent prompt template: diff --git a/tools/skillgen/fragments/dispatch/opencode-mention.md b/tools/skillgen/fragments/dispatch/opencode-mention.md index f87f5fb51b..3017731eb0 100644 --- a/tools/skillgen/fragments/dispatch/opencode-mention.md +++ b/tools/skillgen/fragments/dispatch/opencode-mention.md @@ -10,7 +10,7 @@ Dispatch one `@mention` per chunk — ALL in the same response: @agent Chunk 2 of TOTAL_CHUNKS: [next chunk] ``` -Wait for all agents to return. Parse each response as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. If the `@agent` path cannot write chunk files, fall back to the serial path that writes each `graphify-out/.graphify_chunk_NN.json` before merge. +Wait for all agents to return. Parse each response as JSON and write it as data to its assigned CHUNK_PATH from B1, retaining that assignment's FILE_LIST. If an agent cannot write files, the controller writes its inline response to the same path. Do not accumulate directly into a semantic aggregate: B3 validates each current-run chunk before caching or merge. Subagent prompt template: diff --git a/tools/skillgen/fragments/dispatch/task-tool-disk-trae.md b/tools/skillgen/fragments/dispatch/task-tool-disk-trae.md index 5ee3a4a62f..c61a4b45dc 100644 --- a/tools/skillgen/fragments/dispatch/task-tool-disk-trae.md +++ b/tools/skillgen/fragments/dispatch/task-tool-disk-trae.md @@ -10,13 +10,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: diff --git a/tools/skillgen/fragments/dispatch/task-tool-disk.md b/tools/skillgen/fragments/dispatch/task-tool-disk.md index d3c9caae6e..bcdab5f6bb 100644 --- a/tools/skillgen/fragments/dispatch/task-tool-disk.md +++ b/tools/skillgen/fragments/dispatch/task-tool-disk.md @@ -9,13 +9,9 @@ Pass the extraction prompt as the task description: Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +Each subagent writes its result to its assigned CHUNK_PATH. Collect results as each `Task` completes and parse each as JSON. -CHUNK_PATH must be an **absolute** path — derive it before dispatching: -```bash -PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392) -# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" -``` +CHUNK_PATH is the absolute `path` returned by B1 for this exact FILE_LIST. Copy both from the same controller assignment; do not construct a numbered filename. Subagent prompt template: diff --git a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md index 4b278b28d3..b04a302c2a 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md @@ -28,4 +28,6 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. + +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. ``` diff --git a/tools/skillgen/fragments/references/shared/extraction-spec.md b/tools/skillgen/fragments/references/shared/extraction-spec.md index 388df7674f..1692ebfdc0 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec.md @@ -65,6 +65,8 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +FILE_LIST is an ownership boundary: every node, edge, and hyperedge must originate in one of this chunk's files. Never define a stub node with another file's source_file, even if that file belongs to another dispatched chunk or is referenced here; doing so can replace its complete persisted extraction. An edge originating in your file may reference a known external node using its exact complete ID without defining that node. Edge endpoints and hyperedge members must be exact complete node IDs, never abbreviated paths or guessed IDs. A supplied ID stem/prefix is a naming hint; do not use it as an endpoint unless it was explicitly supplied as the complete ID of a known node. If the exact referenced entity ID is unknown, omit that unresolved relationship instead of inventing an endpoint. The incremental merge reports unresolved incoming endpoints before persistence. Never put the dispatch run identifier in entity IDs. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index 3632fd4126..9f49d5c0d9 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -92,6 +92,17 @@ from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +# Bind the actual destructive payload to current dispatch + owned cache + AST. +# Code-only/deletion-only updates skip B entirely and must not read an old manifest. +# SPEC_PATH is the same installed extraction-spec.md used in B0/B3. +semantic_types = ('document', 'paper', 'image') +semantic_files = [f for t in semantic_types for f in incremental.get('new_files', {}).get(t, [])] +video_changed = bool(incremental.get('new_files', {}).get('video')) +if video_changed: + semantic_files += json.loads(Path('graphify-out/.graphify_transcripts.json').read_text(encoding=\"utf-8\")) +if semantic_files or video_changed: + from graphify.skill_merge import load_skill_update_extraction + new_extraction = load_skill_update_extraction(semantic_files, root='INPUT_PATH', prompt_file='SPEC_PATH') deleted = list(incremental.get('deleted_files', [])) # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in @@ -118,6 +129,15 @@ G = build_merge( ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') +# Diagnose raw incoming references against the final node set BEFORE rewriting +# extraction with surviving edges, which would hide unresolved prefixes (#3947). +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +references = {**new_extraction, 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)]} +summary = diagnose_extraction(references, directed=IS_DIRECTED, root='INPUT_PATH') +if summary['dangling_endpoint_edges'] or summary['missing_endpoint_edges']: + print('Unresolved incoming references before persistence:') + print(format_diagnostic_report(summary)) + # Write merged result back to .graphify_extract.json so Step 4 sees the full graph merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], From be3b45463c98e977b08e0035ddbf14a655d2e559 Mon Sep 17 00:00:00 2001 From: Huanyi Xie <74809554+xiehuanyi@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:43:09 +0300 Subject: [PATCH 2/2] fix(skill): scope fresh semantic chunks with the existing guard Keep fresh source ownership checks before caching and update merges, on current v8. AI assistance: Codex drafted the patch and regression tests. Co-Authored-By: Codex --- graphify/skill-agents.md | 7 ++ graphify/skill-amp.md | 7 ++ graphify/skill-claw.md | 7 ++ graphify/skill-codex.md | 7 ++ graphify/skill-copilot.md | 7 ++ graphify/skill-droid.md | 7 ++ graphify/skill-kilo.md | 7 ++ graphify/skill-kiro.md | 7 ++ graphify/skill-opencode.md | 7 ++ graphify/skill-pi.md | 7 ++ graphify/skill-trae.md | 7 ++ graphify/skill-vscode.md | 7 ++ graphify/skill-windows.md | 7 ++ graphify/skill.md | 7 ++ .../agents/references/extraction-spec.md | 5 + graphify/skills/agents/references/update.md | 6 + .../skills/amp/references/extraction-spec.md | 5 + graphify/skills/amp/references/update.md | 6 + .../claude/references/extraction-spec.md | 5 + graphify/skills/claude/references/update.md | 6 + .../skills/claw/references/extraction-spec.md | 3 + graphify/skills/claw/references/update.md | 6 + .../codex/references/extraction-spec.md | 3 + graphify/skills/codex/references/update.md | 6 + .../copilot/references/extraction-spec.md | 5 + graphify/skills/copilot/references/update.md | 6 + .../droid/references/extraction-spec.md | 5 + graphify/skills/droid/references/update.md | 6 + .../skills/kilo/references/extraction-spec.md | 5 + graphify/skills/kilo/references/update.md | 6 + .../skills/kiro/references/extraction-spec.md | 3 + graphify/skills/kiro/references/update.md | 6 + .../opencode/references/extraction-spec.md | 5 + graphify/skills/opencode/references/update.md | 6 + .../skills/pi/references/extraction-spec.md | 3 + graphify/skills/pi/references/update.md | 6 + .../skills/trae/references/extraction-spec.md | 5 + graphify/skills/trae/references/update.md | 6 + .../vscode/references/extraction-spec.md | 5 + graphify/skills/vscode/references/update.md | 6 + .../windows/references/extraction-spec.md | 5 + graphify/skills/windows/references/update.md | 6 + tests/test_skill_semantic_scope.py | 107 ++++++++++++++++++ .../expected/graphify__skill-agents.md | 7 ++ .../skillgen/expected/graphify__skill-amp.md | 7 ++ .../skillgen/expected/graphify__skill-claw.md | 7 ++ .../expected/graphify__skill-codex.md | 7 ++ .../expected/graphify__skill-copilot.md | 7 ++ .../expected/graphify__skill-droid.md | 7 ++ .../skillgen/expected/graphify__skill-kilo.md | 7 ++ .../skillgen/expected/graphify__skill-kiro.md | 7 ++ .../expected/graphify__skill-opencode.md | 7 ++ tools/skillgen/expected/graphify__skill-pi.md | 7 ++ .../skillgen/expected/graphify__skill-trae.md | 7 ++ .../expected/graphify__skill-vscode.md | 7 ++ .../expected/graphify__skill-windows.md | 7 ++ tools/skillgen/expected/graphify__skill.md | 7 ++ ...ls__agents__references__extraction-spec.md | 5 + ...ify__skills__agents__references__update.md | 6 + ...kills__amp__references__extraction-spec.md | 5 + ...aphify__skills__amp__references__update.md | 6 + ...ls__claude__references__extraction-spec.md | 5 + ...ify__skills__claude__references__update.md | 6 + ...ills__claw__references__extraction-spec.md | 3 + ...phify__skills__claw__references__update.md | 6 + ...lls__codex__references__extraction-spec.md | 3 + ...hify__skills__codex__references__update.md | 6 + ...s__copilot__references__extraction-spec.md | 5 + ...fy__skills__copilot__references__update.md | 6 + ...lls__droid__references__extraction-spec.md | 5 + ...hify__skills__droid__references__update.md | 6 + ...ills__kilo__references__extraction-spec.md | 5 + ...phify__skills__kilo__references__update.md | 6 + ...ills__kiro__references__extraction-spec.md | 3 + ...phify__skills__kiro__references__update.md | 6 + ...__opencode__references__extraction-spec.md | 5 + ...y__skills__opencode__references__update.md | 6 + ...skills__pi__references__extraction-spec.md | 3 + ...raphify__skills__pi__references__update.md | 6 + ...ills__trae__references__extraction-spec.md | 5 + ...phify__skills__trae__references__update.md | 6 + ...ls__vscode__references__extraction-spec.md | 5 + ...ify__skills__vscode__references__update.md | 6 + ...s__windows__references__extraction-spec.md | 5 + ...fy__skills__windows__references__update.md | 6 + tools/skillgen/fragments/core/core.md | 7 ++ .../shared/extraction-spec-compact.md | 3 + .../references/shared/extraction-spec.md | 5 + .../fragments/references/shared/update.md | 6 + 89 files changed, 616 insertions(+) create mode 100644 tests/test_skill_semantic_scope.py diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index f09e56ca8e..745d133a53 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -298,12 +298,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index f09e56ca8e..745d133a53 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -298,12 +298,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 612da0090a..6b5c28795e 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index d826d76e61..4bb279c0ea 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -298,12 +298,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 612da0090a..6b5c28795e 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index ada369b7dd..98b1c68546 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -298,12 +298,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 9b043233fa..ec6c178806 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 612da0090a..6b5c28795e 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 5cb74e81ae..d7f3b459eb 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -293,12 +293,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 612da0090a..6b5c28795e 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 2037e539f6..ba4813e845 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -299,12 +299,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 9002650e4a..423ea55834 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -297,12 +297,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 1246089d50..626613469c 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -329,12 +329,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal @' import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding="utf-8")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skill.md b/graphify/skill.md index 612da0090a..6b5c28795e 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/graphify/skills/agents/references/extraction-spec.md b/graphify/skills/agents/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/agents/references/extraction-spec.md +++ b/graphify/skills/agents/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/amp/references/extraction-spec.md b/graphify/skills/amp/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/amp/references/extraction-spec.md +++ b/graphify/skills/amp/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/claude/references/extraction-spec.md b/graphify/skills/claude/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/claude/references/extraction-spec.md +++ b/graphify/skills/claude/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/claw/references/extraction-spec.md b/graphify/skills/claw/references/extraction-spec.md index 4b278b28d3..d1f284b6a1 100644 --- a/graphify/skills/claw/references/extraction-spec.md +++ b/graphify/skills/claw/references/extraction-spec.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/codex/references/extraction-spec.md b/graphify/skills/codex/references/extraction-spec.md index 4b278b28d3..d1f284b6a1 100644 --- a/graphify/skills/codex/references/extraction-spec.md +++ b/graphify/skills/codex/references/extraction-spec.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/copilot/references/extraction-spec.md b/graphify/skills/copilot/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/copilot/references/extraction-spec.md +++ b/graphify/skills/copilot/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/droid/references/extraction-spec.md b/graphify/skills/droid/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/droid/references/extraction-spec.md +++ b/graphify/skills/droid/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/kilo/references/extraction-spec.md b/graphify/skills/kilo/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/kilo/references/extraction-spec.md +++ b/graphify/skills/kilo/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/kiro/references/extraction-spec.md b/graphify/skills/kiro/references/extraction-spec.md index 4b278b28d3..d1f284b6a1 100644 --- a/graphify/skills/kiro/references/extraction-spec.md +++ b/graphify/skills/kiro/references/extraction-spec.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/opencode/references/extraction-spec.md b/graphify/skills/opencode/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/opencode/references/extraction-spec.md +++ b/graphify/skills/opencode/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/pi/references/extraction-spec.md b/graphify/skills/pi/references/extraction-spec.md index 4b278b28d3..d1f284b6a1 100644 --- a/graphify/skills/pi/references/extraction-spec.md +++ b/graphify/skills/pi/references/extraction-spec.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/trae/references/extraction-spec.md b/graphify/skills/trae/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/trae/references/extraction-spec.md +++ b/graphify/skills/trae/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/vscode/references/extraction-spec.md b/graphify/skills/vscode/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/vscode/references/extraction-spec.md +++ b/graphify/skills/vscode/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/graphify/skills/windows/references/extraction-spec.md b/graphify/skills/windows/references/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/graphify/skills/windows/references/extraction-spec.md +++ b/graphify/skills/windows/references/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 3632fd4126..2817ad2c1e 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tests/test_skill_semantic_scope.py b/tests/test_skill_semantic_scope.py new file mode 100644 index 0000000000..ccf6ed2380 --- /dev/null +++ b/tests/test_skill_semantic_scope.py @@ -0,0 +1,107 @@ +"""The generated skill must scope fresh chunks before destructive update merges.""" +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +from graphify.build import build_from_json +from graphify.export import to_json +from tools.skillgen import gen + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _python_blocks(body: str, start: str, end: str) -> list[str]: + section = body.split(start, 1)[1].split(end, 1)[0] + sources = [] + for block in re.findall(r"```bash\n(.*?)```", section, re.S): + match = re.search(r' -c "\n(.*)\n"\s*$', block, re.S) + if match: + sources.append(match.group(1).replace('\\"', '"')) + return sources + + +@pytest.mark.parametrize("source_form", ["relative", "absolute"]) +def test_generated_update_retains_undispatched_rich_nodes(tmp_path, source_form): + """A growing graph must not hide replacement of rich foreign nodes by a stub.""" + artifacts = gen.render_all(gen.load_platforms(), only="claude") + body = next(a.content for a in artifacts if a.path == "graphify/skill.md") + update = next(a.content for a in artifacts + if a.path == "graphify/skills/claude/references/update.md") + b3 = _python_blocks(body, "**Step B3", "#### Part C") + part_c = _python_blocks(body, "#### Part C", "\n### ")[:1] + merge = next(src for src in _python_blocks(update, "Then:", "Then run Steps") + if "G = build_merge(" in src) + + corpus = tmp_path / "corpus" + corpus.mkdir() + for filename in ("fresh.md", "foreign.md", "cached.md"): + (corpus / filename).write_text(f"# {filename}\n", encoding="utf-8") + spec = tmp_path / "spec.md" + spec.write_text("# Spec\n", encoding="utf-8") + output = tmp_path / "graphify-out" + output.mkdir() + + def node(nid, filename): + source = str(corpus / filename) if source_form == "absolute" else filename + return {"id": nid, "label": nid, "file_type": "document", "source_file": source} + + rich = [node(f"foreign_rich_{i}", "foreign.md") for i in range(23)] + cached = node("cached_real", "cached.md") + prior = build_from_json({"nodes": rich + [cached], "edges": []}, root=corpus) + assert to_json(prior, {}, output / "graph.json") + fresh = [node(f"fresh_{i}", "fresh.md") for i in range(35)] + payload = { + "nodes": fresh + [node("foreign_stub", "foreign.md")], + "edges": [ + {"source": "fresh_0", "target": "foreign_rich_0", "relation": "references", + "source_file": fresh[0]["source_file"], "confidence": "EXTRACTED"}, + {"source": "fresh_0", "target": "foreign_stub", "relation": "references", + "source_file": fresh[0]["source_file"], "confidence": "INFERRED"}, + ], + "hyperedges": [{"id": "foreign_h", "nodes": ["fresh_0", "fresh_1"], + "source_file": node("unused", "foreign.md")["source_file"]}], + "input_tokens": 10, "output_tokens": 5, + } + files = {"document": [str(corpus / name) + for name in ("fresh.md", "foreign.md", "cached.md")]} + sidecars = { + ".graphify_chunk_01.json": payload, + ".graphify_cached.json": {"nodes": [cached], "edges": [], "hyperedges": []}, + ".graphify_ast.json": {"nodes": [], "edges": []}, + ".graphify_incremental.json": {"files": files, "deleted_files": [], + "new_files": {"document": [str(corpus / "fresh.md")]}}, + } + for filename, data in sidecars.items(): + (output / filename).write_text(json.dumps(data), encoding="utf-8") + (output / ".graphify_uncached.txt").write_text( + str(corpus / "fresh.md") + "\n", encoding="utf-8") + logs = [] + for source in b3 + part_c + [merge]: + source = source.replace("INPUT_PATH", corpus.as_posix()).replace( + "SPEC_PATH", spec.as_posix()).replace("IS_DIRECTED", "False") + result = subprocess.run( + [sys.executable, "-c", source], cwd=tmp_path, check=True, + capture_output=True, text=True, + env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}, + ) + logs.append(result.stdout) + + merged = json.loads((output / ".graphify_extract.json").read_text(encoding="utf-8")) + assert {n["id"] for n in merged["nodes"]} == { + n["id"] for n in rich + [cached] + fresh + } + assert {(e["source"], e["target"]) for e in merged["edges"]} == { + ("fresh_0", "foreign_rich_0") + } + assert merged["hyperedges"] == [] + fresh_result = json.loads( + (output / ".graphify_semantic_new.json").read_text(encoding="utf-8")) + assert {n["id"] for n in fresh_result["nodes"]} == {n["id"] for n in fresh} + assert "foreign.md" in "".join(logs) and "out-of-scope" in "".join(logs) diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index f09e56ca8e..745d133a53 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -298,12 +298,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index f09e56ca8e..745d133a53 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -298,12 +298,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 612da0090a..6b5c28795e 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index d826d76e61..4bb279c0ea 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -298,12 +298,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 612da0090a..6b5c28795e 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index ada369b7dd..98b1c68546 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -298,12 +298,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 9b043233fa..ec6c178806 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 612da0090a..6b5c28795e 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 5cb74e81ae..d7f3b459eb 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -293,12 +293,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 612da0090a..6b5c28795e 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 2037e539f6..ba4813e845 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -299,12 +299,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 9002650e4a..423ea55834 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -297,12 +297,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 1246089d50..626613469c 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -329,12 +329,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal @' import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding="utf-8")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 612da0090a..6b5c28795e 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -301,12 +301,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md index 4b278b28d3..d1f284b6a1 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md index 4b278b28d3..d1f284b6a1 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md index 4b278b28d3..d1f284b6a1 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md index 4b278b28d3..d1f284b6a1 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index 412535088c..7ef91b1643 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -230,12 +230,19 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.cache import scope_semantic_result chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + # source_file is a destructive replacement key during --update. Scope the + # fresh result BEFORE both cache writes and assembly with cached/AST nodes. + dropped_files, dropped_count = scope_semantic_result(d, root=Path('INPUT_PATH'), allowed_source_files=uncached) + if dropped_count: + print(f'[graphify scope] {c}: dropped {dropped_count} out-of-scope items from {sorted(dropped_files)}') all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) diff --git a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md index 4b278b28d3..d1f284b6a1 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md @@ -28,4 +28,7 @@ Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +Never create stub nodes for files outside FILE_LIST. Use edge-only references +attributed to the originating FILE_LIST file and the complete known target ID; +a stem or ID prefix is never a valid target. Do not guess or invent a target node. ``` diff --git a/tools/skillgen/fragments/references/shared/extraction-spec.md b/tools/skillgen/fragments/references/shared/extraction-spec.md index 388df7674f..f0f6a341b6 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec.md @@ -65,6 +65,11 @@ Generate the extraction JSON matching this schema exactly: source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +Never create a stub node for a file outside FILE_LIST. Express a reference to an +external entity as an edge attributed to the originating FILE_LIST file, using +the complete known target ID. A stem or ID prefix alone is never a valid target +ID; do not guess one or invent a node to make a reference resolve. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index 3632fd4126..2817ad2c1e 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -82,6 +82,12 @@ fi Then: +For semantic changes, Step B3's collection commands apply the existing +`scope_semantic_result` guard to fresh semantic chunks using the files dispatched +from `.graphify_uncached.txt`, before caching or mixing them with cached/AST +results. `source_file` is a destructive replacement key: an undispatched file's +stub must never reach `build_merge` and replace that file's existing contribution. + ```bash $(cat graphify-out/.graphify_python) -c " import json