From 382b669481ccaa3fd9a27b749a4963a25c59807a Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 19 Jun 2026 15:37:11 +0100 Subject: [PATCH] Fix remaining #1392 skill bugs: --directed propagation, content-only semantic scope, cache staleness, video update, transcribe robustness Closes the non-crash tier of #1392 in the Claude-path skill fragments: - #6/#7: build_from_json/build_merge now take directed=IS_DIRECTED in Step 4, Step 5 rebuild, and the --update merge/diff, with prose telling the agent to substitute IS_DIRECTED like INPUT_PATH (a --directed run no longer silently rebuilds undirected and collapses reciprocal edges) - #10: semantic extraction only flattens document/paper/image, not code (AST already covers code) so subagents stop re-reading every source file - #12: .graphify_cached.json is deleted on a cache miss so Part C never merges a stale cache from a prior run - #11: --update now transcribes changed video files and moves transcripts to documents before the semantic pipeline - #4/#5/#23: transcribe writes via write_text (no shell redirect), uses GRAPHIFY_WHISPER_MODEL/PROMPT env, status to stderr - #2/#3: add-watch and exports use $(cat graphify-out/.graphify_python) explicitly; MCP Desktop config documents the absolute interpreter path - #21: extraction-spec example id namespaced (auth_session_validatetoken) - #22: query term split keeps tokens >= 3 chars aider/devin monoliths are pinned by the roundtrip invariant and excluded; their own Step 1 already instructs replacing python3 with the resolved interpreter. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/skill-amp.md | 15 +++++++++++---- graphify/skill-claw.md | 15 +++++++++++---- graphify/skill-codex.md | 15 +++++++++++---- graphify/skill-copilot.md | 15 +++++++++++---- graphify/skill-droid.md | 15 +++++++++++---- graphify/skill-kilo.md | 15 +++++++++++---- graphify/skill-kiro.md | 15 +++++++++++---- graphify/skill-opencode.md | 15 +++++++++++---- graphify/skill-pi.md | 15 +++++++++++---- graphify/skill-trae.md | 15 +++++++++++---- graphify/skill-vscode.md | 15 +++++++++++---- graphify/skill-windows.md | 15 +++++++++++---- graphify/skill.md | 15 +++++++++++---- graphify/skills/amp/references/add-watch.md | 2 +- graphify/skills/amp/references/exports.md | 6 +++--- .../skills/amp/references/extraction-spec.md | 2 +- graphify/skills/amp/references/query.md | 2 +- graphify/skills/amp/references/transcribe.md | 16 ++++++++++------ graphify/skills/amp/references/update.md | 8 ++++++-- graphify/skills/claude/references/add-watch.md | 2 +- graphify/skills/claude/references/exports.md | 6 +++--- .../skills/claude/references/extraction-spec.md | 2 +- graphify/skills/claude/references/query.md | 2 +- graphify/skills/claude/references/transcribe.md | 16 ++++++++++------ graphify/skills/claude/references/update.md | 8 ++++++-- graphify/skills/claw/references/add-watch.md | 2 +- graphify/skills/claw/references/exports.md | 6 +++--- .../skills/claw/references/extraction-spec.md | 2 +- graphify/skills/claw/references/query.md | 2 +- graphify/skills/claw/references/transcribe.md | 16 ++++++++++------ graphify/skills/claw/references/update.md | 8 ++++++-- graphify/skills/codex/references/add-watch.md | 2 +- graphify/skills/codex/references/exports.md | 6 +++--- .../skills/codex/references/extraction-spec.md | 2 +- graphify/skills/codex/references/query.md | 2 +- graphify/skills/codex/references/transcribe.md | 16 ++++++++++------ graphify/skills/codex/references/update.md | 8 ++++++-- graphify/skills/copilot/references/add-watch.md | 2 +- graphify/skills/copilot/references/exports.md | 6 +++--- .../skills/copilot/references/extraction-spec.md | 2 +- graphify/skills/copilot/references/query.md | 2 +- graphify/skills/copilot/references/transcribe.md | 16 ++++++++++------ graphify/skills/copilot/references/update.md | 8 ++++++-- graphify/skills/droid/references/add-watch.md | 2 +- graphify/skills/droid/references/exports.md | 6 +++--- .../skills/droid/references/extraction-spec.md | 2 +- graphify/skills/droid/references/query.md | 2 +- graphify/skills/droid/references/transcribe.md | 16 ++++++++++------ graphify/skills/droid/references/update.md | 8 ++++++-- graphify/skills/kilo/references/add-watch.md | 2 +- graphify/skills/kilo/references/exports.md | 6 +++--- .../skills/kilo/references/extraction-spec.md | 2 +- graphify/skills/kilo/references/query.md | 2 +- graphify/skills/kilo/references/transcribe.md | 16 ++++++++++------ graphify/skills/kilo/references/update.md | 8 ++++++-- graphify/skills/kiro/references/add-watch.md | 2 +- graphify/skills/kiro/references/exports.md | 6 +++--- .../skills/kiro/references/extraction-spec.md | 2 +- graphify/skills/kiro/references/query.md | 2 +- graphify/skills/kiro/references/transcribe.md | 16 ++++++++++------ graphify/skills/kiro/references/update.md | 8 ++++++-- graphify/skills/opencode/references/add-watch.md | 2 +- graphify/skills/opencode/references/exports.md | 6 +++--- .../opencode/references/extraction-spec.md | 2 +- graphify/skills/opencode/references/query.md | 2 +- .../skills/opencode/references/transcribe.md | 16 ++++++++++------ graphify/skills/opencode/references/update.md | 8 ++++++-- graphify/skills/pi/references/add-watch.md | 2 +- graphify/skills/pi/references/exports.md | 6 +++--- graphify/skills/pi/references/extraction-spec.md | 2 +- graphify/skills/pi/references/query.md | 2 +- graphify/skills/pi/references/transcribe.md | 16 ++++++++++------ graphify/skills/pi/references/update.md | 8 ++++++-- graphify/skills/trae/references/add-watch.md | 2 +- graphify/skills/trae/references/exports.md | 6 +++--- .../skills/trae/references/extraction-spec.md | 2 +- graphify/skills/trae/references/query.md | 2 +- graphify/skills/trae/references/transcribe.md | 16 ++++++++++------ graphify/skills/trae/references/update.md | 8 ++++++-- graphify/skills/vscode/references/add-watch.md | 2 +- graphify/skills/vscode/references/exports.md | 6 +++--- .../skills/vscode/references/extraction-spec.md | 2 +- graphify/skills/vscode/references/query.md | 2 +- graphify/skills/vscode/references/transcribe.md | 16 ++++++++++------ graphify/skills/vscode/references/update.md | 8 ++++++-- graphify/skills/windows/references/add-watch.md | 2 +- graphify/skills/windows/references/exports.md | 6 +++--- .../skills/windows/references/extraction-spec.md | 2 +- graphify/skills/windows/references/query.md | 2 +- graphify/skills/windows/references/transcribe.md | 16 ++++++++++------ graphify/skills/windows/references/update.md | 8 ++++++-- tools/skillgen/expected/graphify__skill-amp.md | 15 +++++++++++---- tools/skillgen/expected/graphify__skill-claw.md | 15 +++++++++++---- tools/skillgen/expected/graphify__skill-codex.md | 15 +++++++++++---- .../skillgen/expected/graphify__skill-copilot.md | 15 +++++++++++---- tools/skillgen/expected/graphify__skill-droid.md | 15 +++++++++++---- tools/skillgen/expected/graphify__skill-kilo.md | 15 +++++++++++---- tools/skillgen/expected/graphify__skill-kiro.md | 15 +++++++++++---- .../expected/graphify__skill-opencode.md | 15 +++++++++++---- tools/skillgen/expected/graphify__skill-pi.md | 15 +++++++++++---- tools/skillgen/expected/graphify__skill-trae.md | 15 +++++++++++---- .../skillgen/expected/graphify__skill-vscode.md | 15 +++++++++++---- .../skillgen/expected/graphify__skill-windows.md | 15 +++++++++++---- tools/skillgen/expected/graphify__skill.md | 15 +++++++++++---- ...aphify__skills__amp__references__add-watch.md | 2 +- ...graphify__skills__amp__references__exports.md | 6 +++--- ...__skills__amp__references__extraction-spec.md | 2 +- .../graphify__skills__amp__references__query.md | 2 +- ...phify__skills__amp__references__transcribe.md | 16 ++++++++++------ .../graphify__skills__amp__references__update.md | 8 ++++++-- ...ify__skills__claude__references__add-watch.md | 2 +- ...phify__skills__claude__references__exports.md | 6 +++--- ...kills__claude__references__extraction-spec.md | 2 +- ...raphify__skills__claude__references__query.md | 2 +- ...fy__skills__claude__references__transcribe.md | 16 ++++++++++------ ...aphify__skills__claude__references__update.md | 8 ++++++-- ...phify__skills__claw__references__add-watch.md | 2 +- ...raphify__skills__claw__references__exports.md | 6 +++--- ..._skills__claw__references__extraction-spec.md | 2 +- .../graphify__skills__claw__references__query.md | 2 +- ...hify__skills__claw__references__transcribe.md | 16 ++++++++++------ ...graphify__skills__claw__references__update.md | 8 ++++++-- ...hify__skills__codex__references__add-watch.md | 2 +- ...aphify__skills__codex__references__exports.md | 6 +++--- ...skills__codex__references__extraction-spec.md | 2 +- ...graphify__skills__codex__references__query.md | 2 +- ...ify__skills__codex__references__transcribe.md | 16 ++++++++++------ ...raphify__skills__codex__references__update.md | 8 ++++++-- ...fy__skills__copilot__references__add-watch.md | 2 +- ...hify__skills__copilot__references__exports.md | 6 +++--- ...ills__copilot__references__extraction-spec.md | 2 +- ...aphify__skills__copilot__references__query.md | 2 +- ...y__skills__copilot__references__transcribe.md | 16 ++++++++++------ ...phify__skills__copilot__references__update.md | 8 ++++++-- ...hify__skills__droid__references__add-watch.md | 2 +- ...aphify__skills__droid__references__exports.md | 6 +++--- ...skills__droid__references__extraction-spec.md | 2 +- ...graphify__skills__droid__references__query.md | 2 +- ...ify__skills__droid__references__transcribe.md | 16 ++++++++++------ ...raphify__skills__droid__references__update.md | 8 ++++++-- ...phify__skills__kilo__references__add-watch.md | 2 +- ...raphify__skills__kilo__references__exports.md | 6 +++--- ..._skills__kilo__references__extraction-spec.md | 2 +- .../graphify__skills__kilo__references__query.md | 2 +- ...hify__skills__kilo__references__transcribe.md | 16 ++++++++++------ ...graphify__skills__kilo__references__update.md | 8 ++++++-- ...phify__skills__kiro__references__add-watch.md | 2 +- ...raphify__skills__kiro__references__exports.md | 6 +++--- ..._skills__kiro__references__extraction-spec.md | 2 +- .../graphify__skills__kiro__references__query.md | 2 +- ...hify__skills__kiro__references__transcribe.md | 16 ++++++++++------ ...graphify__skills__kiro__references__update.md | 8 ++++++-- ...y__skills__opencode__references__add-watch.md | 2 +- ...ify__skills__opencode__references__exports.md | 6 +++--- ...lls__opencode__references__extraction-spec.md | 2 +- ...phify__skills__opencode__references__query.md | 2 +- ...__skills__opencode__references__transcribe.md | 16 ++++++++++------ ...hify__skills__opencode__references__update.md | 8 ++++++-- ...raphify__skills__pi__references__add-watch.md | 2 +- .../graphify__skills__pi__references__exports.md | 6 +++--- ...y__skills__pi__references__extraction-spec.md | 2 +- .../graphify__skills__pi__references__query.md | 2 +- ...aphify__skills__pi__references__transcribe.md | 16 ++++++++++------ .../graphify__skills__pi__references__update.md | 8 ++++++-- ...phify__skills__trae__references__add-watch.md | 2 +- ...raphify__skills__trae__references__exports.md | 6 +++--- ..._skills__trae__references__extraction-spec.md | 2 +- .../graphify__skills__trae__references__query.md | 2 +- ...hify__skills__trae__references__transcribe.md | 16 ++++++++++------ ...graphify__skills__trae__references__update.md | 8 ++++++-- ...ify__skills__vscode__references__add-watch.md | 2 +- ...phify__skills__vscode__references__exports.md | 6 +++--- ...kills__vscode__references__extraction-spec.md | 2 +- ...raphify__skills__vscode__references__query.md | 2 +- ...fy__skills__vscode__references__transcribe.md | 16 ++++++++++------ ...aphify__skills__vscode__references__update.md | 8 ++++++-- ...fy__skills__windows__references__add-watch.md | 2 +- ...hify__skills__windows__references__exports.md | 6 +++--- ...ills__windows__references__extraction-spec.md | 2 +- ...aphify__skills__windows__references__query.md | 2 +- ...y__skills__windows__references__transcribe.md | 16 ++++++++++------ ...phify__skills__windows__references__update.md | 8 ++++++-- tools/skillgen/fragments/core/core.md | 15 +++++++++++---- .../fragments/references/query/default.md | 2 +- .../fragments/references/shared/add-watch.md | 2 +- .../fragments/references/shared/exports.md | 6 +++--- .../references/shared/extraction-spec-compact.md | 2 +- .../references/shared/extraction-spec.md | 2 +- .../fragments/references/shared/transcribe.md | 16 ++++++++++------ .../fragments/references/shared/update.md | 8 ++++++-- 190 files changed, 892 insertions(+), 487 deletions(-) diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 5740924..8cea9f3 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -374,7 +381,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -392,7 +399,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -455,7 +462,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 9872ae6..3df00d5 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index af9ce49..ee64830 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -374,7 +381,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -392,7 +399,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -455,7 +462,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 9872ae6..3df00d5 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 80c5675..38f2781 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -374,7 +381,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -392,7 +399,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -455,7 +462,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index fdc0bb7..6ccb172 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 9872ae6..3df00d5 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index d300a0e..aa2e9a1 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -369,7 +376,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -387,7 +394,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -450,7 +457,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 9872ae6..3df00d5 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 20b8b36..2c0a574 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -375,7 +382,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -393,7 +400,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -456,7 +463,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 87044fb..2166d08 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -373,7 +380,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -391,7 +398,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -454,7 +461,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index d0680f2..64d2cbc 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -241,12 +241,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -399,7 +406,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -417,7 +424,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -480,7 +487,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill.md b/graphify/skill.md index 9872ae6..3df00d5 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skills/amp/references/add-watch.md b/graphify/skills/amp/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/amp/references/add-watch.md +++ b/graphify/skills/amp/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/amp/references/exports.md b/graphify/skills/amp/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/amp/references/exports.md +++ b/graphify/skills/amp/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/amp/references/extraction-spec.md b/graphify/skills/amp/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/amp/references/extraction-spec.md +++ b/graphify/skills/amp/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/amp/references/query.md b/graphify/skills/amp/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/amp/references/query.md +++ b/graphify/skills/amp/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/amp/references/transcribe.md b/graphify/skills/amp/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/amp/references/transcribe.md +++ b/graphify/skills/amp/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/claude/references/add-watch.md b/graphify/skills/claude/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/claude/references/add-watch.md +++ b/graphify/skills/claude/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/claude/references/exports.md b/graphify/skills/claude/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/claude/references/exports.md +++ b/graphify/skills/claude/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/claude/references/extraction-spec.md b/graphify/skills/claude/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/claude/references/extraction-spec.md +++ b/graphify/skills/claude/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/claude/references/query.md b/graphify/skills/claude/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/claude/references/query.md +++ b/graphify/skills/claude/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/claude/references/transcribe.md b/graphify/skills/claude/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/claude/references/transcribe.md +++ b/graphify/skills/claude/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/claw/references/add-watch.md b/graphify/skills/claw/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/claw/references/add-watch.md +++ b/graphify/skills/claw/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/claw/references/exports.md b/graphify/skills/claw/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/claw/references/exports.md +++ b/graphify/skills/claw/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/claw/references/extraction-spec.md b/graphify/skills/claw/references/extraction-spec.md index 58aee32..3e5a1a3 100644 --- a/graphify/skills/claw/references/extraction-spec.md +++ b/graphify/skills/claw/references/extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/claw/references/query.md b/graphify/skills/claw/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/claw/references/query.md +++ b/graphify/skills/claw/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/claw/references/transcribe.md b/graphify/skills/claw/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/claw/references/transcribe.md +++ b/graphify/skills/claw/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/codex/references/add-watch.md b/graphify/skills/codex/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/codex/references/add-watch.md +++ b/graphify/skills/codex/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/codex/references/exports.md b/graphify/skills/codex/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/codex/references/exports.md +++ b/graphify/skills/codex/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/codex/references/extraction-spec.md b/graphify/skills/codex/references/extraction-spec.md index 58aee32..3e5a1a3 100644 --- a/graphify/skills/codex/references/extraction-spec.md +++ b/graphify/skills/codex/references/extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/codex/references/query.md b/graphify/skills/codex/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/codex/references/query.md +++ b/graphify/skills/codex/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/codex/references/transcribe.md b/graphify/skills/codex/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/codex/references/transcribe.md +++ b/graphify/skills/codex/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/copilot/references/add-watch.md b/graphify/skills/copilot/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/copilot/references/add-watch.md +++ b/graphify/skills/copilot/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/copilot/references/exports.md b/graphify/skills/copilot/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/copilot/references/exports.md +++ b/graphify/skills/copilot/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/copilot/references/extraction-spec.md b/graphify/skills/copilot/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/copilot/references/extraction-spec.md +++ b/graphify/skills/copilot/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/copilot/references/query.md b/graphify/skills/copilot/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/copilot/references/query.md +++ b/graphify/skills/copilot/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/copilot/references/transcribe.md b/graphify/skills/copilot/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/copilot/references/transcribe.md +++ b/graphify/skills/copilot/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/droid/references/add-watch.md b/graphify/skills/droid/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/droid/references/add-watch.md +++ b/graphify/skills/droid/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/droid/references/exports.md b/graphify/skills/droid/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/droid/references/exports.md +++ b/graphify/skills/droid/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/droid/references/extraction-spec.md b/graphify/skills/droid/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/droid/references/extraction-spec.md +++ b/graphify/skills/droid/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/droid/references/query.md b/graphify/skills/droid/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/droid/references/query.md +++ b/graphify/skills/droid/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/droid/references/transcribe.md b/graphify/skills/droid/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/droid/references/transcribe.md +++ b/graphify/skills/droid/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/kilo/references/add-watch.md b/graphify/skills/kilo/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/kilo/references/add-watch.md +++ b/graphify/skills/kilo/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/kilo/references/exports.md b/graphify/skills/kilo/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/kilo/references/exports.md +++ b/graphify/skills/kilo/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/kilo/references/extraction-spec.md b/graphify/skills/kilo/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/kilo/references/extraction-spec.md +++ b/graphify/skills/kilo/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/kilo/references/query.md b/graphify/skills/kilo/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/kilo/references/query.md +++ b/graphify/skills/kilo/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/kilo/references/transcribe.md b/graphify/skills/kilo/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/kilo/references/transcribe.md +++ b/graphify/skills/kilo/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/kiro/references/add-watch.md b/graphify/skills/kiro/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/kiro/references/add-watch.md +++ b/graphify/skills/kiro/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/kiro/references/exports.md b/graphify/skills/kiro/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/kiro/references/exports.md +++ b/graphify/skills/kiro/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/kiro/references/extraction-spec.md b/graphify/skills/kiro/references/extraction-spec.md index 58aee32..3e5a1a3 100644 --- a/graphify/skills/kiro/references/extraction-spec.md +++ b/graphify/skills/kiro/references/extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/kiro/references/query.md b/graphify/skills/kiro/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/kiro/references/query.md +++ b/graphify/skills/kiro/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/kiro/references/transcribe.md b/graphify/skills/kiro/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/kiro/references/transcribe.md +++ b/graphify/skills/kiro/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/opencode/references/add-watch.md b/graphify/skills/opencode/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/opencode/references/add-watch.md +++ b/graphify/skills/opencode/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/opencode/references/exports.md b/graphify/skills/opencode/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/opencode/references/exports.md +++ b/graphify/skills/opencode/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/opencode/references/extraction-spec.md b/graphify/skills/opencode/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/opencode/references/extraction-spec.md +++ b/graphify/skills/opencode/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/opencode/references/query.md b/graphify/skills/opencode/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/opencode/references/query.md +++ b/graphify/skills/opencode/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/opencode/references/transcribe.md b/graphify/skills/opencode/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/opencode/references/transcribe.md +++ b/graphify/skills/opencode/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/pi/references/add-watch.md b/graphify/skills/pi/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/pi/references/add-watch.md +++ b/graphify/skills/pi/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/pi/references/exports.md b/graphify/skills/pi/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/pi/references/exports.md +++ b/graphify/skills/pi/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/pi/references/extraction-spec.md b/graphify/skills/pi/references/extraction-spec.md index 58aee32..3e5a1a3 100644 --- a/graphify/skills/pi/references/extraction-spec.md +++ b/graphify/skills/pi/references/extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/pi/references/query.md b/graphify/skills/pi/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/pi/references/query.md +++ b/graphify/skills/pi/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/pi/references/transcribe.md b/graphify/skills/pi/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/pi/references/transcribe.md +++ b/graphify/skills/pi/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/trae/references/add-watch.md b/graphify/skills/trae/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/trae/references/add-watch.md +++ b/graphify/skills/trae/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/trae/references/exports.md b/graphify/skills/trae/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/trae/references/exports.md +++ b/graphify/skills/trae/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/trae/references/extraction-spec.md b/graphify/skills/trae/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/trae/references/extraction-spec.md +++ b/graphify/skills/trae/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/trae/references/query.md b/graphify/skills/trae/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/trae/references/query.md +++ b/graphify/skills/trae/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/trae/references/transcribe.md b/graphify/skills/trae/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/trae/references/transcribe.md +++ b/graphify/skills/trae/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/vscode/references/add-watch.md b/graphify/skills/vscode/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/vscode/references/add-watch.md +++ b/graphify/skills/vscode/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/vscode/references/exports.md b/graphify/skills/vscode/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/vscode/references/exports.md +++ b/graphify/skills/vscode/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/vscode/references/extraction-spec.md b/graphify/skills/vscode/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/vscode/references/extraction-spec.md +++ b/graphify/skills/vscode/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/vscode/references/query.md b/graphify/skills/vscode/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/vscode/references/query.md +++ b/graphify/skills/vscode/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/vscode/references/transcribe.md b/graphify/skills/vscode/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/vscode/references/transcribe.md +++ b/graphify/skills/vscode/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/windows/references/add-watch.md b/graphify/skills/windows/references/add-watch.md index 78b6870..7784434 100644 --- a/graphify/skills/windows/references/add-watch.md +++ b/graphify/skills/windows/references/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/windows/references/exports.md b/graphify/skills/windows/references/exports.md index 1c5d8c0..242ff86 100644 --- a/graphify/skills/windows/references/exports.md +++ b/graphify/skills/windows/references/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/graphify/skills/windows/references/extraction-spec.md b/graphify/skills/windows/references/extraction-spec.md index 2501c94..2cc1919 100644 --- a/graphify/skills/windows/references/extraction-spec.md +++ b/graphify/skills/windows/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/windows/references/query.md b/graphify/skills/windows/references/query.md index cb93304..3ed5f65 100644 --- a/graphify/skills/windows/references/query.md +++ b/graphify/skills/windows/references/query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/graphify/skills/windows/references/transcribe.md b/graphify/skills/windows/references/transcribe.md index d9cc698..b967f83 100644 --- a/graphify/skills/windows/references/transcribe.md +++ b/graphify/skills/windows/references/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 6c41b9b..b00917c 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 5740924..8cea9f3 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -374,7 +381,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -392,7 +399,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -455,7 +462,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 9872ae6..3df00d5 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index af9ce49..ee64830 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -374,7 +381,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -392,7 +399,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -455,7 +462,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 9872ae6..3df00d5 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index 80c5675..38f2781 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -374,7 +381,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -392,7 +399,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -455,7 +462,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index fdc0bb7..6ccb172 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 9872ae6..3df00d5 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index d300a0e..aa2e9a1 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -369,7 +376,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -387,7 +394,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -450,7 +457,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 9872ae6..3df00d5 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 20b8b36..2c0a574 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -375,7 +382,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -393,7 +400,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -456,7 +463,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 87044fb..2166d08 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -373,7 +380,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -391,7 +398,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -454,7 +461,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index d0680f2..64d2cbc 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -241,12 +241,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -399,7 +406,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -417,7 +424,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -480,7 +487,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 9872ae6..3df00d5 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -219,12 +219,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -377,7 +384,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -395,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +465,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md b/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__amp__references__exports.md b/tools/skillgen/expected/graphify__skills__amp__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__query.md b/tools/skillgen/expected/graphify__skills__amp__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__query.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md b/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md b/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__claude__references__exports.md b/tools/skillgen/expected/graphify__skills__claude__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__query.md b/tools/skillgen/expected/graphify__skills__claude__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md b/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md b/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__claw__references__exports.md b/tools/skillgen/expected/graphify__skills__claw__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md index 58aee32..3e5a1a3 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__query.md b/tools/skillgen/expected/graphify__skills__claw__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md b/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md b/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__codex__references__exports.md b/tools/skillgen/expected/graphify__skills__codex__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md index 58aee32..3e5a1a3 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__query.md b/tools/skillgen/expected/graphify__skills__codex__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__query.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md b/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md b/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__exports.md b/tools/skillgen/expected/graphify__skills__copilot__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__query.md b/tools/skillgen/expected/graphify__skills__copilot__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__query.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md b/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md b/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__droid__references__exports.md b/tools/skillgen/expected/graphify__skills__droid__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__query.md b/tools/skillgen/expected/graphify__skills__droid__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__query.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md b/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md b/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__exports.md b/tools/skillgen/expected/graphify__skills__kilo__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__query.md b/tools/skillgen/expected/graphify__skills__kilo__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md b/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md b/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__exports.md b/tools/skillgen/expected/graphify__skills__kiro__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md index 58aee32..3e5a1a3 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__query.md b/tools/skillgen/expected/graphify__skills__kiro__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md b/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md b/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__exports.md b/tools/skillgen/expected/graphify__skills__opencode__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__query.md b/tools/skillgen/expected/graphify__skills__opencode__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md b/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md b/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__pi__references__exports.md b/tools/skillgen/expected/graphify__skills__pi__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md index 58aee32..3e5a1a3 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__query.md b/tools/skillgen/expected/graphify__skills__pi__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__query.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md b/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md b/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__trae__references__exports.md b/tools/skillgen/expected/graphify__skills__trae__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__query.md b/tools/skillgen/expected/graphify__skills__trae__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__query.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md b/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md b/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__exports.md b/tools/skillgen/expected/graphify__skills__vscode__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__query.md b/tools/skillgen/expected/graphify__skills__vscode__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md b/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md b/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__windows__references__exports.md b/tools/skillgen/expected/graphify__skills__windows__references__exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__query.md b/tools/skillgen/expected/graphify__skills__windows__references__query.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__query.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__query.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md b/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index e6d44a9..3a06445 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -178,12 +178,19 @@ from graphify.cache import check_semantic_cache from pathlib import Path detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -all_files = [f for files in detect['files'].values() for f in files] +# Only content files go to semantic extraction. Code is already covered structurally +# by the AST pass (Part A); flattening every category here makes subagents re-read +# every source file (#1392). Video is transcribed to a document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) +# Always (re)write the cache file: write hits, else DELETE any leftover from a prior +# run so Part C never merges a stale .graphify_cached.json (#1392). if cached_nodes or cached_edges or cached_hyperedges: Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " @@ -312,7 +319,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. ```bash mkdir -p graphify-out @@ -330,7 +337,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -393,7 +400,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH') +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/fragments/references/query/default.md b/tools/skillgen/fragments/references/query/default.md index cb93304..3ed5f65 100644 --- a/tools/skillgen/fragments/references/query/default.md +++ b/tools/skillgen/fragments/references/query/default.md @@ -88,7 +88,7 @@ G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] +terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) # Find best-matching start nodes scored = [] diff --git a/tools/skillgen/fragments/references/shared/add-watch.md b/tools/skillgen/fragments/references/shared/add-watch.md index 78b6870..7784434 100644 --- a/tools/skillgen/fragments/references/shared/add-watch.md +++ b/tools/skillgen/fragments/references/shared/add-watch.md @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/fragments/references/shared/exports.md b/tools/skillgen/fragments/references/shared/exports.md index 1c5d8c0..242ff86 100644 --- a/tools/skillgen/fragments/references/shared/exports.md +++ b/tools/skillgen/fragments/references/shared/exports.md @@ -59,17 +59,17 @@ graphify export graphml ### Step 7d - MCP server (only if --mcp flag) ```bash -python3 -m graphify.serve graphify-out/graph.json +$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json ``` This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. -To configure in Claude Desktop, add to `claude_desktop_config.json`: +To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: ```json { "mcpServers": { "graphify": { - "command": "python3", + "command": "", "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] } } diff --git a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md index 58aee32..3e5a1a3 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/fragments/references/shared/extraction-spec.md b/tools/skillgen/fragments/references/shared/extraction-spec.md index 2501c94..2cc1919 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/fragments/references/shared/transcribe.md b/tools/skillgen/fragments/references/shared/transcribe.md index d9cc698..b967f83 100644 --- a/tools/skillgen/fragments/references/shared/transcribe.md +++ b/tools/skillgen/fragments/references/shared/transcribe.md @@ -19,14 +19,15 @@ Read the top god node labels from detect output or analysis, then compose a shor - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` -Set it as `WHISPER_PROMPT` to use in the next command. +**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command. **Step 2 - Transcribe:** ```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported) +export GRAPHIFY_WHISPER_PROMPT="" $(cat graphify-out/.graphify_python) -c " -import json, os +import json, os, sys from pathlib import Path from graphify.transcribe import transcribe_all @@ -35,8 +36,11 @@ video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json +# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper +# print progress to stdout, which would otherwise corrupt the JSON file (#1392). +Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\") +print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr) +" ``` After transcription: @@ -45,4 +49,4 @@ After transcription: - Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` - If transcription fails for a file, print a warning and continue with the rest -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, `export GRAPHIFY_WHISPER_MODEL=` (it must be exported, not just assigned) before running the command above. diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index 6c41b9b..b00917c 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -63,7 +63,7 @@ print('code_only:', code_only) If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. +If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. If no new files exist (only deletions), create an empty extraction so the merge step can prune: @@ -106,11 +106,15 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). +# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else +# False. Without it a --directed --update silently rebuilds undirected and collapses +# reciprocal A<->B edges (#1392). G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', + directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -156,7 +160,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links')