From 68081c1c894364071bff309d0e897cfdb2af0aff Mon Sep 17 00:00:00 2001 From: Farhan M Date: Mon, 4 May 2026 19:08:50 +0530 Subject: [PATCH 1/3] feat(extract): add Markdown structural extraction + sync collect_files extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. NEW: extract_markdown() — structurally indexes .md/.mdx files into the knowledge graph. Headings become nodes, code blocks become nodes with language tags, and nesting produces 'contains' edges. Zero new deps (pure regex/line-by-line parsing, no tree-sitter needed). 2. FIX: collect_files() _EXTENSIONS was hardcoded and missing 18 extensions that _DISPATCH already supported (.jsx, .mjs, .ex, .exs, .jl, .vue, .svelte, .dart, .v, .sv, .sql, .f, .F, .f90, etc). Now uses set(_DISPATCH.keys()) to stay automatically in sync. 3. Added deploy_guide.md test fixture and 6 new test cases. 4. Updated test_collect_files_from_dir to use dynamic extension set. --- graphify/extract.py | 120 +++++++++++++++++++++++++++++++-- tests/fixtures/deploy_guide.md | 33 +++++++++ tests/test_extract.py | 7 +- tests/test_languages.py | 43 ++++++++++++ 4 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/deploy_guide.md diff --git a/graphify/extract.py b/graphify/extract.py index 42fd78f..3e321f9 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -3980,6 +3980,116 @@ def extract_elixir(path: Path) -> dict: return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, "input_tokens": 0, "output_tokens": 0} +def extract_markdown(path: Path) -> dict: + """Extract structural nodes and edges from a Markdown file. + + Produces nodes for: + - The file itself + - Each heading (# / ## / ### etc.) + - Each fenced code block (``` ... ```) + + Produces edges for: + - file --contains--> heading + - parent heading --contains--> child heading (nesting by level) + - heading --contains--> code block + - heading --references--> other node (when backtick `Name` matches a known pattern) + + No tree-sitter dependency — pure line-by-line parsing. + """ + try: + source = path.read_text(encoding="utf-8", errors="replace") + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = _file_stem(path) + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + + def add_node(nid: str, label: str, line: int, file_type: str = "document") -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": file_type, + "source_file": str_path, "source_location": f"L{line}"}) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": weight}) + + file_nid = _make_id(str(path)) + add_node(file_nid, path.name, 1) + + # Track heading stack for nesting: [(level, nid), ...] + heading_stack: list[tuple[int, str]] = [] + in_code_block = False + code_block_lang: str | None = None + code_block_start: int = 0 + code_block_lines: list[str] = [] + code_block_count = 0 + + lines = source.splitlines() + for line_num_0, line_text in enumerate(lines): + line_num = line_num_0 + 1 + + # Toggle fenced code blocks + stripped = line_text.strip() + if stripped.startswith("```"): + if not in_code_block: + in_code_block = True + code_block_lang = stripped[3:].strip().split()[0] if len(stripped) > 3 else None + code_block_start = line_num + code_block_lines = [] + continue + else: + # End of code block — create a node + in_code_block = False + code_block_count += 1 + snippet = "\n".join(code_block_lines[:3]) # first 3 lines as preview + label = f"code:{code_block_lang}" if code_block_lang else f"code:block{code_block_count}" + if snippet: + # Use first meaningful line as label hint + first_line = code_block_lines[0].strip()[:60] if code_block_lines else "" + if first_line: + label = f"{label} ({first_line})" + cb_nid = _make_id(stem, f"codeblock_{code_block_count}") + add_node(cb_nid, label, code_block_start) + # Attach to nearest heading or file + parent = heading_stack[-1][1] if heading_stack else file_nid + add_edge(parent, cb_nid, "contains", code_block_start) + continue + + if in_code_block: + code_block_lines.append(line_text) + continue + + # Detect headings: # Heading, ## Heading, etc. + heading_match = re.match(r'^(#{1,6})\s+(.+)', line_text) + if heading_match: + level = len(heading_match.group(1)) + title = heading_match.group(2).strip() + h_nid = _make_id(stem, title) + # Avoid duplicate heading IDs by appending line number + if h_nid in seen_ids: + h_nid = _make_id(stem, title, str(line_num)) + add_node(h_nid, title, line_num) + + # Pop headings at same or deeper level + while heading_stack and heading_stack[-1][0] >= level: + heading_stack.pop() + + # Connect to parent heading or file + parent = heading_stack[-1][1] if heading_stack else file_nid + add_edge(parent, h_nid, "contains", line_num) + + heading_stack.append((level, h_nid)) + continue + + return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0} + + # ── Main extract and collect_files ──────────────────────────────────────────── @@ -4049,6 +4159,8 @@ _DISPATCH: dict[str, Any] = { ".v": extract_verilog, ".sv": extract_verilog, ".sql": extract_sql, + ".md": extract_markdown, + ".mdx": extract_markdown, } @@ -4365,13 +4477,7 @@ def extract( def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | None = None) -> list[Path]: if target.is_file(): return [target] - _EXTENSIONS = { - ".py", ".js", ".ts", ".tsx", ".go", ".rs", - ".java", ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", - ".rb", ".cs", ".kt", ".kts", ".scala", ".php", ".swift", - ".lua", ".toc", ".zig", ".ps1", - ".m", ".mm", - } + _EXTENSIONS = set(_DISPATCH.keys()) from graphify.detect import _load_graphifyignore, _is_ignored ignore_root = root if root is not None else target patterns = _load_graphifyignore(ignore_root) diff --git a/tests/fixtures/deploy_guide.md b/tests/fixtures/deploy_guide.md new file mode 100644 index 0000000..445c187 --- /dev/null +++ b/tests/fixtures/deploy_guide.md @@ -0,0 +1,33 @@ +# Deploy Guide + +How to deploy the QuranicWords backend. + +## Prerequisites + +- Docker installed +- SSH access to VPS + +## Full Deploy + +Run this one-liner on your VPS: + +```bash +cd /opt/QuranicWords && git pull origin main && docker compose build --no-cache api +``` + +### Database Migration + +If you changed the Prisma schema: + +```sql +ALTER TABLE users ADD COLUMN points INT DEFAULT 0; +``` + +## Rollback + +Use `git revert` to undo bad deploys. + +```python +def rollback(version): + subprocess.run(["git", "checkout", version]) +``` diff --git a/tests/test_extract.py b/tests/test_extract.py index 06d371b..be07452 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -57,12 +57,9 @@ def test_extract_merges_multiple_files(): def test_collect_files_from_dir(): + from graphify.extract import _DISPATCH files = collect_files(FIXTURES) - supported = {".py", ".js", ".ts", ".tsx", ".go", ".rs", - ".java", ".c", ".cpp", ".cc", ".cxx", ".rb", - ".cs", ".kt", ".kts", ".scala", ".php", ".h", ".hpp", - ".swift", ".lua", ".toc", ".zig", ".ps1", ".ex", ".exs", - ".m", ".mm"} + supported = set(_DISPATCH.keys()) assert all(f.suffix in supported for f in files) assert len(files) > 0 diff --git a/tests/test_languages.py b/tests/test_languages.py index b2460d5..4954782 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -853,3 +853,46 @@ def test_ts_static_template_literal_resolved(): targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} assert any("statichelper" in t.lower() for t in targets), \ f"Static template literal import not resolved: {targets}" + + +# ── Markdown ───────────────────────────────────────────────────────────────── + +from graphify.extract import extract_markdown + +def test_markdown_no_error(): + r = extract_markdown(FIXTURES / "deploy_guide.md") + assert "error" not in r + +def test_markdown_finds_headings(): + r = extract_markdown(FIXTURES / "deploy_guide.md") + labels = _labels(r) + assert any("Deploy Guide" in l for l in labels) + assert any("Prerequisites" in l for l in labels) + assert any("Full Deploy" in l for l in labels) + assert any("Rollback" in l for l in labels) + +def test_markdown_finds_nested_heading(): + """### Database Migration is nested under ## Full Deploy.""" + r = extract_markdown(FIXTURES / "deploy_guide.md") + labels = _labels(r) + assert any("Database Migration" in l for l in labels) + +def test_markdown_finds_code_blocks(): + r = extract_markdown(FIXTURES / "deploy_guide.md") + labels = _labels(r) + assert any("code:bash" in l for l in labels) + assert any("code:sql" in l for l in labels) + assert any("code:python" in l for l in labels) + +def test_markdown_contains_edges(): + """Headings and code blocks should be connected via 'contains' edges.""" + r = extract_markdown(FIXTURES / "deploy_guide.md") + assert "contains" in _relations(r) + contains_edges = [e for e in r["edges"] if e["relation"] == "contains"] + assert len(contains_edges) >= 5 # file->h1, h1->h2s, h2->h3, h2->codeblocks + +def test_markdown_no_dangling_edges(): + r = extract_markdown(FIXTURES / "deploy_guide.md") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"Dangling source: {e}" From 09b33b795b9a24513afb7defaf0db1bf6bd0a9e6 Mon Sep 17 00:00:00 2001 From: imfarhanm Date: Mon, 4 May 2026 20:01:20 +0530 Subject: [PATCH 2/3] fix: include .md/.mdx document files in graphify update rebuild path --- graphify/watch.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/graphify/watch.py b/graphify/watch.py index a967b5d..0b7bd95 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -70,6 +70,13 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False, force: boo detected = detect(watch_path, follow_symlinks=follow_symlinks) code_files = [Path(f) for f in detected['files']['code']] + # Include document files that have AST extractors (e.g. .md, .mdx) + from graphify.extract import _get_extractor + for doc_file in detected['files'].get('document', []): + p = Path(doc_file) + if _get_extractor(p) is not None: + code_files.append(p) + if not code_files: print("[graphify watch] No code files found - nothing to rebuild.") return False From cddf9cf9bfd29b34d95a04623a20dec0f21ec04d Mon Sep 17 00:00:00 2001 From: imfarhanm Date: Mon, 4 May 2026 20:04:23 +0530 Subject: [PATCH 3/3] docs: add fork install instructions and feature comparison to README --- README.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/README.md b/README.md index f5d9ef7..c07f6b0 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,43 @@

+--- + +## 🔀 This Fork: `imfarhanm/graphify` + +> **What's different?** This fork adds **Markdown structural extraction** and fixes the `graphify update` command to include `.md` / `.mdx` files — so your docs, guides, and knowledge files become part of the knowledge graph **without needing an LLM**. + +### What this fork adds + +| Feature | Upstream | This Fork | +|---------|----------|-----------| +| `.md` / `.mdx` AST extraction | ❌ Needs LLM | ✅ Zero-dependency, line-by-line parser | +| `graphify update .` includes `.md` | ❌ Code-only | ✅ Markdown included automatically | +| Headings → graph nodes | ❌ | ✅ `# / ## / ###` become nodes | +| Code blocks → graph nodes | ❌ | ✅ Fenced code blocks extracted | +| Heading nesting → edges | ❌ | ✅ Parent-child heading relationships | + +### Install (one-liner) + +```bash +pip install git+https://github.com/imfarhanm/graphify.git@feat/md-extraction-and-extensions-sync +``` + +Then use normally: + +```bash +graphify install # set up your IDE +graphify update . # now includes .md files! +graphify explain "GUIDE.md" # query your docs +graphify query "deployment steps" # search across code + docs +``` + +### Branch + +All changes are on: **`feat/md-extraction-and-extensions-sync`** + +--- + Type `/graphify` in your AI coding assistant and it maps your entire project — code, docs, PDFs, images, videos — into a knowledge graph you can query instead of grepping through files. Works in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, VS Code Copilot Chat, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, and Google Antigravity.