From 71d7ab9d7138224d5198415dabb3941430f3c5d7 Mon Sep 17 00:00:00 2001 From: jerp86 Date: Sun, 12 Jul 2026 10:23:36 +0100 Subject: [PATCH] fix(build): merge markdown quick-scan doc node into its semantic _doc twin (#1799) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The semantic pass mints a document node _doc; the markdown quick-scan (extract_markdown) mints the bare . After a semantic build, a `graphify update` (AST path) re-runs the quick-scan and the graph ends up with BOTH — one document as two disconnected nodes, the file's edges split between them (semantic `references`/hyperedges on the _doc twin, quick-scan cross-links on the bare one). path/query traversals dead-end on the wrong twin; degree and communities split. build_from_json now reconciles the pair: when and _doc both exist with the same source_file and both are file_type=document, remap the bare node into the semantic _doc node (canonical, richer edges) and repoint its edges and hyperedges. Remap-induced self-loops are dropped; pre-existing ones are left alone. Gated to document twins for the same file, so a code symbol `foo` and an unrelated `foo_doc` never merge. Regression tests for both. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 ++ graphify/build.py | 59 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_build.py | 46 +++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd04699..87660e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.13 (unreleased) +- Fix: a markdown file that went through semantic extraction is no longer duplicated into two disconnected nodes on later `graphify update` (#1799, thanks @jerp86). The semantic pass mints `_doc` while the markdown quick-scan mints the bare ``, so the file's edges split across two twins (a docs->code path query would dead-end on the bare half; centrality and communities split too). `build_from_json` now merges the bare quick-scan node into the semantic `_doc` node when both share the same `source_file` and are `file_type: document`, consolidating their edges/hyperedges onto one node. Gated so an unrelated code symbol `foo` and `foo_doc` never merge. + - Fix: incremental `graphify update` no longer silently evicts nodes for a file that left the scan corpus but still exists on disk (#1795, thanks @CJNA). `_reconcile_existing_graph` read "source absent from the collected corpus" as "deleted", but that's also what an ignore-rule/filter change looks like (e.g. an upgrade that starts honoring `.gitignore`) — in one 27k-node graph the first rebuild after such an upgrade mass-evicted 655 nodes whose files were present the whole time. Eviction now fails closed: a corpus-absent source is only evicted when `Path(identity).exists()` is False (true deletion), otherwise its nodes/edges/hyperedges are preserved and a loud line reports how many were kept and why. True deletions and renames evict as before; a full `extract --force` still purges deliberate exclusions. - Fix: `build_merge` no longer silently deletes a re-extracted file's fresh nodes when that file is also passed in `prune_sources` (#1796, thanks @erichkusuki). A file present in `new_chunks` is being replaced, not deleted, so it's now excluded from the prune set — "replace" wins over a contradictory "delete" of the same source. Previously, following the old edit-workflow (pass the changed file in `prune_sources`) deleted the just-built concept whenever an edit kept a node's label. Genuine deletions (a file in `prune_sources` but not `new_chunks`) still prune. diff --git a/graphify/build.py b/graphify/build.py index 64597f1..428fa1c 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -348,6 +348,38 @@ def graph_has_legacy_ids(nodes: list, root: str | Path | None = None, sample: in return False +def _doc_twin_remap(nodes: list) -> dict[str, str]: + """Map a markdown quick-scan's bare doc node ```` to the semantic + ``_doc`` node for the SAME file (#1799). + + The markdown quick-scan (``extract_markdown``) mints a file node with the + bare id ``_make_id(path)`` while the semantic pass mints ``_doc`` for + the same document. A ``graphify update`` after a semantic build leaves both, + splitting the file's edges across two disconnected nodes. Canonicalize to the + semantic ``_doc`` node (it carries the richer references/hyperedges). Gated to + ``file_type == "document"`` on BOTH twins with an identical ``source_file``, + so an unrelated code symbol ``foo`` and ``foo_doc`` never merge. + """ + by_id: dict[str, dict] = {} + for n in nodes: + if isinstance(n, dict) and n.get("id"): + by_id[str(n["id"])] = n + remap: dict[str, str] = {} + for nid, node in by_id.items(): + if not nid.endswith("_doc"): + continue + bare = by_id.get(nid[:-4]) + if bare is None: + continue + sf = node.get("source_file") + if not sf or bare.get("source_file") != sf: + continue + if node.get("file_type") != "document" or bare.get("file_type") != "document": + continue + remap[nid[:-4]] = nid + return remap + + def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: """Build a NetworkX graph from an extraction dict. @@ -428,6 +460,33 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if isinstance(he, dict) and isinstance(he.get("nodes"), list): he["nodes"] = [_rekey.get(n, n) for n in he["nodes"]] + # Merge markdown quick-scan bare doc nodes into their semantic `_doc` twin + # for the same file, so a document is one node regardless of which pipeline + # touched it last (#1799). + _doc_remap = _doc_twin_remap(extraction.get("nodes", [])) + if _doc_remap: + extraction["nodes"] = [ + n for n in extraction.get("nodes", []) + if not (isinstance(n, dict) and n.get("id") in _doc_remap) + ] + _new_edges = [] + for edge in extraction.get("edges", []): + if isinstance(edge, dict): + s0, t0 = edge.get("source"), edge.get("target") + if s0 in _doc_remap: + edge["source"] = _doc_remap[s0] + if t0 in _doc_remap: + edge["target"] = _doc_remap[t0] + # Drop only self-loops the remap itself collapsed (a bare->_doc + # link becoming doc->doc); leave any pre-existing self-loop alone. + if edge.get("source") == edge.get("target") and (s0 in _doc_remap or t0 in _doc_remap): + continue + _new_edges.append(edge) + extraction["edges"] = _new_edges + for he in extraction.get("hyperedges", []) or []: + if isinstance(he, dict) and isinstance(he.get("nodes"), list): + he["nodes"] = [_doc_remap.get(n, n) for n in he["nodes"]] + G: nx.Graph = nx.DiGraph() if directed else nx.Graph() for node in extraction.get("nodes", []): # Skip dict nodes with a missing or non-hashable id (e.g. a list emitted diff --git a/tests/test_build.py b/tests/test_build.py index c0bc0e3..872a2e1 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -924,3 +924,49 @@ def test_cross_family_reference_to_unknown_ext_is_kept(): } G = build_from_json(ext, directed=False) assert G.has_edge("pkg_json", "src_app_ts"), "config->code reference (unknown ext) must be kept" + + +def test_markdown_doc_twin_merges_into_semantic_doc_node(): + """#1799: the markdown quick-scan's bare `` doc node and the semantic + `_doc` node for the same file must collapse to one node, with edges + consolidated — otherwise a document is two disconnected halves and traversals + dead-end on the wrong twin.""" + ext = { + "nodes": [ + {"id": "docs_readme_doc", "label": "README", "file_type": "document", + "source_file": "docs/readme.md", "source_location": "L1"}, + {"id": "docs_readme", "label": "readme.md", "file_type": "document", + "source_file": "docs/readme.md", "source_location": "L1"}, + {"id": "code_auth", "label": "auth", "file_type": "code", + "source_file": "auth.py", "source_location": "L1"}, + {"id": "docs_guide", "label": "guide.md", "file_type": "document", + "source_file": "docs/guide.md", "source_location": "L1"}, + ], + "edges": [ + {"source": "docs_readme_doc", "target": "code_auth", "relation": "references", + "source_file": "docs/readme.md", "confidence": "INFERRED", "weight": 1.0}, + {"source": "docs_guide", "target": "docs_readme", "relation": "references", + "source_file": "docs/guide.md", "confidence": "EXTRACTED", "weight": 1.0}, + ], + } + G = build_from_json(ext, directed=False) + assert "docs_readme" not in G.nodes() # bare twin merged away + assert "docs_readme_doc" in G.nodes() # semantic node is canonical + assert G.has_edge("docs_guide", "docs_readme_doc") # quick-scan edge repointed + assert G.has_edge("docs_readme_doc", "code_auth") # semantic edge kept + + +def test_doc_twin_merge_does_not_touch_code_symbols(): + """#1799 guard: a code symbol `foo` and an unrelated `foo_doc` (not + file_type=document) must NOT merge, even sharing a source_file.""" + ext = { + "nodes": [ + {"id": "m_foo", "label": "foo", "file_type": "code", + "source_file": "m.py", "source_location": "L1"}, + {"id": "m_foo_doc", "label": "foo rationale", "file_type": "rationale", + "source_file": "m.py", "source_location": "L2"}, + ], + "edges": [], + } + G = build_from_json(ext, directed=False) + assert {"m_foo", "m_foo_doc"} <= set(G.nodes())