fix(build): merge markdown quick-scan doc node into its semantic _doc twin (#1799)
The semantic pass mints a document node <slug>_doc; the markdown quick-scan (extract_markdown) mints the bare <slug>. After a semantic build, a `graphify update` (AST path) re-runs the quick-scan and the graph ends up with BOTH — one document as two disconnected nodes, the file's edges split between them (semantic `references`/hyperedges on the _doc twin, quick-scan cross-links on the bare one). path/query traversals dead-end on the wrong twin; degree and communities split. build_from_json now reconciles the pair: when <slug> and <slug>_doc both exist with the same source_file and both are file_type=document, remap the bare node into the semantic _doc node (canonical, richer edges) and repoint its edges and hyperedges. Remap-induced self-loops are dropped; pre-existing ones are left alone. Gated to document twins for the same file, so a code symbol `foo` and an unrelated `foo_doc` never merge. Regression tests for both. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
committed by
safishamsi
co-authored by
Claude Opus 4.8
parent
591da764a1
commit
71d7ab9d71
@@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu
|
||||
|
||||
## 0.9.13 (unreleased)
|
||||
|
||||
- Fix: a markdown file that went through semantic extraction is no longer duplicated into two disconnected nodes on later `graphify update` (#1799, thanks @jerp86). The semantic pass mints `<slug>_doc` while the markdown quick-scan mints the bare `<slug>`, so the file's edges split across two twins (a docs->code path query would dead-end on the bare half; centrality and communities split too). `build_from_json` now merges the bare quick-scan node into the semantic `_doc` node when both share the same `source_file` and are `file_type: document`, consolidating their edges/hyperedges onto one node. Gated so an unrelated code symbol `foo` and `foo_doc` never merge.
|
||||
|
||||
- Fix: incremental `graphify update` no longer silently evicts nodes for a file that left the scan corpus but still exists on disk (#1795, thanks @CJNA). `_reconcile_existing_graph` read "source absent from the collected corpus" as "deleted", but that's also what an ignore-rule/filter change looks like (e.g. an upgrade that starts honoring `.gitignore`) — in one 27k-node graph the first rebuild after such an upgrade mass-evicted 655 nodes whose files were present the whole time. Eviction now fails closed: a corpus-absent source is only evicted when `Path(identity).exists()` is False (true deletion), otherwise its nodes/edges/hyperedges are preserved and a loud line reports how many were kept and why. True deletions and renames evict as before; a full `extract --force` still purges deliberate exclusions.
|
||||
|
||||
- Fix: `build_merge` no longer silently deletes a re-extracted file's fresh nodes when that file is also passed in `prune_sources` (#1796, thanks @erichkusuki). A file present in `new_chunks` is being replaced, not deleted, so it's now excluded from the prune set — "replace" wins over a contradictory "delete" of the same source. Previously, following the old edit-workflow (pass the changed file in `prune_sources`) deleted the just-built concept whenever an edit kept a node's label. Genuine deletions (a file in `prune_sources` but not `new_chunks`) still prune.
|
||||
|
||||
@@ -348,6 +348,38 @@ def graph_has_legacy_ids(nodes: list, root: str | Path | None = None, sample: in
|
||||
return False
|
||||
|
||||
|
||||
def _doc_twin_remap(nodes: list) -> dict[str, str]:
|
||||
"""Map a markdown quick-scan's bare doc node ``<slug>`` to the semantic
|
||||
``<slug>_doc`` node for the SAME file (#1799).
|
||||
|
||||
The markdown quick-scan (``extract_markdown``) mints a file node with the
|
||||
bare id ``_make_id(path)`` while the semantic pass mints ``<slug>_doc`` for
|
||||
the same document. A ``graphify update`` after a semantic build leaves both,
|
||||
splitting the file's edges across two disconnected nodes. Canonicalize to the
|
||||
semantic ``_doc`` node (it carries the richer references/hyperedges). Gated to
|
||||
``file_type == "document"`` on BOTH twins with an identical ``source_file``,
|
||||
so an unrelated code symbol ``foo`` and ``foo_doc`` never merge.
|
||||
"""
|
||||
by_id: dict[str, dict] = {}
|
||||
for n in nodes:
|
||||
if isinstance(n, dict) and n.get("id"):
|
||||
by_id[str(n["id"])] = n
|
||||
remap: dict[str, str] = {}
|
||||
for nid, node in by_id.items():
|
||||
if not nid.endswith("_doc"):
|
||||
continue
|
||||
bare = by_id.get(nid[:-4])
|
||||
if bare is None:
|
||||
continue
|
||||
sf = node.get("source_file")
|
||||
if not sf or bare.get("source_file") != sf:
|
||||
continue
|
||||
if node.get("file_type") != "document" or bare.get("file_type") != "document":
|
||||
continue
|
||||
remap[nid[:-4]] = nid
|
||||
return remap
|
||||
|
||||
|
||||
def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph:
|
||||
"""Build a NetworkX graph from an extraction dict.
|
||||
|
||||
@@ -428,6 +460,33 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat
|
||||
if isinstance(he, dict) and isinstance(he.get("nodes"), list):
|
||||
he["nodes"] = [_rekey.get(n, n) for n in he["nodes"]]
|
||||
|
||||
# Merge markdown quick-scan bare doc nodes into their semantic `_doc` twin
|
||||
# for the same file, so a document is one node regardless of which pipeline
|
||||
# touched it last (#1799).
|
||||
_doc_remap = _doc_twin_remap(extraction.get("nodes", []))
|
||||
if _doc_remap:
|
||||
extraction["nodes"] = [
|
||||
n for n in extraction.get("nodes", [])
|
||||
if not (isinstance(n, dict) and n.get("id") in _doc_remap)
|
||||
]
|
||||
_new_edges = []
|
||||
for edge in extraction.get("edges", []):
|
||||
if isinstance(edge, dict):
|
||||
s0, t0 = edge.get("source"), edge.get("target")
|
||||
if s0 in _doc_remap:
|
||||
edge["source"] = _doc_remap[s0]
|
||||
if t0 in _doc_remap:
|
||||
edge["target"] = _doc_remap[t0]
|
||||
# Drop only self-loops the remap itself collapsed (a bare->_doc
|
||||
# link becoming doc->doc); leave any pre-existing self-loop alone.
|
||||
if edge.get("source") == edge.get("target") and (s0 in _doc_remap or t0 in _doc_remap):
|
||||
continue
|
||||
_new_edges.append(edge)
|
||||
extraction["edges"] = _new_edges
|
||||
for he in extraction.get("hyperedges", []) or []:
|
||||
if isinstance(he, dict) and isinstance(he.get("nodes"), list):
|
||||
he["nodes"] = [_doc_remap.get(n, n) for n in he["nodes"]]
|
||||
|
||||
G: nx.Graph = nx.DiGraph() if directed else nx.Graph()
|
||||
for node in extraction.get("nodes", []):
|
||||
# Skip dict nodes with a missing or non-hashable id (e.g. a list emitted
|
||||
|
||||
@@ -924,3 +924,49 @@ def test_cross_family_reference_to_unknown_ext_is_kept():
|
||||
}
|
||||
G = build_from_json(ext, directed=False)
|
||||
assert G.has_edge("pkg_json", "src_app_ts"), "config->code reference (unknown ext) must be kept"
|
||||
|
||||
|
||||
def test_markdown_doc_twin_merges_into_semantic_doc_node():
|
||||
"""#1799: the markdown quick-scan's bare `<slug>` doc node and the semantic
|
||||
`<slug>_doc` node for the same file must collapse to one node, with edges
|
||||
consolidated — otherwise a document is two disconnected halves and traversals
|
||||
dead-end on the wrong twin."""
|
||||
ext = {
|
||||
"nodes": [
|
||||
{"id": "docs_readme_doc", "label": "README", "file_type": "document",
|
||||
"source_file": "docs/readme.md", "source_location": "L1"},
|
||||
{"id": "docs_readme", "label": "readme.md", "file_type": "document",
|
||||
"source_file": "docs/readme.md", "source_location": "L1"},
|
||||
{"id": "code_auth", "label": "auth", "file_type": "code",
|
||||
"source_file": "auth.py", "source_location": "L1"},
|
||||
{"id": "docs_guide", "label": "guide.md", "file_type": "document",
|
||||
"source_file": "docs/guide.md", "source_location": "L1"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "docs_readme_doc", "target": "code_auth", "relation": "references",
|
||||
"source_file": "docs/readme.md", "confidence": "INFERRED", "weight": 1.0},
|
||||
{"source": "docs_guide", "target": "docs_readme", "relation": "references",
|
||||
"source_file": "docs/guide.md", "confidence": "EXTRACTED", "weight": 1.0},
|
||||
],
|
||||
}
|
||||
G = build_from_json(ext, directed=False)
|
||||
assert "docs_readme" not in G.nodes() # bare twin merged away
|
||||
assert "docs_readme_doc" in G.nodes() # semantic node is canonical
|
||||
assert G.has_edge("docs_guide", "docs_readme_doc") # quick-scan edge repointed
|
||||
assert G.has_edge("docs_readme_doc", "code_auth") # semantic edge kept
|
||||
|
||||
|
||||
def test_doc_twin_merge_does_not_touch_code_symbols():
|
||||
"""#1799 guard: a code symbol `foo` and an unrelated `foo_doc` (not
|
||||
file_type=document) must NOT merge, even sharing a source_file."""
|
||||
ext = {
|
||||
"nodes": [
|
||||
{"id": "m_foo", "label": "foo", "file_type": "code",
|
||||
"source_file": "m.py", "source_location": "L1"},
|
||||
{"id": "m_foo_doc", "label": "foo rationale", "file_type": "rationale",
|
||||
"source_file": "m.py", "source_location": "L2"},
|
||||
],
|
||||
"edges": [],
|
||||
}
|
||||
G = build_from_json(ext, directed=False)
|
||||
assert {"m_foo", "m_foo_doc"} <= set(G.nodes())
|
||||
|
||||
Reference in New Issue
Block a user