fix(watch): preserve semantic edges of re-extracted sources (#1865)
A full graphify update evicted every edge whose source_file was a re-extracted source, including LLM semantic edges (e.g. semantically_similar_to) from Markdown docs that also have an AST extractor. The AST pass cannot regenerate those edges, so they were silently lost while their concept nodes survived — node eviction is provenance-aware via _origin (#1116), edge eviction was not. Tag AST-extracted edges with the same _origin=ast marker nodes already carry, and scope rebuild-driven edge eviction to that tier: re-extraction replaces a source's AST edges, while its semantic edges survive until a semantic re-extraction supersedes them. Deletion-driven eviction stays provenance-blind, so edges of deleted or excluded sources are still purged regardless of tier. Edges from graphs built before this change lack the marker; a stale AST edge from a file changed exactly once between the old and new version can linger until its source is deleted — the same migration trade-off the #1116 node marker made.
This commit is contained in:
+6
-1
@@ -4874,9 +4874,14 @@ def extract(
|
||||
# Tag AST provenance so the incremental watch rebuild can distinguish
|
||||
# AST-extracted nodes from semantic/LLM nodes. On a full re-extraction
|
||||
# the watcher drops any AST-marked node missing from the fresh output
|
||||
# even when its source file still exists (#1116).
|
||||
# even when its source file still exists (#1116). Edges carry the same
|
||||
# marker so edge eviction can be tier-scoped: re-extracting a source
|
||||
# replaces its AST edges without evicting the semantic edges the AST
|
||||
# pass cannot regenerate (#1865).
|
||||
for n in all_nodes:
|
||||
n["_origin"] = "ast"
|
||||
for e in all_edges:
|
||||
e["_origin"] = "ast"
|
||||
|
||||
return {
|
||||
"nodes": all_nodes,
|
||||
|
||||
+13
-5
@@ -408,11 +408,11 @@ def _reconcile_existing_graph(
|
||||
}
|
||||
node_evicted_source_identities = set(deleted_source_identities)
|
||||
hyperedge_evicted_source_identities = set(deleted_source_identities)
|
||||
# Deletion evicts edges regardless of tier; re-extraction only owns a
|
||||
# source's AST-tier edges (checked per-edge below, #1865).
|
||||
edge_evicted_source_identities = set(deleted_source_identities)
|
||||
if not full_rebuild:
|
||||
node_evicted_source_identities.update(rebuilt_source_identities)
|
||||
edge_evicted_source_identities = (
|
||||
node_evicted_source_identities | rebuilt_source_identities
|
||||
)
|
||||
|
||||
# Reconcile every rebuild against the current watched corpus. Hook change
|
||||
# lists can contain only a rename destination, so explicit paths alone
|
||||
@@ -485,14 +485,22 @@ def _reconcile_existing_graph(
|
||||
]
|
||||
all_ids = new_ast_ids | {node["id"] for node in preserved_nodes}
|
||||
|
||||
# Edges are owned by source_file. Re-extraction must replace an owner's
|
||||
# previous edges, while edges from unchanged or semantic sources survive.
|
||||
# Edges are owned by source_file, but ownership is tier-scoped: the AST
|
||||
# pass replaces a re-extracted source's AST edges, while that source's
|
||||
# semantic/LLM edges — which the AST pass cannot regenerate — survive
|
||||
# until a semantic re-extraction supersedes them. Same provenance rule
|
||||
# the node reconciliation above applies via _origin (#1865). Deletion
|
||||
# eviction stays provenance-blind.
|
||||
preserved_edges = [
|
||||
edge
|
||||
for edge in existing.get("links", existing.get("edges", []))
|
||||
if edge.get("source") in all_ids
|
||||
and edge.get("target") in all_ids
|
||||
and not source_paths.is_evicted(edge, edge_evicted_source_identities)
|
||||
and not (
|
||||
edge.get("_origin") == "ast"
|
||||
and source_paths.is_evicted(edge, rebuilt_source_identities)
|
||||
)
|
||||
]
|
||||
|
||||
new_hyperedge_ids = {
|
||||
|
||||
@@ -386,6 +386,72 @@ def test_rebuild_code_preserves_hyperedges_for_rebuilt_surviving_source(
|
||||
}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"changed_paths",
|
||||
[None, [Path("auth.md")]],
|
||||
ids=["full-update", "incremental-doc-update"],
|
||||
)
|
||||
def test_rebuild_code_preserves_semantic_edges_from_reextracted_doc(
|
||||
tmp_path, changed_paths
|
||||
):
|
||||
"""#1865: AST-only updates must not evict semantic edges whose source_file
|
||||
is a re-extracted document; only that source's AST-tier edges are replaced."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
corpus.mkdir()
|
||||
(corpus / "auth.md").write_text(
|
||||
"# Token Validation\n\nVerifies bearer tokens.\n", encoding="utf-8"
|
||||
)
|
||||
(corpus / "login.md").write_text(
|
||||
"# Session Verification\n\nVerifies login sessions.\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
data = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
node_ids = {n["id"] for n in data["nodes"]}
|
||||
assert {"auth_token_validation", "login_session_verification"} <= node_ids
|
||||
|
||||
data["links"].extend([
|
||||
{
|
||||
"source": "auth_token_validation",
|
||||
"target": "login_session_verification",
|
||||
"relation": "semantically_similar_to",
|
||||
"confidence": "INFERRED",
|
||||
"source_file": "auth.md",
|
||||
},
|
||||
# A stale AST-tier edge of the same source must still be evicted.
|
||||
{
|
||||
"source": "auth_token_validation",
|
||||
"target": "login_session_verification",
|
||||
"relation": "references",
|
||||
"_origin": "ast",
|
||||
"source_file": "auth.md",
|
||||
},
|
||||
])
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
assert _rebuild_code(
|
||||
corpus,
|
||||
changed_paths=changed_paths,
|
||||
no_cluster=True,
|
||||
acquire_lock=False,
|
||||
) is True
|
||||
|
||||
after = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
relations = {
|
||||
(e.get("source"), e.get("target"), e.get("relation"))
|
||||
for e in after["links"]
|
||||
}
|
||||
assert (
|
||||
"auth_token_validation", "login_session_verification", "semantically_similar_to"
|
||||
) in relations, "semantic edge from a re-extracted doc must survive an AST-only update"
|
||||
assert (
|
||||
"auth_token_validation", "login_session_verification", "references"
|
||||
) not in relations, "stale AST-tier edge of a re-extracted source must be evicted"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"changed_paths",
|
||||
[None, [Path("only.py")]],
|
||||
|
||||
Reference in New Issue
Block a user