fix(build): don't bind imports/references across a language boundary (#1749)
The extraction spec forbids cross-language `calls` edges, and build already dropped cross-language INFERRED `calls`. But `imports`/`references` had no such guard: an unresolved Python `import time` resolved by bare stem (the #1504 old-stem alias) onto a `src/time.ts` file node, welding a polyglot repo's two language halves together. In the reporter's repo three such edges were the only bridge between 2409 Python and 1403 TS nodes, so every backend<->frontend shortest path routed through time.ts, inflating its betweenness ~90x and making it the #1 reported god node. Hoist the interop-family map to a module constant and extend the edge-loop guard to `imports`/`imports_from`/`references`. For these relations the edge is dropped only when BOTH endpoints are known code languages of different families, so a config/manifest -> code reference (unknown ext) is never mistaken for a phantom. `calls` behavior is unchanged (still INFERRED-only, still drops when either family differs). Regression tests: py->ts import dropped, ts->ts import kept, config->code reference kept. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
philberndt
committed by
safishamsi
safishamsi
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
8b7ffc5b15
commit
bdc6e531a6
@@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu
|
||||
|
||||
## 0.9.12 (unreleased)
|
||||
|
||||
- Fix: `imports`/`references` edges no longer bind across a language boundary (#1749, thanks @philberndt). The spec already forbids cross-language `calls`, but an unresolved Python `import time` could still resolve by bare stem onto a `src/time.ts` file node — welding a polyglot repo's halves together at a phantom edge (in the reporter's repo, 3 such edges were the *only* thing bridging 2409 Python nodes to 1403 TS nodes, inflating `time.ts` betweenness ~90x and making it the #1 "god node"). The build-time cross-language guard now covers `imports`/`imports_from`/`references` in addition to `calls`, dropping an edge only when both endpoints are known code languages of different interop families (so a config/manifest → code reference is untouched).
|
||||
|
||||
- Fix: files whose extractor bailed out for a missing optional dependency no longer vanish without a trace (#1745, thanks @rithyKabir). `.sql` files (and other extra-gated languages) have a dispatch entry, so the #1689 no-extractor warning can't fire, and `extract_sql` returns an error result when `tree-sitter-sql` is absent, so the #1666 zero-node warning skips it too — the graph built "successfully" while an entire SQL corpus contributed nothing. `extract()` now surfaces these grouped by extension, naming the extra that restores the language (e.g. `pip install "graphifyy[sql]"`).
|
||||
|
||||
- Fix: `build_from_json` is deterministic across process runs again (#1753, thanks @erasmust-dotcom). The ghost-node merge iterated `set(G.nodes())`, so which node survived a `(basename, label)` collision depended on CPython's per-process string-hash seed — rebuilding the same extraction JSON in a fresh process could silently pick a different canonical id (breaking the cluster→relabel workflow with a `KeyError` on an id that vanished). The Pass 1/Pass 2 loops now iterate in sorted order. Additionally, two non-AST (semantic) nodes sharing a key but from *different* files are now treated as distinct concepts and both survive (mirroring the AST/AST ambiguity guard #1257) instead of one arbitrarily merging away; a genuine same-file duplicate still collapses.
|
||||
|
||||
+43
-23
@@ -33,6 +33,26 @@ from .paths import default_graph_json as _default_graph_json
|
||||
from .validate import validate_extraction
|
||||
|
||||
|
||||
# Language interop families, keyed by extension, for the cross-language phantom-edge
|
||||
# guard in the edge loop below. Families group by REAL interop (JS/TS share a module
|
||||
# graph; C/C++/ObjC share a compilation unit via headers; JVM langs share bytecode),
|
||||
# so a legitimate TS->JS import or C impl->header call survives, while a Python
|
||||
# `import time` binding to a `time.ts` (#1749) or a cross-language INFERRED `calls`
|
||||
# edge (#1547/#1556) is dropped. Kept local to build.py (not imported from extract.py,
|
||||
# which imports build.py — a cycle) and deliberately mirrors extract._LANG_FAMILY_BY_EXT.
|
||||
_EDGE_LANG_FAMILY: dict[str, str] = {
|
||||
".py": "py", ".pyi": "py",
|
||||
".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js",
|
||||
".ts": "js", ".tsx": "js", ".mts": "js", ".cts": "js",
|
||||
".go": "go", ".rs": "rs",
|
||||
".java": "jvm", ".kt": "jvm", ".scala": "jvm", ".groovy": "jvm",
|
||||
".c": "c", ".h": "c", ".cc": "c", ".cpp": "c", ".hpp": "c",
|
||||
".cxx": "c", ".hh": "c", ".hxx": "c",
|
||||
".cu": "c", ".cuh": "c", ".metal": "c", ".m": "c", ".mm": "c",
|
||||
".rb": "rb", ".php": "php", ".cs": "cs", ".swift": "swift", ".lua": "lua",
|
||||
}
|
||||
|
||||
|
||||
# Synonym mapper for known invalid file_type values that LLM subagents commonly
|
||||
# emit. Keeps semantic intent close (markdown→document, tool→code) and falls
|
||||
# back to "concept" for any other invalid value (see #840).
|
||||
@@ -628,31 +648,31 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat
|
||||
)
|
||||
if "source_file" in attrs:
|
||||
attrs["source_file"] = _norm_source_file(attrs["source_file"], _root)
|
||||
# Drop cross-language INFERRED `calls` edges — same short names (render,
|
||||
# parse, etc.) appear across language boundaries in multi-language chunks,
|
||||
# producing phantom edges that don't represent real call relationships.
|
||||
if attrs.get("relation") == "calls" and attrs.get("confidence") == "INFERRED":
|
||||
_LANG_FAMILY: dict[str, str] = {
|
||||
".py": "py", ".pyi": "py",
|
||||
".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js",
|
||||
".ts": "js", ".tsx": "js", ".mts": "js", ".cts": "js",
|
||||
".go": "go", ".rs": "rs",
|
||||
".java": "jvm", ".kt": "jvm", ".scala": "jvm", ".groovy": "jvm",
|
||||
# C, C++, and ObjC interoperate within one compilation unit: a method
|
||||
# declared in a shared `.h` is defined/called from a `.c`/`.cpp`/`.m`
|
||||
# sibling, so a cross-file INFERRED call from impl to its header decl
|
||||
# is legitimate, not a phantom name-collision across languages. Treat
|
||||
# the whole C family as one so the receiver-typed C++/ObjC member-call
|
||||
# resolvers' header-targeting edges survive build (#1547/#1556).
|
||||
".c": "c", ".h": "c", ".cc": "c", ".cpp": "c", ".hpp": "c",
|
||||
".cxx": "c", ".hh": "c", ".hxx": "c",
|
||||
".cu": "c", ".cuh": "c", ".metal": "c", ".m": "c", ".mm": "c",
|
||||
".rb": "rb", ".php": "php", ".cs": "cs", ".swift": "swift", ".lua": "lua",
|
||||
}
|
||||
# Drop cross-language phantom edges — the same short names (render, parse,
|
||||
# time, ...) recur across language boundaries, so an unresolved target can
|
||||
# bind to a same-named node in another language. The extraction spec forbids
|
||||
# this for `calls`; it is equally invalid for `imports`/`references` (a
|
||||
# Python `import time` must not bind to a `time.ts`, #1749).
|
||||
_edge_rel = attrs.get("relation")
|
||||
if _edge_rel in ("calls", "imports", "imports_from", "references"):
|
||||
src_ext = Path(G.nodes[src].get("source_file") or "").suffix.lower()
|
||||
tgt_ext = Path(G.nodes[tgt].get("source_file") or "").suffix.lower()
|
||||
if src_ext and tgt_ext and _LANG_FAMILY.get(src_ext) != _LANG_FAMILY.get(tgt_ext):
|
||||
continue
|
||||
src_fam = _EDGE_LANG_FAMILY.get(src_ext)
|
||||
tgt_fam = _EDGE_LANG_FAMILY.get(tgt_ext)
|
||||
if _edge_rel == "calls":
|
||||
# Unchanged #1547/#1556 behavior: only INFERRED calls, and drop as
|
||||
# soon as either family differs (an unknown ext counts as different).
|
||||
if (
|
||||
attrs.get("confidence") == "INFERRED"
|
||||
and src_ext and tgt_ext and src_fam != tgt_fam
|
||||
):
|
||||
continue
|
||||
else:
|
||||
# imports/references: drop only when BOTH endpoints are known code
|
||||
# languages of different families, so a config->code reference
|
||||
# (unknown ext, e.g. a manifest) is never mistaken for a phantom.
|
||||
if src_fam is not None and tgt_fam is not None and src_fam != tgt_fam:
|
||||
continue
|
||||
# Preserve original edge direction - undirected graphs lose it otherwise,
|
||||
# causing display functions to show edges backwards.
|
||||
attrs["_src"] = src
|
||||
|
||||
@@ -877,3 +877,50 @@ def test_semantic_rekey_relative_vs_absolute_source_file():
|
||||
# absolute path with no resolvable root → skipped, not remapped to an abs-path id
|
||||
ab = [{"id": "api_readme", "source_file": "/abs/docs/v1/api/README.md", "type": "document"}]
|
||||
assert _semantic_id_remap(ab, None) == {}
|
||||
|
||||
|
||||
def test_cross_language_imports_references_are_dropped():
|
||||
"""#1749: an `imports`/`references` edge must not bind across a language
|
||||
family. A Python `import time` that resolved by bare stem onto a `time.ts`
|
||||
file node welds the two language halves together at a phantom edge; the spec
|
||||
forbids this for `calls` and it is equally invalid here."""
|
||||
ext = {
|
||||
"nodes": [
|
||||
{"id": "backend_worker_py", "label": "worker.py", "file_type": "code",
|
||||
"source_file": "backend/worker.py", "source_location": "L1", "_origin": "ast"},
|
||||
{"id": "src_time_ts", "label": "time.ts", "file_type": "code",
|
||||
"source_file": "src/time.ts", "source_location": "L1", "_origin": "ast"},
|
||||
{"id": "src_util_ts", "label": "util.ts", "file_type": "code",
|
||||
"source_file": "src/util.ts", "source_location": "L1", "_origin": "ast"},
|
||||
],
|
||||
"edges": [
|
||||
# phantom: Python file importing a TS file (cross-language)
|
||||
{"source": "backend_worker_py", "target": "src_time_ts", "relation": "imports",
|
||||
"confidence": "EXTRACTED", "source_file": "backend/worker.py", "weight": 1.0},
|
||||
# legit: TS importing TS (same family) must survive
|
||||
{"source": "src_time_ts", "target": "src_util_ts", "relation": "imports",
|
||||
"confidence": "EXTRACTED", "source_file": "src/time.ts", "weight": 1.0},
|
||||
],
|
||||
}
|
||||
G = build_from_json(ext, directed=False)
|
||||
assert not G.has_edge("backend_worker_py", "src_time_ts"), "cross-language import must be dropped"
|
||||
assert G.has_edge("src_time_ts", "src_util_ts"), "same-family (TS->TS) import must survive"
|
||||
|
||||
|
||||
def test_cross_family_reference_to_unknown_ext_is_kept():
|
||||
"""The #1749 guard only drops when BOTH endpoints are known code languages,
|
||||
so a reference from a config/manifest (unknown ext) to a code file is kept."""
|
||||
ext = {
|
||||
"nodes": [
|
||||
{"id": "pkg_json", "label": "package.json", "file_type": "code",
|
||||
"source_file": "package.json", "source_location": "L1", "_origin": "ast"},
|
||||
{"id": "src_app_ts", "label": "app.ts", "file_type": "code",
|
||||
"source_file": "src/app.ts", "source_location": "L1", "_origin": "ast"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "pkg_json", "target": "src_app_ts", "relation": "references",
|
||||
"confidence": "EXTRACTED", "source_file": "package.json", "weight": 1.0},
|
||||
],
|
||||
}
|
||||
G = build_from_json(ext, directed=False)
|
||||
assert G.has_edge("pkg_json", "src_app_ts"), "config->code reference (unknown ext) must be kept"
|
||||
|
||||
Reference in New Issue
Block a user