diff --git a/CHANGELOG.md b/CHANGELOG.md index 4be9546..dd0452e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.12 (unreleased) +- Fix: `imports`/`references` edges no longer bind across a language boundary (#1749, thanks @philberndt). The spec already forbids cross-language `calls`, but an unresolved Python `import time` could still resolve by bare stem onto a `src/time.ts` file node — welding a polyglot repo's halves together at a phantom edge (in the reporter's repo, 3 such edges were the *only* thing bridging 2409 Python nodes to 1403 TS nodes, inflating `time.ts` betweenness ~90x and making it the #1 "god node"). The build-time cross-language guard now covers `imports`/`imports_from`/`references` in addition to `calls`, dropping an edge only when both endpoints are known code languages of different interop families (so a config/manifest → code reference is untouched). + - Fix: files whose extractor bailed out for a missing optional dependency no longer vanish without a trace (#1745, thanks @rithyKabir). `.sql` files (and other extra-gated languages) have a dispatch entry, so the #1689 no-extractor warning can't fire, and `extract_sql` returns an error result when `tree-sitter-sql` is absent, so the #1666 zero-node warning skips it too — the graph built "successfully" while an entire SQL corpus contributed nothing. `extract()` now surfaces these grouped by extension, naming the extra that restores the language (e.g. `pip install "graphifyy[sql]"`). - Fix: `build_from_json` is deterministic across process runs again (#1753, thanks @erasmust-dotcom). The ghost-node merge iterated `set(G.nodes())`, so which node survived a `(basename, label)` collision depended on CPython's per-process string-hash seed — rebuilding the same extraction JSON in a fresh process could silently pick a different canonical id (breaking the cluster→relabel workflow with a `KeyError` on an id that vanished). The Pass 1/Pass 2 loops now iterate in sorted order. Additionally, two non-AST (semantic) nodes sharing a key but from *different* files are now treated as distinct concepts and both survive (mirroring the AST/AST ambiguity guard #1257) instead of one arbitrarily merging away; a genuine same-file duplicate still collapses. diff --git a/graphify/build.py b/graphify/build.py index 8ae6e78..1686d19 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -33,6 +33,26 @@ from .paths import default_graph_json as _default_graph_json from .validate import validate_extraction +# Language interop families, keyed by extension, for the cross-language phantom-edge +# guard in the edge loop below. Families group by REAL interop (JS/TS share a module +# graph; C/C++/ObjC share a compilation unit via headers; JVM langs share bytecode), +# so a legitimate TS->JS import or C impl->header call survives, while a Python +# `import time` binding to a `time.ts` (#1749) or a cross-language INFERRED `calls` +# edge (#1547/#1556) is dropped. Kept local to build.py (not imported from extract.py, +# which imports build.py — a cycle) and deliberately mirrors extract._LANG_FAMILY_BY_EXT. +_EDGE_LANG_FAMILY: dict[str, str] = { + ".py": "py", ".pyi": "py", + ".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js", + ".ts": "js", ".tsx": "js", ".mts": "js", ".cts": "js", + ".go": "go", ".rs": "rs", + ".java": "jvm", ".kt": "jvm", ".scala": "jvm", ".groovy": "jvm", + ".c": "c", ".h": "c", ".cc": "c", ".cpp": "c", ".hpp": "c", + ".cxx": "c", ".hh": "c", ".hxx": "c", + ".cu": "c", ".cuh": "c", ".metal": "c", ".m": "c", ".mm": "c", + ".rb": "rb", ".php": "php", ".cs": "cs", ".swift": "swift", ".lua": "lua", +} + + # Synonym mapper for known invalid file_type values that LLM subagents commonly # emit. Keeps semantic intent close (markdown→document, tool→code) and falls # back to "concept" for any other invalid value (see #840). @@ -628,31 +648,31 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat ) if "source_file" in attrs: attrs["source_file"] = _norm_source_file(attrs["source_file"], _root) - # Drop cross-language INFERRED `calls` edges — same short names (render, - # parse, etc.) appear across language boundaries in multi-language chunks, - # producing phantom edges that don't represent real call relationships. - if attrs.get("relation") == "calls" and attrs.get("confidence") == "INFERRED": - _LANG_FAMILY: dict[str, str] = { - ".py": "py", ".pyi": "py", - ".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js", - ".ts": "js", ".tsx": "js", ".mts": "js", ".cts": "js", - ".go": "go", ".rs": "rs", - ".java": "jvm", ".kt": "jvm", ".scala": "jvm", ".groovy": "jvm", - # C, C++, and ObjC interoperate within one compilation unit: a method - # declared in a shared `.h` is defined/called from a `.c`/`.cpp`/`.m` - # sibling, so a cross-file INFERRED call from impl to its header decl - # is legitimate, not a phantom name-collision across languages. Treat - # the whole C family as one so the receiver-typed C++/ObjC member-call - # resolvers' header-targeting edges survive build (#1547/#1556). - ".c": "c", ".h": "c", ".cc": "c", ".cpp": "c", ".hpp": "c", - ".cxx": "c", ".hh": "c", ".hxx": "c", - ".cu": "c", ".cuh": "c", ".metal": "c", ".m": "c", ".mm": "c", - ".rb": "rb", ".php": "php", ".cs": "cs", ".swift": "swift", ".lua": "lua", - } + # Drop cross-language phantom edges — the same short names (render, parse, + # time, ...) recur across language boundaries, so an unresolved target can + # bind to a same-named node in another language. The extraction spec forbids + # this for `calls`; it is equally invalid for `imports`/`references` (a + # Python `import time` must not bind to a `time.ts`, #1749). + _edge_rel = attrs.get("relation") + if _edge_rel in ("calls", "imports", "imports_from", "references"): src_ext = Path(G.nodes[src].get("source_file") or "").suffix.lower() tgt_ext = Path(G.nodes[tgt].get("source_file") or "").suffix.lower() - if src_ext and tgt_ext and _LANG_FAMILY.get(src_ext) != _LANG_FAMILY.get(tgt_ext): - continue + src_fam = _EDGE_LANG_FAMILY.get(src_ext) + tgt_fam = _EDGE_LANG_FAMILY.get(tgt_ext) + if _edge_rel == "calls": + # Unchanged #1547/#1556 behavior: only INFERRED calls, and drop as + # soon as either family differs (an unknown ext counts as different). + if ( + attrs.get("confidence") == "INFERRED" + and src_ext and tgt_ext and src_fam != tgt_fam + ): + continue + else: + # imports/references: drop only when BOTH endpoints are known code + # languages of different families, so a config->code reference + # (unknown ext, e.g. a manifest) is never mistaken for a phantom. + if src_fam is not None and tgt_fam is not None and src_fam != tgt_fam: + continue # Preserve original edge direction - undirected graphs lose it otherwise, # causing display functions to show edges backwards. attrs["_src"] = src diff --git a/tests/test_build.py b/tests/test_build.py index 5d2e2df..c0bc0e3 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -877,3 +877,50 @@ def test_semantic_rekey_relative_vs_absolute_source_file(): # absolute path with no resolvable root → skipped, not remapped to an abs-path id ab = [{"id": "api_readme", "source_file": "/abs/docs/v1/api/README.md", "type": "document"}] assert _semantic_id_remap(ab, None) == {} + + +def test_cross_language_imports_references_are_dropped(): + """#1749: an `imports`/`references` edge must not bind across a language + family. A Python `import time` that resolved by bare stem onto a `time.ts` + file node welds the two language halves together at a phantom edge; the spec + forbids this for `calls` and it is equally invalid here.""" + ext = { + "nodes": [ + {"id": "backend_worker_py", "label": "worker.py", "file_type": "code", + "source_file": "backend/worker.py", "source_location": "L1", "_origin": "ast"}, + {"id": "src_time_ts", "label": "time.ts", "file_type": "code", + "source_file": "src/time.ts", "source_location": "L1", "_origin": "ast"}, + {"id": "src_util_ts", "label": "util.ts", "file_type": "code", + "source_file": "src/util.ts", "source_location": "L1", "_origin": "ast"}, + ], + "edges": [ + # phantom: Python file importing a TS file (cross-language) + {"source": "backend_worker_py", "target": "src_time_ts", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "backend/worker.py", "weight": 1.0}, + # legit: TS importing TS (same family) must survive + {"source": "src_time_ts", "target": "src_util_ts", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "src/time.ts", "weight": 1.0}, + ], + } + G = build_from_json(ext, directed=False) + assert not G.has_edge("backend_worker_py", "src_time_ts"), "cross-language import must be dropped" + assert G.has_edge("src_time_ts", "src_util_ts"), "same-family (TS->TS) import must survive" + + +def test_cross_family_reference_to_unknown_ext_is_kept(): + """The #1749 guard only drops when BOTH endpoints are known code languages, + so a reference from a config/manifest (unknown ext) to a code file is kept.""" + ext = { + "nodes": [ + {"id": "pkg_json", "label": "package.json", "file_type": "code", + "source_file": "package.json", "source_location": "L1", "_origin": "ast"}, + {"id": "src_app_ts", "label": "app.ts", "file_type": "code", + "source_file": "src/app.ts", "source_location": "L1", "_origin": "ast"}, + ], + "edges": [ + {"source": "pkg_json", "target": "src_app_ts", "relation": "references", + "confidence": "EXTRACTED", "source_file": "package.json", "weight": 1.0}, + ], + } + G = build_from_json(ext, directed=False) + assert G.has_edge("pkg_json", "src_app_ts"), "config->code reference (unknown ext) must be kept"