fix(extract): case-sensitive cross-file resolution in case-sensitive languages (#1581)
Cross-file name resolution folded case for every language, so `from pathlib import Path` resolved to a shell script's `export PATH=...` node — one variable becoming the corpus's #1 god-node (266 false incoming edges on a real repo), polluting god-node rankings, affected blast-radius, and clustering. Reported with a precise diagnosis by @sheik-hiiobd. Case is semantic in Python/Rust/Go/Java/C#/Kotlin/Swift/Ruby/C/C++/JS/TS: `Path` (class), `PATH` (env var), `path` (variable) are distinct. Fix gates folding by language at the two resolution sites the repro exercised: - global cross-file CALL resolver: index by exact case; a folded index is built only for case-insensitive-language nodes (PHP/SQL/Nim) and consulted only when the calling file is such a language. - type-reference STUB rewire (_rewire_unique_stub_nodes): match stubs to real defs by exact case, with a folded fallback restricted to case-insensitive- language definitions — so a case-sensitive `PATH` can never absorb a `Path`. For case-sensitive languages this only ever removes false edges. Concept/doc dedup (dedup.py, guarded to non-code nodes) is intentionally left folding. Regression tests: Python `Path` no longer hits shell `PATH`; a case-differing cross-file ref doesn't resolve; exact-case resolution still works; PHP fold preserved. Full suite 2777. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
532a20e775
commit
784e9c833e
@@ -4,6 +4,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu
|
|||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
- Fix: cross-file name resolution now respects case in case-sensitive languages (#1581, thanks @sheik-hiiobd). Resolution matched identifiers case-insensitively for every language, so in Python/Rust/Go/Java/etc. `from pathlib import Path` resolved to an unrelated shell-script `export PATH=...` node — a single variable becoming the corpus's #1 god-node (266 false incoming edges on one real repo), inflating god-node rankings, `affected` blast-radius, and community assignment. Both the cross-file call resolver and the type-reference stub-rewire now match by exact case; only genuinely case-insensitive languages (PHP functions/classes, SQL, Nim) still fold. For case-sensitive languages this only ever removes false edges.
|
||||||
- Fix: Julia qualified / relative / scoped-selected imports now emit edges (#1580, thanks @Synvoya). Only bare `using Foo` was handled; `using Base.Threads` (scoped), `using ..Parent` (relative import_path), and the scoped package of `import Base.Threads: nthreads` were dropped.
|
- Fix: Julia qualified / relative / scoped-selected imports now emit edges (#1580, thanks @Synvoya). Only bare `using Foo` was handled; `using Base.Threads` (scoped), `using ..Parent` (relative import_path), and the scoped package of `import Base.Threads: nthreads` were dropped.
|
||||||
- Fix: Rust tuple-struct field types now emit `references` edges (#1582, thanks @Synvoya). `struct Wrapper(Logger, Vec<Config>);` referenced nothing — positional fields nest under `ordered_field_declaration_list` with no `field_declaration` wrapper, the same shape as tuple enum variants (#1579); that path wasn't traversed for structs.
|
- Fix: Rust tuple-struct field types now emit `references` edges (#1582, thanks @Synvoya). `struct Wrapper(Logger, Vec<Config>);` referenced nothing — positional fields nest under `ordered_field_declaration_list` with no `field_declaration` wrapper, the same shape as tuple enum variants (#1579); that path wasn't traversed for structs.
|
||||||
- Fix: SystemVerilog class properties with leading qualifiers now emit field `references` (#1583, thanks @Synvoya). The field regex only matched unqualified `<type> <name>;`, so `rand Config x;` / `protected Base b;` (qualifier + type + name) failed to match and their type references were dropped.
|
- Fix: SystemVerilog class properties with leading qualifiers now emit field `references` (#1583, thanks @Synvoya). The field regex only matched unqualified `<type> <name>;`, so `rand Config x;` / `protected Base b;` (qualifier + type + name) failed to match and their type references were dropped.
|
||||||
|
|||||||
+50
-8
@@ -9032,9 +9032,27 @@ def _canonicalize_csharp_namespace_nodes(all_nodes: list[dict], all_edges: list[
|
|||||||
all_nodes[:] = [node for node in all_nodes if id(node) not in drop_node_ids]
|
all_nodes[:] = [node for node in all_nodes if id(node) not in drop_node_ids]
|
||||||
|
|
||||||
|
|
||||||
def _node_label_key(node: dict) -> str:
|
# Languages whose identifiers are case-insensitive, so cross-file name resolution
|
||||||
|
# may fold case. Everywhere else, case is semantic (`Path` the class vs `PATH` the
|
||||||
|
# env var are distinct) and folding manufactures false edges / super-hubs (#1581).
|
||||||
|
_CASE_INSENSITIVE_EXTS = frozenset({
|
||||||
|
".php", ".phtml", ".php3", ".php4", ".php5", ".php7", ".phps", # PHP fns/classes
|
||||||
|
".sql", # SQL identifiers
|
||||||
|
".nim", ".nims", ".nimble", # Nim (style-insensitive)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _lang_is_case_insensitive(source_file: object) -> bool:
|
||||||
|
"""True when the file's language resolves identifiers case-insensitively (#1581)."""
|
||||||
|
if not source_file:
|
||||||
|
return False
|
||||||
|
return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS
|
||||||
|
|
||||||
|
|
||||||
|
def _node_label_key(node: dict, fold: bool = False) -> str:
|
||||||
label = str(node.get("label", "")).strip()
|
label = str(node.get("label", "")).strip()
|
||||||
return re.sub(r"[^a-zA-Z0-9]+", "", label).lower()
|
key = re.sub(r"[^a-zA-Z0-9]+", "", label)
|
||||||
|
return key.lower() if fold else key
|
||||||
|
|
||||||
|
|
||||||
def _is_type_like_definition(node: dict) -> bool:
|
def _is_type_like_definition(node: dict) -> bool:
|
||||||
@@ -9052,7 +9070,8 @@ def _is_type_like_definition(node: dict) -> bool:
|
|||||||
|
|
||||||
def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None:
|
def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None:
|
||||||
"""Map unresolved no-source stubs to a unique real definition with the same label."""
|
"""Map unresolved no-source stubs to a unique real definition with the same label."""
|
||||||
real_by_label: dict[str, list[dict]] = {}
|
real_by_label: dict[str, list[dict]] = {} # exact-case (all languages)
|
||||||
|
real_by_label_ci: dict[str, list[dict]] = {} # case-INSENSITIVE-language reals only
|
||||||
stubs: list[dict] = []
|
stubs: list[dict] = []
|
||||||
|
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
@@ -9061,7 +9080,13 @@ def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None:
|
|||||||
continue
|
continue
|
||||||
if node.get("source_file"):
|
if node.get("source_file"):
|
||||||
if _is_type_like_definition(node):
|
if _is_type_like_definition(node):
|
||||||
|
# Match stubs case-SENSITIVELY: a `Path` reference must not rewire to a
|
||||||
|
# `PATH` env var (#1581). Fold only for genuinely case-insensitive
|
||||||
|
# languages, where `foo` legitimately resolves to `Foo`.
|
||||||
real_by_label.setdefault(key, []).append(node)
|
real_by_label.setdefault(key, []).append(node)
|
||||||
|
if _lang_is_case_insensitive(node.get("source_file")):
|
||||||
|
real_by_label_ci.setdefault(
|
||||||
|
_node_label_key(node, fold=True), []).append(node)
|
||||||
continue
|
continue
|
||||||
stubs.append(node)
|
stubs.append(node)
|
||||||
|
|
||||||
@@ -9072,7 +9097,12 @@ def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None:
|
|||||||
continue
|
continue
|
||||||
candidates = real_by_label.get(_node_label_key(stub), [])
|
candidates = real_by_label.get(_node_label_key(stub), [])
|
||||||
if len(candidates) != 1:
|
if len(candidates) != 1:
|
||||||
continue
|
# No unique exact match — fall back to a case-insensitive match, but
|
||||||
|
# only against case-insensitive-language definitions (so a case-sensitive
|
||||||
|
# `PATH` can never absorb a `Path` reference).
|
||||||
|
candidates = real_by_label_ci.get(_node_label_key(stub, fold=True), [])
|
||||||
|
if len(candidates) != 1:
|
||||||
|
continue
|
||||||
target_id = candidates[0].get("id")
|
target_id = candidates[0].get("id")
|
||||||
if isinstance(target_id, str) and target_id and target_id != stub_id:
|
if isinstance(target_id, str) and target_id and target_id != stub_id:
|
||||||
remap[stub_id] = target_id
|
remap[stub_id] = target_id
|
||||||
@@ -15464,15 +15494,21 @@ def extract(
|
|||||||
# Build label -> node_id index for cross-file call resolution.
|
# Build label -> node_id index for cross-file call resolution.
|
||||||
# Skip rationale nodes (their labels are docstring text, not callable
|
# Skip rationale nodes (their labels are docstring text, not callable
|
||||||
# identifiers, and they were polluting matches for short names — #563).
|
# identifiers, and they were polluting matches for short names — #563).
|
||||||
global_label_to_nids: dict[str, list[str]] = {}
|
global_label_to_nids: dict[str, list[str]] = {} # exact-case (all languages)
|
||||||
|
global_label_to_nids_ci: dict[str, list[str]] = {} # case-INSENSITIVE-language nodes
|
||||||
for n in all_nodes:
|
for n in all_nodes:
|
||||||
if n.get("file_type") == "rationale" or n.get("type") == "namespace":
|
if n.get("file_type") == "rationale" or n.get("type") == "namespace":
|
||||||
continue
|
continue
|
||||||
raw = n.get("label", "")
|
raw = n.get("label", "")
|
||||||
normalised = raw.strip("()").lstrip(".")
|
normalised = raw.strip("()").lstrip(".")
|
||||||
if normalised:
|
if normalised:
|
||||||
key = normalised.lower()
|
# Case is semantic in most languages, so index (and match, below) by exact
|
||||||
global_label_to_nids.setdefault(key, []).append(n["id"])
|
# case — folding collapses `Path` (class) into `PATH` (env var) and makes a
|
||||||
|
# single shell variable the #1 god-node (#1581). Only case-insensitive
|
||||||
|
# languages (PHP/SQL/Nim) also get a folded key for legitimate fold-matching.
|
||||||
|
global_label_to_nids.setdefault(normalised, []).append(n["id"])
|
||||||
|
if _lang_is_case_insensitive(n.get("source_file")):
|
||||||
|
global_label_to_nids_ci.setdefault(normalised.lower(), []).append(n["id"])
|
||||||
|
|
||||||
# Callable-def ids for the indirect_call callable guard, read from the `_callable`
|
# Callable-def ids for the indirect_call callable guard, read from the `_callable`
|
||||||
# marker on the FINAL (post-remap) nodes — so a callback resolves only to a real
|
# marker on the FINAL (post-remap) nodes — so a callback resolves only to a real
|
||||||
@@ -15532,7 +15568,13 @@ def extract(
|
|||||||
# and collides with any top-level function named "log" in the corpus.
|
# and collides with any top-level function named "log" in the corpus.
|
||||||
if rc.get("is_member_call"):
|
if rc.get("is_member_call"):
|
||||||
continue
|
continue
|
||||||
candidates = global_label_to_nids.get(callee.lower(), [])
|
# Exact-case match first (case is semantic). Fold only when the CALLING
|
||||||
|
# file's language is case-insensitive, and only against the folded index of
|
||||||
|
# case-insensitive-language definitions — so a Python `Path()` call can never
|
||||||
|
# resolve to a shell `PATH` node (#1581).
|
||||||
|
candidates = global_label_to_nids.get(callee, [])
|
||||||
|
if not candidates and _lang_is_case_insensitive(rc.get("source_file")):
|
||||||
|
candidates = global_label_to_nids_ci.get(callee.lower(), [])
|
||||||
if not candidates:
|
if not candidates:
|
||||||
continue
|
continue
|
||||||
caller = rc["caller_nid"]
|
caller = rc["caller_nid"]
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
"""Cross-file name resolution respects case in case-sensitive languages (#1581).
|
||||||
|
|
||||||
|
Case is semantic in most languages: `Path` (a class), `PATH` (an env var), and
|
||||||
|
`path` (a variable) are distinct. Cross-file resolution used to fold case for every
|
||||||
|
language, so `from pathlib import Path` (ubiquitous) resolved to a shell script's
|
||||||
|
`export PATH=...` node — turning one shell variable into the corpus's #1 god-node.
|
||||||
|
|
||||||
|
These tests pin: case-sensitive languages match by exact case (removing that false
|
||||||
|
edge), while genuinely case-insensitive languages (PHP) still fold.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from graphify.extract import extract
|
||||||
|
|
||||||
|
|
||||||
|
def _extract(tmp_path, files: dict[str, str]):
|
||||||
|
for name, body in files.items():
|
||||||
|
(tmp_path / name).write_text(body)
|
||||||
|
old = os.getcwd()
|
||||||
|
try:
|
||||||
|
os.chdir(tmp_path)
|
||||||
|
r = extract([Path(n) for n in files], cache_root=tmp_path)
|
||||||
|
finally:
|
||||||
|
os.chdir(old)
|
||||||
|
return r
|
||||||
|
|
||||||
|
|
||||||
|
def _labels(r):
|
||||||
|
return {n["id"]: n["label"] for n in r["nodes"]}
|
||||||
|
|
||||||
|
|
||||||
|
def test_python_Path_does_not_resolve_to_shell_PATH(tmp_path):
|
||||||
|
r = _extract(tmp_path, {
|
||||||
|
"run.sh": "export PATH=/usr/local/bin:$PATH\n",
|
||||||
|
"mod.py": (
|
||||||
|
"from pathlib import Path\n"
|
||||||
|
"def load(p: Path) -> Path:\n return Path(p)\n"
|
||||||
|
"def other():\n return load(Path('x'))\n"
|
||||||
|
),
|
||||||
|
})
|
||||||
|
lbl = _labels(r)
|
||||||
|
path_nid = next((n["id"] for n in r["nodes"] if n["label"] == "PATH"), None)
|
||||||
|
assert path_nid is not None
|
||||||
|
# No edge from the Python functions should land on the shell PATH node
|
||||||
|
false_edges = [
|
||||||
|
e for e in r["edges"]
|
||||||
|
if e["target"] == path_nid and lbl.get(e["source"], "").startswith(("load", "other"))
|
||||||
|
]
|
||||||
|
assert not false_edges, f"Python Path leaked onto shell PATH: {false_edges}"
|
||||||
|
# PATH keeps only its own `defines` edge (from run.sh), not a false super-hub
|
||||||
|
assert sum(1 for e in r["edges"] if e["target"] == path_nid) <= 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_case_sensitive_cross_file_ref_respects_case(tmp_path):
|
||||||
|
r = _extract(tmp_path, {
|
||||||
|
"consts.rs": 'pub const PATH: &str = "/x";\n',
|
||||||
|
"use.rs": "struct Wrap(Path);\n", # `Path` — no such node in the corpus
|
||||||
|
})
|
||||||
|
lbl = _labels(r)
|
||||||
|
path_nid = next((n["id"] for n in r["nodes"] if n["label"] == "PATH"), None)
|
||||||
|
xref = [e for e in r["edges"] if e["target"] == path_nid and lbl.get(e["source"]) == "Wrap"]
|
||||||
|
assert not xref, "a `Path` reference must not resolve to a case-differing `PATH`"
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_case_cross_file_still_resolves(tmp_path):
|
||||||
|
r = _extract(tmp_path, {
|
||||||
|
"h.py": "def helper():\n return 1\n",
|
||||||
|
"m.py": "from h import helper\ndef go():\n return helper()\n",
|
||||||
|
})
|
||||||
|
lbl = _labels(r)
|
||||||
|
calls = {(lbl.get(e["source"]), lbl.get(e["target"]))
|
||||||
|
for e in r["edges"] if e["relation"] == "calls"}
|
||||||
|
assert ("go()", "helper()") in calls
|
||||||
|
|
||||||
|
|
||||||
|
def test_php_case_insensitive_resolution_preserved(tmp_path):
|
||||||
|
r = _extract(tmp_path, {
|
||||||
|
"lib.php": "<?php\nfunction Greet() { return 1; }\n",
|
||||||
|
"main.php": "<?php\nfunction run() { return greet(); }\n",
|
||||||
|
})
|
||||||
|
lbl = _labels(r)
|
||||||
|
calls = {(lbl.get(e["source"]), lbl.get(e["target"]))
|
||||||
|
for e in r["edges"] if e["relation"] == "calls"}
|
||||||
|
assert ("run()", "Greet()") in calls, "PHP identifiers are case-insensitive; fold must still apply"
|
||||||
Reference in New Issue
Block a user