diff --git a/graphify/__main__.py b/graphify/__main__.py index 9ca7839..8ca1a29 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -23,6 +23,25 @@ def _default_graph_path() -> str: return str(Path(_GRAPHIFY_OUT) / "graph.json") +def _enforce_graph_size_cap_or_exit(gp: Path) -> None: + """Reject oversized graph files before parsing (CLI exit-on-fail flavor). + + Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the + raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. + Use this from ``__main__.py`` subcommands that already use the ``print + + sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, + ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, + ``global_graph``, ``watch``, ``export``) call the security helper directly + and let the ``ValueError`` propagate. + """ + from graphify.security import check_graph_file_size_cap + try: + check_graph_file_size_cap(gp) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + def _check_skill_version(skill_dst: Path) -> None: """Warn if the installed skill is from an older graphify version.""" version_file = skill_dst.parent / ".graphify_version" @@ -1218,6 +1237,16 @@ def main() -> None: print(" --graph path to graph.json (default graphify-out/graph.json)") print(" explain \"X\" plain-language explanation of a node and its neighbors") print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" diagnose multigraph report same-endpoint edge collapse risk in graph.json") + print(" --graph path to graph/extraction JSON") + print(" (default graphify-out/graph.json)") + print(" --json emit machine-readable JSON") + print(" --max-examples N max same-endpoint examples to print (default 5)") + print(" --directed force directed post-build simulation") + print(" --undirected force undirected post-build simulation") + print(" (default follows JSON directed flag;") + print(" raw extraction with no flag defaults directed)") + print(" --extract-path PATH extractor source for suppression scan") print(" clone clone a GitHub repo locally and print its path for /graphify") print(" merge-driver git merge driver: union-merge two graph.json files (set up via hook install)") print(" merge-graphs merge two or more graph.json files into one cross-repo graph") @@ -1534,6 +1563,7 @@ def main() -> None: if not gp.suffix == ".json": print(f"error: graph file must be a .json file", file=sys.stderr) sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) try: import json as _json import networkx as _nx @@ -1594,6 +1624,7 @@ def main() -> None: if not gp.exists(): print(f"error: graph file not found: {gp}", file=sys.stderr) sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) _raw = json.loads(gp.read_text(encoding="utf-8")) if "links" not in _raw and "edges" in _raw: _raw = dict(_raw, links=_raw["edges"]) @@ -1675,6 +1706,7 @@ def main() -> None: if not gp.exists(): print(f"error: graph file not found: {gp}", file=sys.stderr) sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) _raw = json.loads(gp.read_text(encoding="utf-8")) if "links" not in _raw and "edges" in _raw: _raw = dict(_raw, links=_raw["edges"]) @@ -1713,6 +1745,100 @@ def main() -> None: if len(connections) > 20: print(f" ... and {len(connections) - 20} more") + elif cmd == "diagnose": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd != "multigraph": + print( + "Usage: graphify diagnose multigraph " + "[--graph path] [--json] [--max-examples N] " + "[--directed] [--undirected] [--extract-path path]", + file=sys.stderr, + ) + sys.exit(1) + + graph_path = Path(_default_graph_path()) + max_examples = 5 + directed: bool | None = None + direction_flag: str | None = None + json_output = False + extract_path: Path | None = None + + i = 3 + while i < len(sys.argv): + arg = sys.argv[i] + if arg == "--graph": + i += 1 + if i >= len(sys.argv): + print("error: --graph requires a path", file=sys.stderr) + sys.exit(1) + graph_path = Path(sys.argv[i]) + elif arg == "--json": + json_output = True + elif arg == "--max-examples": + i += 1 + if i >= len(sys.argv): + print("error: --max-examples requires an integer", file=sys.stderr) + sys.exit(1) + try: + max_examples = int(sys.argv[i]) + except ValueError: + print("error: --max-examples requires an integer", file=sys.stderr) + sys.exit(1) + if max_examples < 0: + print("error: --max-examples must be >= 0", file=sys.stderr) + sys.exit(1) + elif arg == "--directed": + if direction_flag == "undirected": + print( + "error: --directed and --undirected are mutually exclusive", + file=sys.stderr, + ) + sys.exit(1) + direction_flag = "directed" + directed = True + elif arg == "--undirected": + if direction_flag == "directed": + print( + "error: --directed and --undirected are mutually exclusive", + file=sys.stderr, + ) + sys.exit(1) + direction_flag = "undirected" + directed = False + elif arg == "--extract-path": + i += 1 + if i >= len(sys.argv): + print("error: --extract-path requires a path", file=sys.stderr) + sys.exit(1) + extract_path = Path(sys.argv[i]) + else: + print(f"error: unknown diagnose option {arg}", file=sys.stderr) + sys.exit(1) + i += 1 + + from graphify.diagnostics import ( + diagnose_file, + format_diagnostic_json, + format_diagnostic_report, + ) + + try: + summary = diagnose_file( + graph_path, + directed=directed, + root=Path(".").resolve(), + max_examples=max_examples, + extract_path=extract_path, + ) + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + if json_output: + print(json.dumps(format_diagnostic_json(summary), indent=2)) + else: + print(format_diagnostic_report(summary)) + elif cmd == "add": if len(sys.argv) < 3: print("Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", file=sys.stderr) @@ -1798,6 +1924,7 @@ def main() -> None: from graphify.report import generate from graphify.export import to_json, to_html print("Loading existing graph...") + _enforce_graph_size_cap_or_exit(graph_json) _raw = json.loads(graph_json.read_text(encoding="utf-8")) _directed = bool(_raw.get("directed", False)) G = build_from_json(_raw, directed=_directed) @@ -1958,6 +2085,7 @@ def main() -> None: if not graph_path.is_file(): print(f"error: graph.json not found at {graph_path}", file=sys.stderr) sys.exit(1) + _enforce_graph_size_cap_or_exit(graph_path) if output_path is None: output_path = graph_path.parent / "GRAPH_TREE.html" out = write_tree_html( @@ -2046,6 +2174,7 @@ def main() -> None: if not gp.exists(): print(f"error: not found: {gp}", file=sys.stderr) sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) data = json.loads(gp.read_text(encoding="utf-8")) # Normalize edges/links key before loading — graphify writes "links" # via node_link_data but older runs may have used "edges" (#738). @@ -2232,6 +2361,7 @@ def main() -> None: from networkx.readwrite import json_graph as _jg from graphify.build import build_from_json as _bfj + _enforce_graph_size_cap_or_exit(graph_path) _raw = json.loads(graph_path.read_text(encoding="utf-8")) if "links" not in _raw and "edges" in _raw: _raw = dict(_raw, links=_raw["edges"]) @@ -2326,6 +2456,7 @@ def main() -> None: elif cmd == "benchmark": from graphify.benchmark import run_benchmark, print_benchmark graph_path = sys.argv[2] if len(sys.argv) > 2 else "graphify-out/graph.json" + _enforce_graph_size_cap_or_exit(Path(graph_path)) # Try to load corpus_words from detect output corpus_words = None detect_path = Path(".graphify_detect.json") diff --git a/graphify/benchmark.py b/graphify/benchmark.py index e57da1c..eabade2 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -98,6 +98,8 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ + from graphify.security import check_graph_file_size_cap + check_graph_file_size_cap(Path(graph_path)) data = json.loads(Path(graph_path).read_text(encoding="utf-8")) try: G = json_graph.node_link_graph(data, edges="links") diff --git a/graphify/build.py b/graphify/build.py index cc229fd..68f50bc 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -303,6 +303,8 @@ def build_merge( # was inserted before the caller. The _src/_tgt direction-preserving # attrs are popped before saving in export.py, so going through the # NetworkX round-trip loses direction permanently (#760). + from graphify.security import check_graph_file_size_cap + check_graph_file_size_cap(graph_path) data = json.loads(graph_path.read_text(encoding="utf-8")) links_key = "links" if "links" in data else "edges" existing_nodes = list(data.get("nodes", [])) diff --git a/graphify/callflow_html.py b/graphify/callflow_html.py index 96d4f1d..3db0874 100644 --- a/graphify/callflow_html.py +++ b/graphify/callflow_html.py @@ -252,6 +252,12 @@ def _node_link_payload(data: dict) -> tuple[list, list] | None: def load_graph(path: str | Path) -> tuple: """Load graph.json. Returns normalized (nodes, edges, hyperedges, metadata).""" + if path: + from graphify.security import check_graph_file_size_cap + try: + check_graph_file_size_cap(Path(path)) + except ValueError as exc: + raise SystemExit(f"ERROR: {exc}") from exc data = read_json(path) if not isinstance(data, dict): raise SystemExit(f"ERROR: graph file must contain a JSON object: {path}") diff --git a/graphify/detect.py b/graphify/detect.py index 16951ba..78f295d 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -4,6 +4,7 @@ import fnmatch import json import os import re +import shlex from enum import Enum from pathlib import Path @@ -112,24 +113,176 @@ _SHEBANG_CODE_INTERPRETERS = { } -def _shebang_file_type(path: Path) -> FileType | None: - """Peek at the first line of an extensionless file for a shebang.""" +def _split_env_s(value: str, rest: list[str]) -> list[str]: + """Re-tokenize an `env -S`/`--split-string` packed command, prepending the + operand to any trailing args. Returns the unpacked argv.""" + packed = " ".join([value, *rest]).strip() + return shlex.split(packed) + + +def _env_command_args(args: list[str], *, allow_split: bool = True) -> list[str]: + """Strip leading env(1) options and var assignments, return the trailing + command argv. Covers macOS/BSD and GNU coreutils env documented spellings. + + POSIX/macOS short forms: + env [-0iv] [-C workdir] [-P utilpath] [-S string] + [-u name] [name=value ...] [utility [argument ...]] + + GNU coreutils long/compact forms additionally supported: + --argv0=ARG / -a ARG / -aARG + --unset=NAME / --unset NAME / -u NAME / -uNAME + --chdir=DIR / --chdir DIR / -C DIR / -CDIR + --split-string=STRING / --split-string STRING + -S STRING / -SSTRING / -vS STRING / -vSSTRING + --ignore-environment / --null / --debug / --list-signal-handling + --default-signal[=SIG] / --ignore-signal[=SIG] / --block-signal[=SIG] + + `-S` / `--split-string` payloads are themselves env-style argument lists + per the GNU shebang synopsis: + #!/usr/bin/env -[v]S[option]... [name=value]... command [args]... + so after splitting the payload we recursively re-parse it with + `allow_split=False` (a nested -S inside a split payload is rejected to + bound recursion). + + Unknown hyphen-prefixed args yield [] (we refuse to guess whether + their next token is an interpreter or an operand). + """ + i = 0 + while i < len(args): + arg = args[i] + + if arg == "--": + return args[i + 1:] + + # Split-string forms: tokenize the packed payload, then re-parse it + # as env args (so leading assignments/flags inside the payload are + # skipped before the interpreter is identified). + if allow_split: + if arg == "-S": + if i + 1 >= len(args): + return [] + return _env_command_args( + _split_env_s(" ".join(args[i + 1:]), []), + allow_split=False, + ) + if arg.startswith("-S") and len(arg) > 2: + return _env_command_args( + _split_env_s(arg[2:], args[i + 1:]), + allow_split=False, + ) + if arg == "-vS": + if i + 1 >= len(args): + return [] + return _env_command_args( + _split_env_s(" ".join(args[i + 1:]), []), + allow_split=False, + ) + if arg.startswith("-vS") and len(arg) > 3: + return _env_command_args( + _split_env_s(arg[3:], args[i + 1:]), + allow_split=False, + ) + if arg.startswith("--split-string="): + return _env_command_args( + _split_env_s(arg.split("=", 1)[1], args[i + 1:]), + allow_split=False, + ) + if arg == "--split-string": + if i + 1 >= len(args): + return [] + return _env_command_args( + _split_env_s(args[i + 1], args[i + 2:]), + allow_split=False, + ) + + # Options with separate required operand + if arg in {"-u", "-C", "-P", "-a", "--unset", "--chdir", "--argv0"}: + if i + 2 > len(args): + return [] + i += 2 + continue + + # Clumped short option + operand + if ( + arg.startswith(("-u", "-C", "-P", "-a")) + and len(arg) > 2 + and not arg.startswith("--") + ): + i += 1 + continue + + # Long option with `=` operand + if arg.startswith(("--unset=", "--chdir=", "--argv0=")): + i += 1 + continue + + # No-operand flags + if arg in {"-", "-i", "-0", "-v", "--ignore-environment", "--null", + "--debug", "--list-signal-handling"}: + i += 1 + continue + + # Signal-handling long flags (with or without =SIG operand — we treat + # them as no-effect for interpreter-resolution purposes) + if arg.startswith(("--default-signal", "--ignore-signal", "--block-signal")): + i += 1 + continue + + # Unknown hyphen-prefixed: refuse to guess + if arg.startswith("-"): + return [] + + # Inline NAME=value assignment + if "=" in arg: + i += 1 + continue + + # First non-option, non-assignment token starts the command argv + return args[i:] + + return [] + + +def _shebang_interpreter(path: Path) -> str | None: + """Return the interpreter name from a shebang line. + + Handles forms that a naive parser misses: + - `#!/usr/bin/env -S python3 -u` (env -S split-args form, anywhere) + - `#!/usr/bin/env -i bash` (no-operand env flags) + - `#!/usr/bin/env -u VAR python3` (env options with operands) + - `#!/usr/bin/env -C /tmp python3` (env -C workdir) + - `#!/usr/bin/env -P /bin python3` (env -P utilpath) + - `#!/usr/bin/env DEBUG=1 python3` (inline var assignment) + - `#!"/usr/local/bin/python with spaces"` (shlex handles quotes) + + Returns the basename of the resolved interpreter, or None if there is + no shebang / the file is unreadable / parsing fails. + """ try: with path.open("rb") as f: - first = f.read(128) + first = f.read(256) if not first.startswith(b"#!"): return None - line = first.split(b"\n")[0].decode(errors="replace") - parts = line[2:].strip().split() + line = first.split(b"\n")[0].decode(errors="replace")[2:].strip() + parts = shlex.split(line) if not parts: return None - interp = parts[0].split("/")[-1] # /usr/bin/env → env - if interp == "env" and len(parts) > 1: - interp = parts[1].split("/")[-1] - if interp in _SHEBANG_CODE_INTERPRETERS: - return FileType.CODE - except OSError: - pass + interp = Path(parts[0]).name + if interp == "env": + env_args = _env_command_args(parts[1:]) + if not env_args: + return None + interp = Path(env_args[0]).name + return interp + except (OSError, ValueError): + return None + + +def _shebang_file_type(path: Path) -> FileType | None: + """Peek at the first line of an extensionless file for a shebang.""" + interp = _shebang_interpreter(path) + if interp in _SHEBANG_CODE_INTERPRETERS: + return FileType.CODE return None diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py new file mode 100644 index 0000000..4d8abe2 --- /dev/null +++ b/graphify/diagnostics.py @@ -0,0 +1,390 @@ +"""Read-only diagnostics for MultiDiGraph readiness.""" + +from __future__ import annotations + +import json +import re +from collections import Counter, defaultdict +from copy import deepcopy +from pathlib import Path +from typing import Any + +import networkx as nx + + +_SUPPRESSION_DECL_RE = re.compile(r"^\s*(?Pseen_[A-Za-z0-9_]+)\s*[:=]") +_TYPE_TUPLE_RE = re.compile(r"set\[tuple\[(?P[^\]]+)\]\]") + + +def _safe_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, (str, int, float, bool)): + return str(value) + return json.dumps(value, sort_keys=True, default=str, ensure_ascii=False) + + +def _edge_list(extraction: dict[str, Any]) -> list[Any]: + edges = extraction.get("edges") + if edges is None: + edges = extraction.get("links") + return edges if isinstance(edges, list) else [] + + +def _node_ids(extraction: dict[str, Any]) -> set[str]: + nodes = extraction.get("nodes", []) + if not isinstance(nodes, list): + return set() + return { + str(node["id"]) + for node in nodes + if isinstance(node, dict) and "id" in node and node.get("id") is not None + } + + +def _canonical_edge(edge: Any) -> dict[str, str]: + if not isinstance(edge, dict): + return { + "source": "", + "target": "", + "relation": "", + "confidence": "", + "source_file": "", + "source_location": "", + "context": "", + "_invalid": "non_object_edge", + } + source = edge.get("source", edge.get("from")) + target = edge.get("target", edge.get("to")) + return { + "source": _safe_text(source), + "target": _safe_text(target), + "relation": _safe_text(edge.get("relation")), + "confidence": _safe_text(edge.get("confidence")), + "source_file": _safe_text(edge.get("source_file")), + "source_location": _safe_text(edge.get("source_location")), + "context": _safe_text(edge.get("context")), + "_invalid": "", + } + + +def _exact_signature(edge: Any) -> str: + if not isinstance(edge, dict): + return "" + normalized = dict(edge) + if "source" not in normalized and "from" in normalized: + normalized["source"] = normalized["from"] + if "target" not in normalized and "to" in normalized: + normalized["target"] = normalized["to"] + normalized.pop("from", None) + normalized.pop("to", None) + return json.dumps( + normalized, + sort_keys=True, + default=str, + ensure_ascii=False, + separators=(",", ":"), + ) + + +def _count_extra(counter: Counter[Any]) -> int: + return sum(count - 1 for count in counter.values() if count > 1) + + +def _variant_group_count( + grouped_edges: dict[tuple[str, str], list[dict[str, str]]], + field: str, + *, + relation_sensitive: bool = False, +) -> int: + groups = 0 + for edges in grouped_edges.values(): + if relation_sensitive: + by_relation: dict[str, set[str]] = defaultdict(set) + for edge in edges: + by_relation[edge["relation"]].add(edge[field]) + groups += sum(1 for values in by_relation.values() if len(values) > 1) + elif len({edge[field] for edge in edges}) > 1: + groups += 1 + return groups + + +def _tuple_arity_from_annotation(line: str) -> int: + match = _TYPE_TUPLE_RE.search(line) + if not match: + return 0 + inside = match.group("inside").strip() + if not inside: + return 0 + return inside.count(",") + 1 + + +def scan_producer_suppression_sites(path: str | Path) -> dict[str, Any]: + """Find likely `seen_*` producer-suppression sets in an extractor file.""" + source_path = Path(path) + if not source_path.exists(): + return { + "path": str(source_path), + "total_sites": 0, + "sites": [], + "error": "file not found", + } + + sites: list[dict[str, Any]] = [] + lines = source_path.read_text(encoding="utf-8").splitlines() + for lineno, line in enumerate(lines, start=1): + match = _SUPPRESSION_DECL_RE.match(line) + if not match: + continue + sites.append( + { + "line": lineno, + "name": match.group("name"), + "tuple_arity": _tuple_arity_from_annotation(line), + "sample": line.strip()[:120], + } + ) + + return { + "path": str(source_path), + "total_sites": len(sites), + "sites": sites, + "error": "", + } + + +def diagnose_extraction( + extraction: dict[str, Any], + *, + directed: bool = True, + root: str | Path | None = None, + max_examples: int = 5, + extract_path: str | Path | None = None, +) -> dict[str, Any]: + """Summarize same-endpoint edge-collapse risk for one JSON graph/extraction dict.""" + from graphify.build import build_from_json + + node_ids = _node_ids(extraction) + raw_edges = _edge_list(extraction) + canonical_edges = [_canonical_edge(edge) for edge in raw_edges] + + exact_counts: Counter[str] = Counter(_exact_signature(edge) for edge in raw_edges) + directed_pairs: Counter[tuple[str, str]] = Counter() + undirected_pairs: Counter[tuple[str, str]] = Counter() + grouped: dict[tuple[str, str], list[dict[str, str]]] = defaultdict(list) + + non_object_edges = 0 + missing_endpoint_edges = 0 + dangling_endpoint_edges = 0 + self_loop_edges = 0 + valid_candidate_edges = 0 + + for edge in canonical_edges: + if edge["_invalid"]: + non_object_edges += 1 + continue + source = edge["source"] + target = edge["target"] + if not source or not target: + missing_endpoint_edges += 1 + continue + if source not in node_ids or target not in node_ids: + dangling_endpoint_edges += 1 + continue + if source == target: + self_loop_edges += 1 + valid_candidate_edges += 1 + directed_pair = (source, target) + undirected_pair = (source, target) if source <= target else (target, source) + directed_pairs[directed_pair] += 1 + undirected_pairs[undirected_pair] += 1 + grouped[directed_pair].append(edge) + + examples: list[dict[str, Any]] = [] + if max_examples > 0: + for (source, target), count in directed_pairs.most_common(): + if count < 2: + continue + edges = grouped[(source, target)] + examples.append( + { + "source": source, + "target": target, + "edge_count": count, + "relations": sorted({edge["relation"] for edge in edges}), + "source_files": sorted({edge["source_file"] for edge in edges}), + "source_locations": sorted({edge["source_location"] for edge in edges}), + "contexts": sorted({edge["context"] for edge in edges}), + } + ) + if len(examples) >= max_examples: + break + + build_error = "" + graph_type = "" + post_build_edge_count: int | None = None + post_build_node_count: int | None = None + try: + graph_input = deepcopy(extraction) + graph: nx.Graph = build_from_json(graph_input, directed=directed, root=root) + graph_type = type(graph).__name__ + post_build_edge_count = graph.number_of_edges() + post_build_node_count = graph.number_of_nodes() + except Exception as exc: + build_error = f"{type(exc).__name__}: {exc}" + + suppression_path = ( + Path(extract_path) if extract_path else Path(__file__).with_name("extract.py") + ) + + return { + "node_count": len(node_ids), + "raw_edge_count": len(raw_edges), + "non_object_edges": non_object_edges, + "missing_endpoint_edges": missing_endpoint_edges, + "dangling_endpoint_edges": dangling_endpoint_edges, + "self_loop_edges": self_loop_edges, + "valid_candidate_edges": valid_candidate_edges, + "exact_duplicate_edges": _count_extra(exact_counts), + "directed_unique_endpoint_pairs": len(directed_pairs), + "directed_same_endpoint_collapsed_edges": _count_extra(directed_pairs), + "undirected_unique_endpoint_pairs": len(undirected_pairs), + "undirected_same_endpoint_collapsed_edges": _count_extra(undirected_pairs), + "same_endpoint_group_count": sum(1 for count in directed_pairs.values() if count > 1), + "relation_variant_groups": _variant_group_count(grouped, "relation"), + "source_file_variant_groups": _variant_group_count( + grouped, "source_file", relation_sensitive=True + ), + "source_location_variant_groups": _variant_group_count( + grouped, "source_location", relation_sensitive=True + ), + "context_variant_groups": _variant_group_count(grouped, "context", relation_sensitive=True), + "post_build_graph_type": graph_type, + "post_build_node_count": post_build_node_count, + "post_build_edge_count": post_build_edge_count, + "post_build_error": build_error, + "producer_suppression": scan_producer_suppression_sites(suppression_path), + "examples": examples, + } + + +def _read_json_file(path: str | Path) -> dict[str, Any]: + """Read a JSON graph after applying Graphify's graph-load size cap.""" + from graphify.security import check_graph_file_size_cap + + json_path = Path(path) + check_graph_file_size_cap(json_path) + data = json.loads(json_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("diagnostic input must be a JSON object") + return data + + +def diagnose_file( + path: str | Path, + *, + directed: bool | None = None, + root: str | Path | None = None, + max_examples: int = 5, + extract_path: str | Path | None = None, +) -> dict[str, Any]: + """Diagnose a graph/extraction JSON file without mutating it. + + When `directed` is None, the JSON's "directed" flag is honored. Raw + extraction JSON that has no "directed" flag defaults to directed analysis. + """ + data = _read_json_file(path) + if directed is None: + raw_directed = data.get("directed") + effective_directed = raw_directed if isinstance(raw_directed, bool) else True + else: + effective_directed = directed + + summary = diagnose_extraction( + data, + directed=effective_directed, + root=root, + max_examples=max_examples, + extract_path=extract_path, + ) + summary["input_path"] = str(path) + summary["effective_directed"] = effective_directed + return summary + + +def format_diagnostic_json(summary: dict[str, Any]) -> dict[str, Any]: + return { + "schema_version": 1, + "summary": { + key: value + for key, value in summary.items() + if key not in {"examples", "producer_suppression"} + }, + "examples": summary.get("examples", []), + "producer_suppression": summary.get("producer_suppression", {}), + "notes": [ + "Diagnostics are read-only.", + "A normal graph.json is already post-build and cannot recover raw producer edges.", + "Producer suppression sites are heuristic source-code evidence.", + ], + } + + +def format_diagnostic_report(summary: dict[str, Any]) -> str: + suppression = summary.get("producer_suppression", {}) + lines = [ + "[graphify] MultiDiGraph edge-collapse diagnostic", + f"input: {summary.get('input_path', '')}", + "input_stage: provided JSON (normal graph.json is post-build)", + f"effective_directed: {summary.get('effective_directed', '')}", + f"nodes: {summary['node_count']}", + f"raw_edges: {summary['raw_edge_count']}", + f"valid_candidate_edges: {summary['valid_candidate_edges']}", + f"missing_endpoint_edges: {summary['missing_endpoint_edges']}", + f"dangling_endpoint_edges: {summary['dangling_endpoint_edges']}", + f"self_loop_edges: {summary['self_loop_edges']}", + f"exact_duplicate_edges: {summary['exact_duplicate_edges']}", + f"directed_unique_endpoint_pairs: {summary['directed_unique_endpoint_pairs']}", + ( + "directed_same_endpoint_collapsed_edges: " + f"{summary['directed_same_endpoint_collapsed_edges']}" + ), + f"undirected_unique_endpoint_pairs: {summary['undirected_unique_endpoint_pairs']}", + ( + "undirected_same_endpoint_collapsed_edges: " + f"{summary['undirected_same_endpoint_collapsed_edges']}" + ), + f"same_endpoint_group_count: {summary['same_endpoint_group_count']}", + f"relation_variant_groups: {summary['relation_variant_groups']}", + f"source_file_variant_groups: {summary['source_file_variant_groups']}", + f"source_location_variant_groups: {summary['source_location_variant_groups']}", + f"context_variant_groups: {summary['context_variant_groups']}", + f"post_build_graph_type: {summary['post_build_graph_type']}", + f"post_build_edges: {summary['post_build_edge_count']}", + f"producer_suppression_sites: {suppression.get('total_sites', 0)}", + ] + if summary.get("post_build_error"): + lines.append(f"post_build_error: {summary['post_build_error']}") + if suppression.get("error"): + lines.append(f"producer_suppression_error: {suppression['error']}") + if suppression.get("sites"): + lines.append("producer_suppression_examples:") + for site in suppression["sites"][:8]: + lines.append( + f" - L{site['line']} {site['name']} arity={site['tuple_arity'] or 'unknown'}" + ) + if summary.get("examples"): + lines.append("examples:") + for example in summary["examples"]: + lines.append( + " - " + f"{example['source']} -> {example['target']} " + f"edges={example['edge_count']} " + f"relations={example['relations']} " + f"locations={example['source_locations']} " + f"contexts={example['contexts']}" + ) + lines.append( + "note: normal graph.json is post-build; raw producer loss must be measured earlier." + ) + return "\n".join(lines) diff --git a/graphify/export.py b/graphify/export.py index a71c927..4c9a447 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -477,6 +477,8 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, existing_path = Path(output_path) if not force and existing_path.exists(): try: + from graphify.security import check_graph_file_size_cap + check_graph_file_size_cap(existing_path) existing_data = json.loads(existing_path.read_text(encoding="utf-8")) existing_n = len(existing_data.get("nodes", [])) new_n = G.number_of_nodes() @@ -744,7 +746,9 @@ def to_html( graphify - {title} - + {_html_styles()} diff --git a/graphify/extract.py b/graphify/extract.py index 9f79f95..298de16 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -5948,11 +5948,14 @@ def extract_bash(path: Path) -> dict: function_bodies: list[tuple[str, Any]] = [] defined_functions: set[str] = set() - def add_node(nid: str, label: str, line: int) -> None: + from graphify.security import sanitize_metadata # module-level cached import + + def add_node(nid: str, label: str, line: int, kind: str = "code") -> None: if nid and nid not in seen_ids: seen_ids.add(nid) nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}"}) + "source_file": str_path, "source_location": f"L{line}", + "metadata": sanitize_metadata({"language": "bash", "kind": kind})}) # noqa: E501 def add_edge(src: str, tgt: str, relation: str, line: int, confidence: str = "EXTRACTED", weight: float = 1.0, @@ -5967,35 +5970,73 @@ def extract_bash(path: Path) -> dict: edges.append(edge) file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) + # file_nid is fully path-derived and never produced by _make_id(stem, func_name), + # so appending "__entry" guarantees a distinct ID from any function node. + entry_nid = file_nid + "__entry" + add_node(file_nid, path.name, 1, kind="file") + add_node(entry_nid, f"{path.name} script", 1, kind="bash_entrypoint") + add_edge(file_nid, entry_nid, "contains", 1) - _BASH_SKIP = frozenset({ - "if", "then", "else", "elif", "fi", "for", "while", "until", "do", - "done", "case", "esac", "in", "return", "exit", "break", "continue", - "echo", "printf", "cd", "set", "local", "export", "readonly", - "declare", "unset", "shift", "read", "test", "[", "[[", ":", "true", - "false", "source", ".", "trap", "wait", "exec", "eval", + _BASH_SOURCE_COMMANDS = frozenset({"source", "."}) + # Parent node types that mean a contained command is part of a substitution + # or expansion, not a real function call. Token-level filtering misses + # these because `$(build)` exposes `build` as a child command whose name + # token has no metacharacters — only the parent does. + _BASH_EXPANSION_PARENTS = frozenset({ + "command_substitution", + "process_substitution", }) + def text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + def is_inside_expansion(node) -> bool: + parent = node.parent + while parent is not None: + if parent.type in _BASH_EXPANSION_PARENTS: + return True + parent = parent.parent + return False + + def literal(node) -> str | None: + # Token-level filter: rejects names containing shell metacharacters. + # Combined with `is_inside_expansion` for parent-context rejection. + raw = text(node).strip() + if not raw: + return None + if raw[0:1] in {"'", '"'} and raw[-1:] == raw[0]: + raw = raw[1:-1] + if any(token in raw for token in ("$", "`", "$(", "<(", ">", "|", ";", "&")): + return None + return raw + def _bash_func_name(node) -> str | None: """Get the name from a function_definition node.""" # bash grammar: function_definition has a word child (the name) for child in node.children: if child.type == "word": - return _read_text(child, source) + return literal(child) return None def walk_calls(body_node, func_nid: str, seen_calls: set) -> None: if body_node is None: return for child in body_node.children: - if child.type == "command": + if child.type == "function_definition": + # Skip nested function definitions — their bodies are walked + # separately, so we don't attribute their calls to the + # enclosing scope. + continue + if child.type == "command" and not is_inside_expansion(child): cmd_name_node = child.child_by_field_name("name") if cmd_name_node is None and child.children: cmd_name_node = child.children[0] if cmd_name_node: - name = _read_text(cmd_name_node, source).strip() - if name and name not in _BASH_SKIP and name in defined_functions: + name = literal(cmd_name_node) + # Defined-functions wins. Skip-lists for external commands + # would create false negatives when a user defines a + # function shadowing an external (`install`, `find`, etc.). + if name and name in defined_functions: tgt = _make_id(stem, name) key = (func_nid, tgt) if tgt and key not in seen_calls: @@ -6012,7 +6053,7 @@ def extract_bash(path: Path) -> dict: if name: fn_nid = _make_id(stem, name) line = node.start_point[0] + 1 - add_node(fn_nid, f"{name}()", line) + add_node(fn_nid, f"{name}()", line, kind="bash_function") add_edge(parent_nid, fn_nid, "defines", line) defined_functions.add(name) # find the compound_statement body @@ -6022,15 +6063,21 @@ def extract_bash(path: Path) -> dict: body = child break function_bodies.append((fn_nid, body)) - return # don't recurse into function body during structural pass + # Recurse into the body so nested function definitions are discovered + # and added to function_bodies for the second-pass walk_calls. + if body is not None: + walk(body, fn_nid) + return if t == "command": + if is_inside_expansion(node): + return cmd_name_node = node.child_by_field_name("name") if cmd_name_node is None and node.children: cmd_name_node = node.children[0] if cmd_name_node: - cmd = _read_text(cmd_name_node, source).strip() - if cmd in ("source", "."): + cmd = literal(cmd_name_node) + if cmd in _BASH_SOURCE_COMMANDS and cmd not in defined_functions: # find the path argument (first word after command name) args = [c for c in node.children if c.type in ("word", "string", "concatenation") @@ -6073,9 +6120,26 @@ def extract_bash(path: Path) -> dict: for child in node.children: walk(child, parent_nid) + # Pre-pass: collect all defined function names so the source-command handler + # in walk() can detect user-defined functions that shadow 'source' / '.' + # regardless of definition order in the file. + def _prescan_functions(node) -> None: + if node.type == "function_definition": + name = _bash_func_name(node) + if name: + defined_functions.add(name) + for child in node.children: + _prescan_functions(child) + else: + for child in node.children: + _prescan_functions(child) + + _prescan_functions(root) walk(root, file_nid) # Second pass: cross-function calls + top_seen: set = set() + walk_calls(root, entry_nid, top_seen) # top-level calls attributed to the entrypoint for fn_nid, body in function_bodies: walk_calls(body, fn_nid, set()) diff --git a/graphify/global_graph.py b/graphify/global_graph.py index dfcc826..c6310f9 100644 --- a/graphify/global_graph.py +++ b/graphify/global_graph.py @@ -28,6 +28,8 @@ def _save_manifest(manifest: dict) -> None: def _load_global_graph() -> nx.Graph: if _GLOBAL_GRAPH.exists(): + from graphify.security import check_graph_file_size_cap + check_graph_file_size_cap(_GLOBAL_GRAPH) data = json.loads(_GLOBAL_GRAPH.read_text(encoding="utf-8")) if "links" not in data and "edges" in data: data = dict(data, links=data["edges"]) @@ -80,6 +82,8 @@ def global_add(source_path: Path, repo_tag: str) -> dict: return {"repo_tag": repo_tag, "nodes_added": 0, "nodes_removed": 0, "skipped": True} # Load source graph + from graphify.security import check_graph_file_size_cap + check_graph_file_size_cap(source_path) data = json.loads(source_path.read_text(encoding="utf-8")) if "links" not in data and "edges" in data: data = dict(data, links=data["edges"]) diff --git a/graphify/multigraph_compat.py b/graphify/multigraph_compat.py new file mode 100644 index 0000000..7ac62e2 --- /dev/null +++ b/graphify/multigraph_compat.py @@ -0,0 +1,212 @@ +"""Runtime compatibility probe for Graphify MultiDiGraph mode. + +Verifies that the current NetworkX runtime supports the behaviors a future +opt-in --multigraph build will rely on. The probe is BEHAVIOR-based, not +version-based — both NX 3.4.2 (Py 3.10 lane) and NX 3.6.1+ (Py 3.11+ lane) +pass. The probe result is cached for the process lifetime via lru_cache. + +No call sites added yet; downstream multigraph PRs will gate on +require_multigraph_capabilities() before enabling MDG mode. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from functools import lru_cache +import sys +from typing import Any + +import networkx as nx +from networkx.readwrite import json_graph + + +@dataclass(frozen=True) +class CapabilityCheck: + name: str + ok: bool + detail: str + + +@dataclass(frozen=True) +class MultigraphCapabilityResult: + python_version: str + networkx_version: str + checks: tuple[CapabilityCheck, ...] + + @property + def ok(self) -> bool: + return all(check.ok for check in self.checks) + + @property + def failed(self) -> tuple[CapabilityCheck, ...]: + return tuple(check for check in self.checks if not check.ok) + + def error_message(self) -> str: + if self.ok: + return ( + "Graphify MultiDiGraph capability probe passed " + f"(Python {self.python_version}, NetworkX {self.networkx_version})." + ) + failed = "; ".join(f"{check.name}: {check.detail}" for check in self.failed) + return ( + "error: --multigraph requires NetworkX keyed MultiDiGraph node-link " + "round-trip support. " + f"Detected Python {self.python_version}, NetworkX {self.networkx_version}. " + f"Failed capability check(s): {failed}. " + "Default simple graph mode remains available." + ) + + +def _check(name: str, func: Callable[[], bool | str]) -> CapabilityCheck: + try: + detail = func() + except Exception as exc: + return CapabilityCheck(name, False, f"{type(exc).__name__}: {exc}") + if detail is True: + return CapabilityCheck(name, True, "ok") + if isinstance(detail, str): + return CapabilityCheck(name, False, detail) + return CapabilityCheck(name, False, f"unexpected result {detail!r}") + + +def _build_probe_graph() -> nx.MultiDiGraph: + graph = nx.MultiDiGraph() + graph.add_node("a", label="A") + graph.add_node("b", label="B") + graph.add_edge("a", "b", key="calls:a.py:L1", relation="calls", source_file="a.py") + graph.add_edge("a", "b", key="imports:a.py:L2", relation="imports", source_file="a.py") + return graph + + +def _probe_keyed_parallel_edges() -> bool | str: + graph = _build_probe_graph() + if not graph.is_multigraph() or not graph.is_directed(): + return f"probe graph type was {type(graph).__name__}" + if graph.number_of_edges("a", "b") != 2: + return f"expected 2 keyed parallel edges, got {graph.number_of_edges('a', 'b')}" + keys = set(graph["a"]["b"].keys()) + expected = {"calls:a.py:L1", "imports:a.py:L2"} + if keys != expected: + return f"expected keys {sorted(expected)}, got {sorted(keys)}" + return True + + +def _probe_node_link_round_trip() -> bool | str: + graph = _build_probe_graph() + data = json_graph.node_link_data(graph, edges="links") + if data.get("multigraph") is not True: + return f"serialized multigraph flag was {data.get('multigraph')!r}" + if data.get("directed") is not True: + return f"serialized directed flag was {data.get('directed')!r}" + links = data.get("links") + if not isinstance(links, list) or len(links) != 2: + length = 0 if not isinstance(links, list) else len(links) + return f"serialized links length was {length}" + serialized_keys: set[str] = set() + for edge in links: + if isinstance(edge, dict): + edge_key = edge.get("key") + if isinstance(edge_key, str): + serialized_keys.add(edge_key) + expected = {"calls:a.py:L1", "imports:a.py:L2"} + if serialized_keys != expected: + return f"serialized keys {sorted(serialized_keys)} did not match {sorted(expected)}" + loaded = json_graph.node_link_graph(data, edges="links") + if not isinstance(loaded, nx.MultiDiGraph): + return f"round-trip graph type was {type(loaded).__name__}" + if loaded.number_of_edges("a", "b") != 2: + return f"round-trip edge count was {loaded.number_of_edges('a', 'b')}" + loaded_keys = set(loaded["a"]["b"].keys()) + if loaded_keys != expected: + return f"round-trip keys {sorted(loaded_keys)} did not match {sorted(expected)}" + return True + + +def _probe_duplicate_key_overwrite_semantics() -> bool | str: + graph = nx.MultiDiGraph() + graph.add_edge("x", "y", key="same", marker="first") + graph.add_edge("x", "y", key="same", marker="second") + edges = list(graph.edges(keys=True, data=True)) + if len(edges) != 1: + return f"expected one edge after duplicate-key add, got {len(edges)}" + if edges[0][3].get("marker") != "second": + return f"expected second attr overwrite, got {edges[0][3].get('marker')!r}" + return True + + +def _probe_reserved_key_attr_rejected() -> bool | str: + """Verify the Python language guarantee that NetworkX add_edge inherits. + + Python forbids passing the same keyword argument twice — once explicitly + and once via **kwargs. This probe confirms that protection still applies + to nx.MultiDiGraph.add_edge: a future loader that builds attrs from JSON + will be reliably protected from accidentally setting `key` via attrs while + also passing `key=` explicitly. + + The probe always passes on any Python 3.x version. Its purpose is to + document the invariant explicitly in the probe suite so that if a future + Python version relaxes this rule (extremely unlikely), the probe surfaces + the regression. + """ + graph = nx.MultiDiGraph() + attrs: dict[str, Any] = {"key": "attr-key", "relation": "calls"} + try: + graph.add_edge("a", "b", key="schema-key", **attrs) + except TypeError: + return True + return "add_edge accepted duplicate key keyword and attr; loader must not rely on this" + + +def _probe_remove_edges_from_two_tuple_semantics() -> bool | str: + graph = nx.MultiDiGraph() + graph.add_edge("a", "b", key="one") + graph.add_edge("a", "b", key="two") + graph.remove_edges_from([("a", "b")]) + remaining = graph.number_of_edges("a", "b") + if remaining != 1: + return f"expected one remaining edge after two-tuple removal, got {remaining}" + return True + + +def _probe_to_undirected_preserves_multigraph_type() -> bool | str: + graph = _build_probe_graph() + undirected = graph.to_undirected() + undirected_view = graph.to_undirected(as_view=True) + if not isinstance(undirected, nx.MultiGraph): + return f"to_undirected() returned {type(undirected).__name__}" + if not isinstance(undirected_view, nx.MultiGraph): + return f"to_undirected(as_view=True) returned {type(undirected_view).__name__}" + return True + + +@lru_cache(maxsize=1) +def probe_multigraph_capabilities() -> MultigraphCapabilityResult: + checks = ( + _check("keyed_parallel_edges", _probe_keyed_parallel_edges), + _check("node_link_edges_links_round_trip", _probe_node_link_round_trip), + _check("duplicate_key_overwrite_semantics", _probe_duplicate_key_overwrite_semantics), + _check("reserved_key_attr_rejected", _probe_reserved_key_attr_rejected), + _check( + "remove_edges_from_two_tuple_semantics", + _probe_remove_edges_from_two_tuple_semantics, + ), + _check( + "to_undirected_preserves_multigraph_type", + _probe_to_undirected_preserves_multigraph_type, + ), + ) + return MultigraphCapabilityResult( + python_version=( + f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + ), + networkx_version=nx.__version__, + checks=checks, + ) + + +def require_multigraph_capabilities() -> MultigraphCapabilityResult: + result = probe_multigraph_capabilities() + if not result.ok: + raise RuntimeError(result.error_message()) + return result diff --git a/graphify/prs.py b/graphify/prs.py index cd0bc0e..7ddb43b 100644 --- a/graphify/prs.py +++ b/graphify/prs.py @@ -318,9 +318,11 @@ def fetch_worktrees() -> dict[str, str]: def _load_graph_json(graph_path: Path) -> dict | None: if not graph_path.exists(): return None + from graphify.security import check_graph_file_size_cap try: + check_graph_file_size_cap(graph_path) return json.loads(graph_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): + except (json.JSONDecodeError, OSError, ValueError): return None diff --git a/graphify/scip_ingest.py b/graphify/scip_ingest.py new file mode 100644 index 0000000..bf3d185 --- /dev/null +++ b/graphify/scip_ingest.py @@ -0,0 +1,363 @@ +"""scip_ingest.py — SCIP JSON ingestion (simplified subset). + +Reads a simplified SCIP-style JSON structure and converts it into +Graphify nodes and edges. NOT a full SCIP protobuf implementation — +this is a skeleton that consumes the simplified shape described below. + +Not wired to the CLI in this phase. + +Entry point: + ingest_scip_json(doc: object, source_file: str = "", + language: str = "python") -> dict[str, Any] + + Returns {"nodes": [...], "edges": [...]} compatible with Graphify's + extraction result format. All edges emitted are endpoint-safe — the + function builds a symbol → node_id index in a first pass and either + resolves relationship targets via that index or creates a stub + external node so `build_from_json()` will keep the edge. + +Supported (simplified) JSON shape: + documents[]: { relative_path, language, symbols[] } + symbols[]: { symbol, kind, display_name, documentation[], + relationships[], occurrences[] } + relationships[]: { symbol, is_reference, is_implementation, + is_type_definition, is_definition } + occurrences[]: { range[], symbol, symbol_roles } + +This shape diverges from the official SCIP protobuf (where occurrences +live on the document, not on each symbol). We consume the simplified +shape that LLM-generated SCIP-style JSON commonly produces. Future +cycles may add document-level occurrence support. +""" + +from __future__ import annotations + +import hashlib +import re +from typing import Any + +from graphify.security import sanitize_metadata + + +def ingest_scip_json( + doc: object, + source_file: str = "", + language: str = "python", +) -> dict[str, Any]: + """Convert a SCIP-style JSON document into Graphify nodes and edges. + + Parameter ``doc`` is ``object`` (not ``dict[str, Any]``) because SCIP + documents come from external tools — we may be handed arbitrary + deserialized JSON. The first check rejects anything that isn't a dict + and returns the empty result. + + Two-pass design: + 1. Build a ``symbol_str → node_id`` index across every valid symbol + in every valid document, plus collect per-symbol metadata. + 2. Emit nodes for every indexed symbol and then emit relationship + edges. Relationship targets are resolved via the index when + present; otherwise a stub ``scip_external`` node is added so + edges never dangle. + """ + nodes: list[dict[str, Any]] = [] + edges: list[dict[str, Any]] = [] + seen_node_ids: set[str] = set() + seen_edges: set[tuple[str, str, str, str | None]] = set() + + if not isinstance(doc, dict): + return {"nodes": nodes, "edges": edges} + + documents = doc.get("documents", []) + if not isinstance(documents, list): + return {"nodes": nodes, "edges": edges} + + # ---- pass 1: build symbol → node_id indices ----------------------------- + # Two indices so relationship resolution can be document-aware: + # per_doc: (symbol_id, doc_path) → node_id (same-document precedence) + # global: symbol_id → list[node_id] (cross-document fallback, + # used only when unambiguous) + per_doc_index: dict[tuple[str, str], str] = {} + global_index: dict[str, list[str]] = {} + # Per-symbol metadata kept for pass-2 node emission (avoids re-walking + # the document tree). + symbol_records: list[dict[str, Any]] = [] + for document in documents: + if not isinstance(document, dict): + continue + doc_path = _coerce_str(document.get("relative_path"), source_file) + doc_language = _coerce_str(document.get("language"), language) + symbols = document.get("symbols", []) + if not isinstance(symbols, list): + continue + for symbol in symbols: + if not isinstance(symbol, dict): + continue + symbol_id = _coerce_str(symbol.get("symbol"), "") + if not symbol_id: + continue + node_id = _make_scip_node_id(symbol_id, doc_path) + per_doc_index.setdefault((symbol_id, doc_path), node_id) + # Dedupe node_ids in the global index — duplicate symbol records + # within the SAME document produce identical node_ids, and we + # don't want them to look like cross-document ambiguity. + candidates = global_index.setdefault(symbol_id, []) + if node_id not in candidates: + candidates.append(node_id) + symbol_records.append( + { + "node_id": node_id, + "symbol_id": symbol_id, + "doc_path": doc_path, + "language": doc_language, + "raw": symbol, + } + ) + + # ---- pass 2: emit nodes + relationship edges ----------------------------- + for record in symbol_records: + _emit_symbol_node(record, nodes, seen_node_ids) + _emit_relationships( + record, + per_doc_index, + global_index, + nodes, + edges, + seen_node_ids, + seen_edges, + ) + + return {"nodes": nodes, "edges": edges} + + +def _emit_symbol_node( + record: dict[str, Any], + nodes: list[dict[str, Any]], + seen_node_ids: set[str], +) -> None: + """Append the canonical node for a SCIP symbol record.""" + node_id = record["node_id"] + if node_id in seen_node_ids: + return + raw = record["raw"] + symbol_id = record["symbol_id"] + doc_path = record["doc_path"] + kind = _coerce_str(raw.get("kind"), "unknown") + display_name = _coerce_str(raw.get("display_name"), "") + documentation = raw.get("documentation", []) + description = "" + if isinstance(documentation, list) and documentation: + first = documentation[0] + if isinstance(first, str): + description = first + occurrences = raw.get("occurrences", []) + sourceline = _first_occurrence_line(occurrences) + suffix = symbol_id.split("#")[-1] if "#" in symbol_id else symbol_id + label = display_name or suffix or symbol_id + seen_node_ids.add(node_id) # label uses display_name or suffix (never empty for valid symbols) + nodes.append( + { + "id": node_id, + "label": label, + "file_type": _scip_kind_to_file_type(kind), + "source_file": doc_path, + "source_location": f"L{sourceline}" if sourceline else "", + "metadata": sanitize_metadata(_build_scip_metadata(symbol_id, kind, description)), + } + ) + + +def _emit_relationships( + record: dict[str, Any], + per_doc_index: dict[tuple[str, str], str], + global_index: dict[str, list[str]], + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]], + seen_node_ids: set[str], + seen_edges: set[tuple[str, str, str, str | None]], +) -> None: + """Append edges (and stub nodes when needed) for a symbol's relationships. + + Relationship target resolution order: + 1. Same-document `(target_symbol, doc_path)` — duplicate local symbol + names across files route to THIS file's symbol, not another's. + 2. Unique cross-document match — when the symbol exists in exactly + one document and that document is different from the source. + 3. Stub external node — for symbols not declared in any document + OR ambiguous duplicates across multiple documents (refusing to + guess silently). + """ + raw = record["raw"] + source_node_id = record["node_id"] + doc_path = record["doc_path"] + occurrences = raw.get("occurrences", []) + sourceline = _first_occurrence_line(occurrences) + relationships = raw.get("relationships") + if not isinstance(relationships, list): + return + for rel in relationships: + if not isinstance(rel, dict): + continue + target_symbol = _coerce_str(rel.get("symbol"), "") + if not target_symbol: + continue + target_node_id = _resolve_relationship_target( + target_symbol, + doc_path, + per_doc_index, + global_index, + ) + if target_node_id is None: + # External relationship target: emit a stub node so the edge + # is never dangling. The stub uses the source document's path + # as its host context. + target_node_id = _make_scip_node_id(target_symbol, doc_path) + if target_node_id not in seen_node_ids: + seen_node_ids.add(target_node_id) + suffix = target_symbol.split("#")[-1] if "#" in target_symbol else target_symbol + nodes.append( + { + "id": target_node_id, + "label": suffix or target_symbol, + "file_type": "code", + "source_file": doc_path, + "source_location": "", + "metadata": sanitize_metadata( + _build_scip_metadata(target_symbol, "external", "") + ), + } + ) + relation = _scip_relation_for(rel) + source_location = f"L{sourceline}" if sourceline else "" + key = (source_node_id, target_node_id, relation, source_location) + if key in seen_edges: + continue + seen_edges.add(key) + edges.append( + { + "source": source_node_id, + "target": target_node_id, + "relation": relation, + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": doc_path, + "source_location": source_location, + "weight": 1.0, + "context": "scip", + "metadata": sanitize_metadata({"scip_relationship": rel}), + } + ) + + +def _resolve_relationship_target( + target_symbol: str, + source_doc_path: str, + per_doc_index: dict[tuple[str, str], str], + global_index: dict[str, list[str]], +) -> str | None: + """Resolve a SCIP relationship target to an emitted node id, or None. + + Resolution order: + 1. Same-document match — `(target_symbol, source_doc_path)`. + 2. Unique cross-document match — exactly one node id in the global + index for this symbol AND it isn't the same document we already + tried. + 3. None — symbol is either absent globally OR ambiguous (defined in + multiple documents). The caller emits a stub external node. + """ + same_doc = per_doc_index.get((target_symbol, source_doc_path)) + if same_doc is not None: + return same_doc + candidates = global_index.get(target_symbol, []) + if len(candidates) == 1: + return candidates[0] + return None + + +def _is_true(value: object) -> bool: + """Return True only when value is exactly the boolean True. + + Used for SCIP relationship flags. Truthy strings like ``"false"`` are + common in untrusted external JSON and must NOT count as a set flag. + """ + return value is True + + +def _scip_relation_for(rel: dict[str, Any]) -> str: + """Pick the Graphify relation tag for a SCIP relationship dict. + + Flags are accepted only when the value is exactly ``True`` — protects + against truthy-but-misleading values like ``"false"`` in external JSON. + """ + if _is_true(rel.get("is_implementation")): + return "scip_impl" + if _is_true(rel.get("is_type_definition")): + return "scip_typed" + if _is_true(rel.get("is_definition")): + return "scip_def" + return "scip_ref" + + +def _first_occurrence_line(occurrences: object) -> int: + """Read the 1-based line number from the first occurrence range, defensively. + + Note: ``bool`` is a subclass of ``int`` in Python — ``isinstance(True, int)`` + is True. We explicitly exclude booleans so a malformed ``range: [True, …]`` + cannot produce ``source_location = "LTrue"``. + """ + if not isinstance(occurrences, list) or not occurrences: + return 0 + first = occurrences[0] + if not isinstance(first, dict): + return 0 + rng = first.get("range", []) + if not isinstance(rng, list) or len(rng) < 1: + return 0 + line = rng[0] + if isinstance(line, bool) or not isinstance(line, int) or line < 0: + return 0 + return line + + +def _coerce_str(value: object, default: str) -> str: + """Return ``value`` if it is a string, else the ``default`` (also a string).""" + if isinstance(value, str): + return value + if isinstance(default, str): + return default + return "" + + +def _make_scip_node_id(symbol: str, source_file: str) -> str: + """Derive a stable Graphify node ID from a SCIP symbol identifier. + + Uses SHA-1 truncated to 12 hex chars (48 bits). This is an identifier, + not a security boundary — collision risk is acceptable at this scale + given the per-document scoping prefix. + """ + raw = f"{source_file}:{symbol}" + h = hashlib.sha1(raw.encode(), usedforsecurity=False).hexdigest()[:12] + parts = symbol.split("#") + suffix = parts[-1] if parts else symbol + suffix = re.sub(r"[^a-zA-Z0-9_]", "_", suffix).strip("_").lower() + if suffix: + return f"scip_{suffix}_{h}" + return f"scip_{h}" + + +def _scip_kind_to_file_type(kind: str) -> str: + """Map SCIP symbol kind to a Graphify file_type.""" + # All SCIP symbols are code entities (functions, methods, classes, …); + # the `kind` is preserved in metadata for downstream consumers. + _ = kind # acknowledged but not currently used for file_type routing + return "code" + + +def _build_scip_metadata(symbol_id: str, kind: str, description: str) -> dict[str, str]: + """Build metadata for a SCIP node.""" + meta: dict[str, str] = { + "scip_symbol": symbol_id, + "scip_kind": kind, + } + if description: + meta["scip_description"] = description + return meta diff --git a/graphify/security.py b/graphify/security.py index a594af3..91b500f 100644 --- a/graphify/security.py +++ b/graphify/security.py @@ -7,7 +7,9 @@ import re import urllib.error import urllib.parse import urllib.request +from collections.abc import Mapping from pathlib import Path +from typing import Any import ipaddress import socket @@ -16,6 +18,12 @@ _ALLOWED_SCHEMES = {"http", "https"} _MAX_FETCH_BYTES = 52_428_800 # 50 MB hard cap for binary downloads _MAX_TEXT_BYTES = 10_485_760 # 10 MB hard cap for HTML / text +# Graph-load memory-bomb cap: reject .json files larger than this before +# JSON-parsing them into a dict. Without this, a multi-gigabyte (or +# specifically crafted) graph.json can exhaust process memory during +# json.loads + node_link_graph rehydration. +_MAX_GRAPH_FILE_BYTES = 512 * 1024 * 1024 # 512 MiB + # AWS metadata, link-local, and common cloud metadata endpoints _BLOCKED_HOSTS = {"metadata.google.internal", "metadata.google.com"} @@ -228,6 +236,29 @@ def validate_graph_path(path: str | Path, base: Path | None = None) -> Path: return resolved +def check_graph_file_size_cap(path: Path) -> None: + """Reject *path* if its size exceeds ``_MAX_GRAPH_FILE_BYTES``. + + Protects callers from memory bombs by failing fast before a multi-GiB + graph.json is read into memory and JSON-parsed. Silently returns when + ``path.stat()`` cannot be read — the caller's own existence/path check + is expected to surface a clearer error in that case. + + Raises: + ValueError - file size exceeds the cap. The message includes the + observed size and the cap so callers can show a usable error. + """ + try: + size = path.stat().st_size + except OSError: + return + if size > _MAX_GRAPH_FILE_BYTES: + raise ValueError( + f"graph file {path} is {size:_d} bytes, " + f"exceeds {_MAX_GRAPH_FILE_BYTES:_d}-byte cap" + ) + + # --------------------------------------------------------------------------- # Label sanitisation (mirrors code-review-graph's _sanitize_name pattern) # --------------------------------------------------------------------------- @@ -248,3 +279,58 @@ def sanitize_label(text: str | None) -> str: if len(text) > _MAX_LABEL_LEN: text = text[:_MAX_LABEL_LEN] return text + + +# --------------------------------------------------------------------------- +# Metadata sanitisation (recursive, bounded, HTML-safe) +# --------------------------------------------------------------------------- + +_METADATA_MAX_VALUE_LEN = 512 +_METADATA_MAX_LIST_ITEMS = 50 + + +def _sanitize_metadata_string(value: object) -> str: + """Return a control-character-free, HTML-escaped, bounded string.""" + text = _CONTROL_CHAR_RE.sub("", str(value)) + text = html.escape(text, quote=True) + if len(text) > _METADATA_MAX_VALUE_LEN: + text = text[:_METADATA_MAX_VALUE_LEN] + return text # html is imported at module level (line 5) + + +def _sanitize_metadata_value(value: object) -> object: + """Sanitize a metadata value while preserving simple JSON-compatible types.""" + if isinstance(value, bool): + # bool is a subclass of int — must be checked first to avoid coercion. + return value + if isinstance(value, str): + return _sanitize_metadata_string(value) + if isinstance(value, dict): + return sanitize_metadata(value) + if isinstance(value, (list, tuple)): + return [_sanitize_metadata_value(item) for item in value[:_METADATA_MAX_LIST_ITEMS]] + if isinstance(value, (int, float)) or value is None: + return value + return _sanitize_metadata_string(value) + + +def sanitize_metadata(metadata: Mapping[str, Any] | None) -> dict[str, object]: + """Sanitize metadata keys and values before graph export. + + Metadata is less constrained than node labels: it can contain nested + dicts, lists, source snippets, external index symbols, and docstring + text. This helper keeps the data JSON-compatible, strips control + characters, escapes HTML-sensitive characters in strings, caps long + strings/lists, and drops entries whose key becomes empty after + sanitization. + """ + if metadata is None: + return {} + + result: dict[str, object] = {} + for key, value in metadata.items(): + clean_key = _sanitize_metadata_string(key) + if not clean_key: + continue + result[clean_key] = _sanitize_metadata_value(value) + return result diff --git a/graphify/semantic_cleanup.py b/graphify/semantic_cleanup.py new file mode 100644 index 0000000..6bac6b0 --- /dev/null +++ b/graphify/semantic_cleanup.py @@ -0,0 +1,319 @@ +# Semantic fragment sanitizer — converts sentence-like rationale nodes into +# attributes on related nodes and removes invalid file_type values. +# +# Currently called from the skill merge scripts (skill-opencode.md, +# skill-codex.md) so that rationale text never leaks into the knowledge +# graph as standalone nodes. (Future: graphify.llm may wire this into +# _parse_llm_json / _merge_into for non-skill code paths; not done in +# this cycle.) +from __future__ import annotations + +import json +import re +from pathlib import Path + +# Labels longer than this many characters, or containing >= this many words, +# are candidates for being sentence-like rationale text rather than entity names. +_RATIONALE_MIN_CHARS = 80 +_RATIONALE_MIN_WORDS = 8 + +# Validation limits for untrusted semantic-fragment payloads. See +# validate_semantic_fragment(). Issue #825: returned-JSON normalization for +# OpenCode and Codex agents requires a Python enforcement boundary so a +# malicious or runaway agent response cannot exhaust memory or escape the +# graphify-out chunk directory via crafted node/edge IDs. +MAX_SEMANTIC_FRAGMENT_BYTES = 25 * 1024 * 1024 +MAX_SEMANTIC_FRAGMENT_NODES = 10_000 +MAX_SEMANTIC_FRAGMENT_EDGES = 100_000 +MAX_SEMANTIC_FRAGMENT_HYPEREDGES = 10_000 +MAX_SEMANTIC_HYPEREDGE_NODES = 256 +MAX_SEMANTIC_ID_LENGTH = 256 +VALID_SEMANTIC_FILE_TYPES = frozenset({"code", "document", "paper", "image", "rationale", "concept"}) +_SEMANTIC_ID_RE = re.compile(r"^[A-Za-z0-9._:-]+$") + + +def validate_semantic_fragment(fragment: object) -> list[str]: + """Return validation errors for an untrusted semantic extraction fragment. + + Empty list means valid. Called by skill merge code before + sanitize_semantic_fragment() so malformed or malicious agent JSON is + rejected before it touches the graph. Parameter is `object` (not `dict`) + because we may be handed arbitrary deserialized JSON — the first check + rejects anything that isn't a dict. + """ + if not isinstance(fragment, dict): + return ["fragment must be a JSON object"] + + errors: list[str] = [] + try: + payload = json.dumps(fragment, ensure_ascii=False).encode("utf-8") + except (TypeError, ValueError) as exc: + return [f"fragment is not JSON-serializable: {exc}"] + + if len(payload) > MAX_SEMANTIC_FRAGMENT_BYTES: + errors.append(f"payload is {len(payload)} bytes; max is {MAX_SEMANTIC_FRAGMENT_BYTES}") + + nodes = fragment.get("nodes", []) + edges = fragment.get("edges", []) + if not isinstance(nodes, list): + errors.append("nodes must be a list") + nodes = [] + elif len(nodes) > MAX_SEMANTIC_FRAGMENT_NODES: + errors.append(f"nodes has {len(nodes)} entries; max is {MAX_SEMANTIC_FRAGMENT_NODES}") + + if not isinstance(edges, list): + errors.append("edges must be a list") + edges = [] + elif len(edges) > MAX_SEMANTIC_FRAGMENT_EDGES: + errors.append(f"edges has {len(edges)} entries; max is {MAX_SEMANTIC_FRAGMENT_EDGES}") + + for i, node in enumerate(nodes): + if not isinstance(node, dict): + errors.append(f"nodes[{i}] must be an object") + continue + _validate_semantic_id(errors, f"nodes[{i}].id", node.get("id")) + file_type = node.get("file_type") + if file_type is not None and file_type not in VALID_SEMANTIC_FILE_TYPES: + errors.append( + f"nodes[{i}].file_type {file_type!r} is not one of " + f"{sorted(VALID_SEMANTIC_FILE_TYPES)}" + ) # validate file_type before any sanitize path can run + + for i, edge in enumerate(edges): + if not isinstance(edge, dict): + errors.append(f"edges[{i}] must be an object") + continue + _validate_semantic_id(errors, f"edges[{i}].source", edge.get("source")) + _validate_semantic_id(errors, f"edges[{i}].target", edge.get("target")) + + hyperedges = fragment.get("hyperedges", []) + if hyperedges is None: + hyperedges = [] + if not isinstance(hyperedges, list): + errors.append("hyperedges must be a list") + else: + if len(hyperedges) > MAX_SEMANTIC_FRAGMENT_HYPEREDGES: + errors.append( + f"hyperedges has {len(hyperedges)} entries; " + f"max is {MAX_SEMANTIC_FRAGMENT_HYPEREDGES}" + ) + for i, he in enumerate(hyperedges): + if not isinstance(he, dict): + errors.append(f"hyperedges[{i}] must be an object") + continue + _validate_semantic_id(errors, f"hyperedges[{i}].id", he.get("id")) + he_nodes = he.get("nodes") + if not isinstance(he_nodes, list): + errors.append(f"hyperedges[{i}].nodes must be a list") + continue + if len(he_nodes) > MAX_SEMANTIC_HYPEREDGE_NODES: + errors.append( + f"hyperedges[{i}].nodes has {len(he_nodes)} entries; " + f"max is {MAX_SEMANTIC_HYPEREDGE_NODES}" + ) + for j, ref in enumerate(he_nodes): + _validate_semantic_id(errors, f"hyperedges[{i}].nodes[{j}]", ref) + + return errors + + +def load_validated_semantic_fragment(path: Path) -> tuple[dict | None, list[str]]: + """Load and validate a semantic chunk, rejecting oversize files before parsing. + + The size guard runs against `path.stat().st_size` so an attacker-supplied + multi-gigabyte chunk file cannot blow up memory at `read_text()` time. + JSON decode errors are returned as validation errors rather than raised, + so callers can `continue` past bad chunks without a try/except. + """ + try: + size = path.stat().st_size + except OSError as exc: + return None, [f"could not stat {path}: {exc}"] + if size > MAX_SEMANTIC_FRAGMENT_BYTES: + return None, [f"payload is {size} bytes; max is {MAX_SEMANTIC_FRAGMENT_BYTES}"] + try: + fragment = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return None, [f"invalid JSON: {exc}"] + except OSError as exc: + return None, [f"could not read {path}: {exc}"] + errors = validate_semantic_fragment(fragment) + return (None, errors) if errors else (fragment, []) + + +def _validate_semantic_id(errors: list[str], field: str, value: object) -> None: + if not isinstance(value, str): + errors.append(f"{field} must be a string") + return + if not value: + errors.append(f"{field} must not be empty") + return + if len(value) > MAX_SEMANTIC_ID_LENGTH: + errors.append(f"{field} is {len(value)} chars; max is {MAX_SEMANTIC_ID_LENGTH}") + if "/" in value or "\\" in value or ".." in value: + errors.append(f"{field} must not contain path separators or '..'") + if not _SEMANTIC_ID_RE.fullmatch(value): + errors.append(f"{field} contains unsupported characters") + + +def sanitize_semantic_fragment(fragment: dict) -> dict: + """Clean up a semantic extraction fragment in-place. + + Operations: + 1. Removes nodes with ``file_type: "rationale"`` or ``file_type: "concept"`` + that were emitted by an LLM (these are not valid semantic entity types). + 2. Detects nodes whose label reads like a sentence / rationale paragraph + AND that participate in a ``rationale_for`` edge, then converts the + label into a ``rationale`` attribute on the target node and removes + the source-node + its edges. The ``rationale_for`` edge signal applies + regardless of the source node's ``file_type`` — sentence-like nodes + with allowed types (``document``, ``code``) are still cleaned up when + they're explicitly marked as rationale. + 3. Strips nodes whose only distinguishing field is the label itself + (empty id — likely LLM hallucination). + 4. Filters hyperedges so they cannot reference removed or unknown node + IDs after the cleanup passes above. A hyperedge with fewer than two + surviving members is dropped. + + Returns the same dict for convenience. + """ + _invalid_ft = frozenset({"rationale", "concept"}) + + nodes: list[dict] = fragment.get("nodes", []) + edges: list[dict] = fragment.get("edges", []) + hyperedges: list[dict] = fragment.get("hyperedges", []) or [] + + # ---- build lookup maps -------------------------------------------------- + node_by_id: dict[str, dict] = {} + for n in nodes: + nid = n.get("id", "") + if nid: + node_by_id[nid] = n + + # Pre-collect node IDs that source a `rationale_for` edge — these are + # candidates for sentence-like cleanup even when file_type is allowed. + rationale_for_sources: set[str] = set() + for e in edges: + if e.get("relation") == "rationale_for": + src = e.get("source", "") + if src: + rationale_for_sources.add(src) + + # ---- pass 1: identify nodes to remove + rationale candidates ----------- + rationale_candidates: list[dict] = [] + remove_ids: set[str] = set() + keep_nodes: list[dict] = [] + for n in nodes: + nid = n.get("id", "") + if not nid: + # Node without an id cannot be referenced — discard. + continue + ft = n.get("file_type", "") + label = n.get("label", "") + if ft in _invalid_ft: + # Explicitly-invalid file_type ("rationale" or "concept"): if + # the label looks like a sentence we may convert to attribute. + if _is_sentence_like_rationale_label(label): + rationale_candidates.append(n) + remove_ids.add(nid) + continue + if nid in rationale_for_sources and _is_sentence_like_rationale_label(label): + # Allowed file_type, but the node sources a `rationale_for` edge + # AND its label is sentence-like prose. Treat it as rationale + # cleanup material rather than a real graph entity. + rationale_candidates.append(n) + remove_ids.add(nid) + continue + keep_nodes.append(n) + + # ---- pass 2: convert sentence-nodes → rationale attributes -------------- + # Only `rationale_for` edges propagate the rationale text. Other outgoing + # edges (e.g. references, conceptually_related_to) are NOT used as + # attribute-propagation paths — that would corrupt unrelated nodes by + # attaching rationale meant for a different target. + rationale_attrs: dict[str, list[str]] = {} + for rn in rationale_candidates: + rn_id = rn.get("id", "") + text = rn.get("label", "").strip() + for e in edges: + if e.get("relation") != "rationale_for": + continue + if e.get("source") != rn_id: + continue + target_id = e.get("target") + if target_id not in node_by_id or target_id in remove_ids: + continue + rationale_attrs.setdefault(target_id, []).append(text) + + for target_id, texts in rationale_attrs.items(): + if target_id in node_by_id and target_id not in remove_ids: + _append_rationale_attr(node_by_id[target_id], texts) + + # ---- pass 3: strip edges referencing removed nodes ---------------------- + keep_edges: list[dict] = [] + for e in edges: + src = e.get("source", "") + tgt = e.get("target", "") + if src in remove_ids or tgt in remove_ids: + continue + keep_edges.append(e) + + # ---- pass 4: filter hyperedges to surviving node IDs -------------------- + surviving_ids: set[str] = {n.get("id", "") for n in keep_nodes} + surviving_ids.discard("") + keep_hyperedges: list[dict] = [] + for he in hyperedges: + if not isinstance(he, dict): + continue + he_nodes = he.get("nodes") + if not isinstance(he_nodes, list): + continue + filtered = [ref for ref in he_nodes if isinstance(ref, str) and ref in surviving_ids] + if len(filtered) < 2: + # A hyperedge needs at least two surviving members to be meaningful. + continue + if len(filtered) != len(he_nodes): + he = dict(he) + he["nodes"] = filtered + keep_hyperedges.append(he) + + fragment["nodes"] = keep_nodes + fragment["edges"] = keep_edges + fragment["hyperedges"] = keep_hyperedges + return fragment + + +def _is_sentence_like_rationale_label(label: str) -> bool: + """Return True if *label* looks like prose / rationale text rather than an + entity or concept name. + + Heuristics (no false positives on short-concept-edge-cases): + - Longer than *_RATIONALE_MIN_CHARS* chars, OR + - At least *_RATIONALE_MIN_WORDS* whitespace-delimited tokens, AND + - Contains at least one sentence-ending punctuation mark (``. ! ?``) or a + colon (common in "Decision: ..." rationales). + """ + if not label: + return False + label = label.strip() + if len(label) < _RATIONALE_MIN_CHARS: + word_count = len(label.split()) + if word_count < _RATIONALE_MIN_WORDS: + return False + # Must look like actual prose: has sentence-ending punctuation or a colon. + return bool(re.search(r"[.!?:]", label)) + + +def _append_rationale_attr(node: dict, texts: list[str]) -> None: + """Append one or more rationale strings to *node*'s ``rationale`` attribute. + + If the attribute already exists the new texts are appended with a + double-newline separator so downstream consumers can distinguish distinct + rationale fragments. + """ + existing = node.get("rationale", "") + new_text = "\n\n".join(texts).strip() + if existing: + node["rationale"] = existing + "\n\n" + new_text + else: + node["rationale"] = new_text diff --git a/graphify/serve.py b/graphify/serve.py index e0c7ae0..6488422 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -6,7 +6,7 @@ import sys from pathlib import Path import networkx as nx from networkx.readwrite import json_graph -from graphify.security import sanitize_label +from graphify.security import sanitize_label, check_graph_file_size_cap from graphify.build import edge_data @@ -17,6 +17,7 @@ def _load_graph(graph_path: str) -> nx.Graph: raise ValueError(f"Graph path must be a .json file, got: {graph_path!r}") if not resolved.exists(): raise FileNotFoundError(f"Graph file not found: {resolved}") + check_graph_file_size_cap(resolved) safe = resolved data = json.loads(safe.read_text(encoding="utf-8")) if "links" not in data and "edges" in data: diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 2c79d2f..9fadd9d 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -267,7 +267,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant named node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use the closest existing `file_type` (`document` for prose, `code` for code-derived concepts). Do NOT invent file_types like `concept` or `rationale` — valid values are only `code|document|paper|image`. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -304,7 +304,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** @@ -322,12 +322,17 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d, errors = load_validated_semantic_fragment(Path(c)) + if errors: + print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) + continue + d = sanitize_semantic_fragment(d) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -359,6 +364,7 @@ Merge cached + new results into `.graphify_semantic.json`: $(cat .graphify_python) -c " import json from pathlib import Path +from graphify.semantic_cleanup import sanitize_semantic_fragment cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} @@ -380,6 +386,7 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } +merged = sanitize_semantic_fragment(merged) Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " @@ -392,6 +399,7 @@ Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphi $(cat .graphify_python) -c " import sys, json from pathlib import Path +from graphify.semantic_cleanup import sanitize_semantic_fragment ast = json.loads(Path('.graphify_ast.json').read_text()) sem = json.loads(Path('.graphify_semantic.json').read_text()) @@ -413,6 +421,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } +merged = sanitize_semantic_fragment(merged) Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) total = len(merged_nodes) edges = len(merged_edges) diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 8d22d35..cedbf76 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -263,7 +263,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant named node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use the closest existing `file_type` (`document` for prose, `code` for code-derived concepts). Do NOT invent file_types like `concept` or `rationale` — valid values are only `code|document|paper|image`. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -300,7 +300,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** @@ -320,12 +320,17 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path +from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d, errors = load_validated_semantic_fragment(Path(c)) + if errors: + print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) + continue + d = sanitize_semantic_fragment(d) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -357,6 +362,7 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path +from graphify.semantic_cleanup import sanitize_semantic_fragment cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} @@ -378,6 +384,7 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } +merged = sanitize_semantic_fragment(merged) Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " @@ -390,6 +397,7 @@ Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.gra $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path +from graphify.semantic_cleanup import sanitize_semantic_fragment ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) @@ -411,6 +419,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } +merged = sanitize_semantic_fragment(merged) Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) total = len(merged_nodes) edges = len(merged_edges) diff --git a/graphify/symbol_resolution.py b/graphify/symbol_resolution.py new file mode 100644 index 0000000..7bc6809 --- /dev/null +++ b/graphify/symbol_resolution.py @@ -0,0 +1,528 @@ +"""Deterministic symbol indexing and conservative cross-file resolution helpers.""" + +from __future__ import annotations + +import ast +import re +import unicodedata +from dataclasses import dataclass +from pathlib import Path +from collections.abc import Sequence +from typing import Any + +from graphify.security import sanitize_metadata + + + +@dataclass(frozen=True) +class ImportedSymbol: + """A Python imported name that can be used as deterministic resolution evidence.""" + + local_name: str + imported_name: str + module_stem: str + source_file: str + source_location: str + + +def normalise_callable_label(label: str) -> str: + """Normalize a node label into the key used for call resolution.""" + + return label.strip().strip("()").lstrip(".").lower() + + +def node_is_resolvable_symbol(node: dict[str, Any]) -> bool: + """Return True when a node is suitable for deterministic symbol lookup. + + Requires ``file_type == "code"`` as the positive gate — only code-class + nodes participate as call targets. ``_EXCLUDED_FILE_TYPES`` is kept as + defensive-in-depth against legacy data, but the primary guard is the + positive code check. Document/paper/image/concept nodes (e.g. a Markdown + heading whose label happens to match a code identifier) MUST NOT become + callees for a raw code call. + """ + + if node.get("file_type") != "code": + return False + label = str(node.get("label", "")).strip() + if not label: + return False + if label.endswith((".py", ".js", ".ts", ".tsx", ".java", ".go", ".rs")): + return False + return bool(normalise_callable_label(label)) + + +def build_label_index(nodes: list[dict[str, Any]]) -> dict[str, list[str]]: + """Build label -> node id list for conservative cross-file resolution.""" + + index: dict[str, list[str]] = {} + for node in nodes: + if not node_is_resolvable_symbol(node): + continue + node_id = node.get("id") + if not node_id: + continue + key = normalise_callable_label(str(node.get("label", ""))) + if not key: + continue + index.setdefault(key, []).append(str(node_id)) + return index + + +def existing_edge_pairs(edges: list[dict[str, Any]]) -> set[tuple[str, str, str]]: + """Return all existing source/target/relation edge triples. + + Includes relation so that a prior "contains" or "method" edge does not + suppress a semantically distinct "calls" edge between the same endpoints (#F5). + """ + + triples: set[tuple[str, str, str]] = set() + for edge in edges: + source = edge.get("source") + target = edge.get("target") + relation = edge.get("relation", "") + if source and target: + triples.add((str(source), str(target), str(relation))) + return triples + + +def iter_raw_calls(per_file: Sequence[object]) -> list[dict[str, Any]]: + """Return raw calls from all per-file extraction fragments. + + Parameter is ``Sequence[object]`` (not ``Sequence[dict[str, Any] | None]``) + because external extraction output may contain arbitrary deserialized + JSON. Defensive against malformed fragments: non-dict per-file entries + are skipped, non-list ``raw_calls`` are treated as empty, and non-dict + items inside the list are silently dropped. The downstream resolvers + assume every returned item is a dict and they expect this guarantee. + """ + + calls: list[dict[str, Any]] = [] + for result in per_file: + if not isinstance(result, dict): + continue + raw_calls = result.get("raw_calls", []) + if not isinstance(raw_calls, list): + continue + for raw_call in raw_calls: + if isinstance(raw_call, dict): + calls.append(raw_call) + return calls + + +def _module_stem(module_name: str | None) -> str: + """Return the final module component used to match Graphify source stems.""" + + if not module_name: + return "" + return module_name.strip(".").split(".")[-1] + + +def parse_python_import_aliases(path: Path) -> dict[str, ImportedSymbol]: + """Parse deterministic Python import aliases from one source file. + + Supported forms: + from helper import transform + from helper import transform as tx + from .helper import transform + + The function deliberately does not resolve plain ``import helper`` member + calls because current raw call records do not preserve the receiver name from + ``helper.transform()``. That can be added later only after raw call facts are + extended to include the receiver expression. + """ + + try: + source = path.read_text(encoding="utf-8", errors="replace") + tree = ast.parse(source) + except (OSError, SyntaxError): + return {} + + aliases: dict[str, ImportedSymbol] = {} + source_file = str(path) + + # Only top-level `from ... import ...` statements count as file-wide + # evidence. Nested/function-local imports do NOT — they're only valid + # inside their lexical scope, and our raw-call records don't currently + # carry enough scope info to match the import site safely. Walking + # ast.walk(tree) would incorrectly justify calls in other scopes. + for node in tree.body: + if not isinstance(node, ast.ImportFrom): + continue + module_stem = _module_stem(node.module) + if not module_stem: + continue + for alias in node.names: + if alias.name == "*": + continue + local_name = alias.asname or alias.name + aliases[local_name] = ImportedSymbol( + local_name=local_name, + imported_name=alias.name, + module_stem=module_stem, + source_file=source_file, + source_location=f"L{getattr(node, 'lineno', 1)}", + ) + + return aliases + + +def _node_source_stem(node: dict[str, Any]) -> str: + """Return the stem of a node's source file.""" + + source_file = str(node.get("source_file", "")) + if not source_file: + return "" + return Path(source_file).stem + + +def build_python_symbol_index(nodes: list[dict[str, Any]]) -> dict[tuple[str, str], list[str]]: + """Build ``(module_stem, normalized_symbol_name) -> node_ids``. + + This index is stricter than the global label index. It uses both the module + stem and the symbol label, which allows import evidence to resolve calls that + global label uniqueness alone cannot safely resolve. + """ + + index: dict[tuple[str, str], list[str]] = {} + for node in nodes: + if not node_is_resolvable_symbol(node): + continue + source_stem = _node_source_stem(node) + if not source_stem: + continue + label = normalise_callable_label(str(node.get("label", ""))) + if not label: + continue + node_id = node.get("id") + if not node_id: + continue + index.setdefault((source_stem, label), []).append(str(node_id)) + return index + + +def find_unique_python_symbol( + symbol_index: dict[tuple[str, str], list[str]], + imported: ImportedSymbol, +) -> str | None: + """Resolve one imported symbol to exactly one Graphify node id.""" + + candidates = symbol_index.get((imported.module_stem, imported.imported_name.lower()), []) + if len(candidates) == 1: + return candidates[0] + return None + + +def resolve_python_import_guided_calls( + per_file: Sequence[object], + paths: Sequence[Path], + all_nodes: list[dict[str, Any]], + all_edges: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Resolve raw Python calls using explicit import evidence. + + Only ``from module import symbol [as alias]`` forms are handled. Member calls + remain skipped because the current raw call fact does not carry receiver + information. + + Parameter ``per_file`` is ``Sequence[object]`` because external extraction + output may contain arbitrary deserialized JSON. Non-dict slots are + treated as empty fragments, and indices past ``len(per_file)`` are also + treated as empty (paths longer than per_file is tolerated). + """ + + symbol_index = build_python_symbol_index(all_nodes) + known_pairs = existing_edge_pairs(all_edges) + # Build result_by_file defensively: + # - skip indices past the end of per_file (paths shorter than per_file + # also OK; the zip-like behavior is what callers expect) + # - non-dict per_file slots fall back to the empty fragment so the + # downstream `.get("raw_calls", [])` lookup never raises + result_by_file: dict[str, dict[str, Any]] = {} + for index, path in enumerate(paths): + if path.suffix != ".py": + continue + slot: Any = per_file[index] if index < len(per_file) else None + result_by_file[str(path)] = slot if isinstance(slot, dict) else {"nodes": [], "edges": []} + resolved_edges: list[dict[str, Any]] = [] + + for path in paths: + if path.suffix != ".py": + continue + source_file = str(path) + aliases = parse_python_import_aliases(path) + if not aliases: + continue + file_result = result_by_file.get(source_file, {"raw_calls": []}) + raw_calls = file_result.get("raw_calls", []) + if not isinstance(raw_calls, list): + continue + for raw_call in raw_calls: + if not isinstance(raw_call, dict): + continue + if raw_call.get("is_member_call"): + continue + callee = str(raw_call.get("callee", "")).strip() + if not callee: + continue + imported = aliases.get(callee) + if imported is None: + continue + target = find_unique_python_symbol(symbol_index, imported) + if target is None: + continue + caller = str(raw_call.get("caller_nid", "")) + if not caller or caller == target: + continue + pair = (caller, target, "calls") + if pair in known_pairs: + continue + known_pairs.add(pair) + resolved_edges.append( + { + "source": caller, + "target": target, + "relation": "calls", + "context": "import_guided_call", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": raw_call.get("source_file", source_file), + "source_location": raw_call.get("source_location") or imported.source_location, + "weight": 1.0, + "metadata": sanitize_metadata({ + "resolver": "python_import_guided", + "local_name": imported.local_name, + "imported_name": imported.imported_name, + "module_stem": imported.module_stem, + "import_source_location": imported.source_location, + }), + } + ) + + return resolved_edges + + +def resolve_cross_file_raw_calls( + per_file: Sequence[dict[str, Any] | None], + all_nodes: list[dict[str, Any]], + all_edges: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Resolve unqualified raw calls conservatively after all files are known. + + This intentionally preserves Graphify's existing behavior: + - member calls are skipped; + - ambiguous labels are skipped; + - only a single unique candidate is emitted; + - emitted edges are INFERRED because the raw call alone is not import proof. + """ + + label_index = build_label_index(all_nodes) + known_pairs = existing_edge_pairs(all_edges) + resolved: list[dict[str, Any]] = [] + + for raw_call in iter_raw_calls(per_file): + callee = str(raw_call.get("callee", "")).strip() + if not callee: + continue + if raw_call.get("is_member_call"): + continue + candidates = label_index.get(callee.lower(), []) + if len(candidates) != 1: + continue + target = candidates[0] + caller = str(raw_call.get("caller_nid", "")) + if not caller: + continue + if target == caller: + continue + pair = (caller, target, "calls") + if pair in known_pairs: + continue + known_pairs.add(pair) + resolved.append( + { + "source": caller, + "target": target, + "relation": "calls", + "context": "call", + "confidence": "INFERRED", + "confidence_score": 0.8, + "source_file": raw_call.get("source_file", ""), + "source_location": raw_call.get("source_location"), + "weight": 1.0, + } + ) + + return resolved + + +def _bash_make_id(*parts: str) -> str: + """Exact copy of extract._make_id — kept here to avoid an import cycle.""" + combined = "_".join(p.strip("_.") for p in parts if p) + combined = unicodedata.normalize("NFKC", combined) + cleaned = re.sub(r"[^\w]+", "_", combined, flags=re.UNICODE) + cleaned = re.sub(r"_+", "_", cleaned) + return cleaned.strip("_").casefold() + + +def _file_node_id_for_path(path: Path, root: Path) -> str: + # Resolve both sides so callers that pass relative or non-canonical roots + # get the same canonical relative path that extract()'s id_remap produces. + # _bash_make_id is an exact copy of extract._make_id, so IDs match. + try: + return _bash_make_id(str(path.resolve().relative_to(root.resolve()))) + except ValueError: + return _bash_make_id(str(path)) # path outside root: hash absolute path as fallback + + +def resolve_bash_source_edges( + per_file: Sequence[dict | None], + paths: Sequence[Path], + root: Path, + existing_edges: list[dict] | None = None, +) -> list[dict]: + """Resolve Bash source/import edges and source-backed function calls. + + Defensive against malformed extraction fragments: non-dict ``per_file`` + entries, missing ``bash_sources``/``raw_calls`` keys, non-dict items in + those lists, and missing/empty ``id`` / ``target_path`` / ``caller_nid`` + fields all yield silent skips rather than ``KeyError``. + + ``bash_sources[].target_path`` contract (Graphify static-analysis policy): + - Absolute paths: resolved as-is. + - Relative paths: resolved against the *source file's* directory + (i.e. ``Path(path).parent / target_path``). + NOTE: this is a deterministic static-analysis policy chosen by + Graphify, NOT bash runtime semantics. At runtime, ``source ./X`` + is resolved against the shell's current working directory. We + prefer source-file-relative because static analysis cannot know + the future CWD; resolving against the file being analyzed gives + deterministic, reproducible edges across runs. + - Inputs of type ``str`` and ``pathlib.Path`` are processed. + Anything else is silently skipped. + """ + path_by_index = [Path(p).resolve() for p in paths] + file_nid_by_path = {p: _file_node_id_for_path(p, root) for p in path_by_index} # resolved paths only + + functions_by_file: dict[str, dict[str, str]] = {} + for result, path in zip(per_file, path_by_index): + if not isinstance(result, dict): + continue + file_nid = file_nid_by_path[path] + nodes = result.get("nodes", []) + if not isinstance(nodes, list): + continue + for node in nodes: + if not isinstance(node, dict): + continue + metadata = node.get("metadata", {}) + if not isinstance(metadata, dict): + continue + if metadata.get("kind") != "bash_function": + continue + name = str(node.get("label", "")).removesuffix("()").strip() + node_id = node.get("id") + if not name or not node_id: + continue + functions_by_file.setdefault(file_nid, {})[name] = str(node_id) + + sourced_files: dict[str, set[str]] = {} + resolved_edges: list[dict] = [] + existing = existing_edge_pairs(existing_edges or []) + + for result, path in zip(per_file, path_by_index): + if not isinstance(result, dict): + continue + src_file_nid = file_nid_by_path[path] + bash_sources = result.get("bash_sources", []) + if not isinstance(bash_sources, list): + continue + for source in bash_sources: + if not isinstance(source, dict): + continue + raw_target = source.get("target_path") + if not isinstance(raw_target, (str, Path)) or not str(raw_target).strip(): + continue + # Relative paths resolve against the source file's directory — + # Graphify static-analysis policy (NOT bash runtime semantics; + # at runtime `source ./X` is CWD-relative, but static analysis + # can't know the future CWD, so we resolve relative to the + # file being analyzed for deterministic, reproducible edges). + candidate = Path(raw_target) + if not candidate.is_absolute(): + candidate = path.parent / candidate + try: + target_path = candidate.resolve() + except (OSError, RuntimeError): + continue + target_file_nid = file_nid_by_path.get(target_path) + if target_file_nid is None: + continue + sourced_files.setdefault(src_file_nid, set()).add(target_file_nid) + key = (src_file_nid, target_file_nid, "imports_from") + if key in existing: + continue + existing.add(key) + resolved_edges.append( + { + "source": src_file_nid, + "target": target_file_nid, + "relation": "imports_from", + "context": "import", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": source.get("source_file", str(path)), + "source_location": source.get("source_location", ""), + "weight": 1.0, + } + ) + + for result, path in zip(per_file, path_by_index): + if not isinstance(result, dict): + continue + caller_file_nid = file_nid_by_path[path] + imported_file_ids = sourced_files.get(caller_file_nid, set()) + if not imported_file_ids: + continue + raw_calls = result.get("raw_calls", []) + if not isinstance(raw_calls, list): + continue + for raw_call in raw_calls: + if not isinstance(raw_call, dict): + continue + if raw_call.get("language") != "bash": + continue + callee = raw_call.get("callee") + caller_nid = raw_call.get("caller_nid") + # callee must be a non-empty string — anything else (list, dict, + # int, None, …) is silently skipped to avoid TypeError on the + # `in functions_by_file[...]` membership check below. + if not isinstance(callee, str) or not callee or not caller_nid: + continue + matches = [ + functions_by_file[file_nid][callee] + for file_nid in imported_file_ids + if callee in functions_by_file.get(file_nid, {}) + ] + if len(matches) != 1: + continue + target = matches[0] + key = (str(caller_nid), target, "calls") + if key in existing: + continue + existing.add(key) + resolved_edges.append( + { + "source": str(caller_nid), + "target": target, + "relation": "calls", + "context": "call", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": raw_call.get("source_file", str(path)), + "source_location": raw_call.get("source_location", ""), + "weight": 1.0, + } + ) + + return resolved_edges diff --git a/graphify/tree_html.py b/graphify/tree_html.py index 8ef177a..3d825ad 100644 --- a/graphify/tree_html.py +++ b/graphify/tree_html.py @@ -569,6 +569,8 @@ def write_tree_html( # kept for CLI compatibility with the older signature; ignored now top_k_edges: int = 0, ) -> Path: + from graphify.security import check_graph_file_size_cap + check_graph_file_size_cap(graph_path) graph = json.loads(graph_path.read_text(encoding="utf-8")) tree = build_tree(graph, root=root, max_children=max_children, project_label=project_label) diff --git a/graphify/watch.py b/graphify/watch.py index ade55a8..71907a3 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -331,6 +331,7 @@ def _rebuild_code( from graphify.analyze import god_nodes, surprising_connections, suggest_questions from graphify.report import generate from graphify.export import to_json, to_html + from graphify.security import check_graph_file_size_cap detected = detect(watch_path, follow_symlinks=follow_symlinks) code_files = [Path(f) for f in detected['files']['code']] @@ -389,6 +390,7 @@ def _rebuild_code( existing_graph_data: dict = {} if existing_graph.exists(): try: + check_graph_file_size_cap(existing_graph) existing = json.loads(existing_graph.read_text(encoding="utf-8")) existing_graph_data = existing new_ast_ids = {n["id"] for n in result["nodes"]} @@ -433,6 +435,7 @@ def _rebuild_code( same_graph = False if existing_graph.exists(): try: + check_graph_file_size_cap(existing_graph) existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) same_graph = ( json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False) @@ -526,6 +529,7 @@ def _rebuild_code( same_report = False if existing_graph.exists(): try: + check_graph_file_size_cap(existing_graph) existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) same_graph = ( json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False) diff --git a/pyproject.toml b/pyproject.toml index 469e515..8dc5fb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,24 @@ all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_ [project.scripts] graphify = "graphify.__main__:main" +[dependency-groups] +dev = [ + "bandit>=1.9.4", + "build>=1.5.0", + "hypothesis>=6.152.7", + "nuitka>=4.1", + "patchelf>=0.17.2.4 ; sys_platform != 'win32'", + "pip-audit>=2.10.0", + "pre-commit>=4.6.0", + "pyright>=1.1.409", + "pytest>=9.0.3", + "pytest-cov>=7.1.0", + "ruff>=0.15.13", + "safety>=3.7.0", + "setuptools>=82.0.1", + "wheel>=0.47.0", +] + [tool.uv] # Install via: uv tool install graphifyy # Run without installing: uvx graphifyy install @@ -91,11 +109,15 @@ norecursedirs = [ [tool.bandit] skips = ["B404"] -[dependency-groups] -dev = [ - "build>=1.5.0", - "nuitka>=4.1", - "patchelf>=0.17.2.4 ; sys_platform != 'win32'", - "setuptools>=82.0.1", - "wheel>=0.47.0", -] +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +# Keep the committed baseline conservative until upstream adopts a broader lint policy. +select = ["E9", "F63", "F7", "F82"] + +[tool.pyright] +include = ["graphify", "tests"] +pythonVersion = "3.10" +typeCheckingMode = "basic" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..835ff5e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +_ANALYZE_WARNING_FILTERS = ( + "ignore:Tensorflow not installed; ParametricUMAP will be unavailable:ImportWarning:umap", + "ignore:Please import `random` from the `scipy\\.sparse` namespace.*:" + "DeprecationWarning:hyppo\\.independence\\.hhg", + "ignore:The keyword argument 'nopython=False' was supplied.*:Warning:numba\\.core\\.decorators", +) + + +def pytest_collection_modifyitems(items: list[Any]) -> None: + for item in items: + if item.path.name != "test_analyze.py": + continue + for warning_filter in _ANALYZE_WARNING_FILTERS: + item.add_marker(pytest.mark.filterwarnings(warning_filter)) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 0c40fa6..b5751ad 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -170,3 +170,14 @@ def test_print_benchmark_survives_cp1252_stdout(tmp_path, monkeypatch, capsys): # ASCII fallbacks must be present, fancy glyphs must not. assert "─" not in written assert "→" not in written + + +def test_run_benchmark_rejects_oversized_graph(monkeypatch, tmp_path): + """#F4: run_benchmark must refuse to read a graph.json that exceeds + the size cap before parsing it into memory.""" + G = _make_graph() + graph_file = tmp_path / "graph.json" + _write_graph(G, graph_file) + monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) + with pytest.raises(ValueError, match="exceeds"): + run_benchmark(str(graph_file)) diff --git a/tests/test_build.py b/tests/test_build.py index 85d59fd..58f2863 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -344,3 +344,15 @@ def test_build_from_json_relative_source_file_unchanged(tmp_path): } G = build_from_json(extraction, root=tmp_path) assert G.nodes["foo_bar"]["source_file"] == "src/foo.py" + + +def test_build_merge_rejects_oversized_existing_graph(monkeypatch, tmp_path): + """#F4: build_merge must refuse to read an existing graph.json that + exceeds the size cap, rather than json.loads-ing it into memory.""" + import pytest + + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps({"nodes": [], "links": []}), encoding="utf-8") + monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) + with pytest.raises(ValueError, match="exceeds"): + build_merge([], graph_path, dedup=False) diff --git a/tests/test_callflow_html.py b/tests/test_callflow_html.py index 0e4c466..9605c9b 100644 --- a/tests/test_callflow_html.py +++ b/tests/test_callflow_html.py @@ -168,3 +168,20 @@ def test_derive_sections_groups_by_architecture_keywords(): assert "extract-pipeline" in ids assert "outputs-docs" in ids assert "tests-fixtures" in ids + + +def test_load_graph_rejects_oversized_file(monkeypatch, tmp_path): + """#F4: callflow_html.load_graph must refuse to read a graph.json that + exceeds the size cap (SystemExit via translated ValueError).""" + import pytest + from graphify.callflow_html import load_graph + + graph_path = tmp_path / "graph.json" + graph_path.write_text( + json.dumps({"nodes": [], "links": []}), + encoding="utf-8", + ) + monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) + with pytest.raises(SystemExit) as excinfo: + load_graph(graph_path) + assert "exceeds" in str(excinfo.value) diff --git a/tests/test_detect.py b/tests/test_detect.py index 7bf8546..26c745d 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -608,8 +608,6 @@ def test_save_manifest_without_filter_unchanged_for_code(tmp_path): manifest = json.loads(Path(manifest_path).read_text()) assert str(py) in manifest assert manifest[str(py)]["ast_hash"] != "" - - # Regression tests for #945 - .gitignore fallback when no .graphifyignore exists def test_gitignore_fallback_when_no_graphifyignore(tmp_path): @@ -672,3 +670,296 @@ def test_detect_extra_excludes_pattern(tmp_path): assert any("main.py" in f for f in code) assert not any("secret.py" in f for f in code) assert not any("legacy" in f for f in code) + + +# --------------------------------------------------------------------------- +# Shebang interpreter parsing +# --------------------------------------------------------------------------- + +def test_shebang_interpreter_plain(tmp_path): + """Plain shebang returns the interpreter basename.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "plain" + script.write_bytes(b"#!/usr/bin/python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + + +def test_shebang_interpreter_env_single_arg(tmp_path): + """`#!/usr/bin/env python3` returns the interpreter, not 'env'.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_single" + script.write_bytes(b"#!/usr/bin/env python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + + +def test_shebang_interpreter_env_dash_s(tmp_path): + """`#!/usr/bin/env -S python3 -u` (-S split-args form) recovers the interpreter.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_dashs" + script.write_bytes(b"#!/usr/bin/env -S python3 -u\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + + +def test_shebang_interpreter_env_with_flags(tmp_path): + """`#!/usr/bin/env -i bash` skips env flags and resolves to the interpreter.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_flags" + script.write_bytes(b"#!/usr/bin/env -i bash\necho hi\n") + assert _shebang_interpreter(script) == "bash" + + +def test_shebang_interpreter_env_with_assignment(tmp_path): + """`#!/usr/bin/env DEBUG=1 python3` skips var=value assignments.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_assign" + script.write_bytes(b"#!/usr/bin/env DEBUG=1 python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + + +def test_shebang_interpreter_no_shebang(tmp_path): + """File without shebang returns None.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "no_shebang" + script.write_bytes(b"print('x')\n") + assert _shebang_interpreter(script) is None + + +def test_shebang_interpreter_quoted_path(tmp_path): + """Quoted interpreter path with spaces parses correctly via shlex.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "quoted" + # Note: actual `#!` on disk wouldn't permit a quoted path on most kernels, + # but shlex must not crash and should produce a reasonable answer + script.write_bytes(b'#!"/usr/local/bin/python3"\nprint("x")\n') + assert _shebang_interpreter(script) == "python3" + + +def test_shebang_file_type_classifies_via_interpreter(tmp_path): + """Classify file type via interpreter, including env -S form.""" + script = tmp_path / "tool" + script.write_bytes(b"#!/usr/bin/env -S python3 -u\nprint('x')\n") + # No extension, must be classified via shebang + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_unreadable_returns_none(tmp_path): + """Unreadable / nonexistent files return None, never raise.""" + from graphify.detect import _shebang_interpreter + missing = tmp_path / "does_not_exist" + assert _shebang_interpreter(missing) is None + + +def test_shebang_interpreter_env_unset_with_operand(tmp_path): + """`env -u VAR python3` skips both -u and its required operand.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_unset" + script.write_bytes(b"#!/usr/bin/env -u PYTHONPATH python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_chdir_with_operand(tmp_path): + """`env -C /tmp python3` skips both -C and its workdir operand.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_chdir" + script.write_bytes(b"#!/usr/bin/env -C /tmp python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_path_with_operand(tmp_path): + """`env -P /bin python3` skips both -P and its utilpath operand.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_path" + script.write_bytes(b"#!/usr/bin/env -P /bin python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_dash_s_after_flag(tmp_path): + """`env -i -S "python3 -u"` handles -S after another env flag.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_flag_dash_s" + script.write_bytes(b'#!/usr/bin/env -i -S "python3 -u"\nprint("x")\n') + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_clumped_u_operand(tmp_path): + """Clumped `-uPYTHONPATH` form (no space between flag and operand) is one arg.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_clumped" + script.write_bytes(b"#!/usr/bin/env -uPYTHONPATH python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_missing_operand_returns_none(tmp_path): + """`env -u` with no operand → not a valid command, return None.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_missing_op" + script.write_bytes(b"#!/usr/bin/env -u\n") + assert _shebang_interpreter(script) is None + + +def test_shebang_interpreter_env_gnu_split_string_equals(tmp_path): + """GNU `--split-string='python3 -u'` (with `=` operand) → python3.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_split_eq" + script.write_bytes(b"#!/usr/bin/env --split-string='python3 -u'\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_gnu_split_string_separate(tmp_path): + """GNU `--split-string "python3 -u"` (separate operand) → python3.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_split_sep" + script.write_bytes(b'#!/usr/bin/env --split-string "python3 -u"\nprint("x")\n') + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_gnu_argv0_operand(tmp_path): + """GNU `-a alias python3` skips both -a and its argv0 operand.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_argv0" + script.write_bytes(b"#!/usr/bin/env -a alias python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_compact_dash_s(tmp_path): + """Compact `-Spython3 -u` form (no space between -S and packed string).""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_compact_dash_s" + script.write_bytes(b"#!/usr/bin/env -Spython3 -u\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_compact_v_then_s(tmp_path): + """Compact `-vSpython3` (-v plus compact -S).""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_compact_vs" + script.write_bytes(b"#!/usr/bin/env -vSpython3 -u\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_long_unset_separate_operand(tmp_path): + """GNU `--unset PYTHONPATH python3` (separate operand).""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_long_unset" + script.write_bytes(b"#!/usr/bin/env --unset PYTHONPATH python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_long_unset_equals(tmp_path): + """GNU `--unset=PYTHONPATH python3` (`=` operand form).""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_long_unset_eq" + script.write_bytes(b"#!/usr/bin/env --unset=PYTHONPATH python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_long_chdir_separate_operand(tmp_path): + """GNU `--chdir /tmp python3` (separate operand).""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_long_chdir" + script.write_bytes(b"#!/usr/bin/env --chdir /tmp python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_long_chdir_equals(tmp_path): + """GNU `--chdir=/tmp python3` (`=` operand form).""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_long_chdir_eq" + script.write_bytes(b"#!/usr/bin/env --chdir=/tmp python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_signal_flags(tmp_path): + """GNU signal-handling flags skip transparently.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_signal" + script.write_bytes(b"#!/usr/bin/env --default-signal=TERM --ignore-signal=PIPE python3\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_unknown_option_returns_none(tmp_path): + """Unknown hyphen-prefixed env option → return None rather than guessing.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_unknown" + script.write_bytes(b"#!/usr/bin/env --no-such-flag python3\n") + # Must refuse to guess: if we can't classify the option, we can't trust + # that the next token is the interpreter. Safer to return None. + assert _shebang_interpreter(script) is None + + +def test_shebang_interpreter_env_dash_s_assignment_before_interpreter(tmp_path): + """`-S` payload may carry NAME=value assignments before the interpreter.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_s_assignment" + script.write_bytes( + b"#!/usr/bin/env -S PYTHONPATH=/opt/custom:${PYTHONPATH} python3\n" + b"print('x')\n" + ) + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_dash_s_flag_before_interpreter(tmp_path): + """`-S` payload may carry env flags (e.g. -i) before the interpreter.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_s_flag" + script.write_bytes(b"#!/usr/bin/env -S -i OLDUSER=${USER} python3\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_long_split_assignment_before_interpreter(tmp_path): + """`--split-string=` payload may carry assignments before the interpreter.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_long_split_assignment" + script.write_bytes( + b"#!/usr/bin/env --split-string='PYTHONPATH=/opt/custom:${PYTHONPATH} python3 -u'\n" + b"print('x')\n" + ) + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_long_split_flag_before_interpreter(tmp_path): + """`--split-string=` payload may carry env flags before the interpreter.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_long_split_flag" + script.write_bytes(b"#!/usr/bin/env --split-string='-i python3 -u'\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE + + +def test_shebang_interpreter_env_nested_split_string_rejected(tmp_path): + """A `-S` payload that itself starts with `-S` is rejected (allow_split=False + on the recursive call bounds the recursion depth at one). Without this guard, + a malicious or strange shebang could spin the parser indefinitely.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_nested_split" + # Outer -S splits into ["-S", "python3", "-u"]; inner -S is treated as an + # unknown option in the recursed pass, so we get None (refuse to guess). + script.write_bytes(b"#!/usr/bin/env -S -S python3 -u\nprint('x')\n") + assert _shebang_interpreter(script) is None + + +def test_shebang_interpreter_env_vs_assignment_before_interpreter(tmp_path): + """`-vS` packed payload also re-parses for leading assignments.""" + from graphify.detect import _shebang_interpreter + script = tmp_path / "env_vs_assignment" + script.write_bytes(b"#!/usr/bin/env -vS DEBUG=1 python3 -u\nprint('x')\n") + assert _shebang_interpreter(script) == "python3" + assert classify_file(script) == FileType.CODE diff --git a/tests/test_export.py b/tests/test_export.py index 832c870..7f0ed5e 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -97,6 +97,33 @@ def test_to_html_contains_visjs(): content = out.read_text() assert "vis-network" in content + +def test_to_html_pins_visjs_version_with_sri(): + """vis-network script tag must use a pinned versioned URL with a sha384 + Subresource Integrity hash and crossorigin=anonymous. Without this, + a compromised CDN could ship arbitrary JavaScript into every rendered + graph viewer. The hash was verified against the upstream file at + https://unpkg.com/vis-network@9.1.6/standalone/umd/vis-network.min.js + (sha384-Ux6phic9PEHJ38YtrijhkzyJ8yQlH8i/+buBR8s3mAZOJrP1gwyvAcIYl3GWtpX1). + Bumping the vis-network version MUST update both the URL and the hash. + """ + G = make_graph() + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.html" + to_html(G, communities, str(out)) + content = out.read_text() + + # Versioned URL — unversioned `vis-network/standalone/...` is rejected. + assert "vis-network@9.1.6/standalone/umd/vis-network.min.js" in content + assert "https://unpkg.com/vis-network/standalone" not in content + + # SRI integrity attribute pinning the known-good hash. + assert 'integrity="sha384-Ux6phic9PEHJ38YtrijhkzyJ8yQlH8i/+buBR8s3mAZOJrP1gwyvAcIYl3GWtpX1"' in content + + # crossorigin="anonymous" is required for SRI on cross-origin scripts. + assert 'crossorigin="anonymous"' in content + def test_to_html_contains_search(): G = make_graph() communities = cluster(G) diff --git a/tests/test_extract.py b/tests/test_extract.py index 30fc83e..8f2a996 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -521,6 +521,175 @@ def test_extract_bash_missing_grammar_returns_error(): assert result["nodes"] == [] +def test_extract_bash_rejects_command_substitution_as_call(tmp_path): + """`$(build)` must not be recorded as a call edge to build().""" + script = tmp_path / "command_substitution.sh" + script.write_text( + "#!/usr/bin/env bash\n" + "build() { echo build; }\n" + "$(build)\n" + ) + result = extract_bash(script) + labels = {n["id"]: n["label"] for n in result["nodes"]} + call_pairs = [ + (labels.get(e["source"], e["source"]), labels.get(e["target"], e["target"])) + for e in result["edges"] + if e["relation"] == "calls" + ] + assert call_pairs == [], f"Command substitution erroneously emitted call edges: {call_pairs}" + + +def test_extract_bash_process_substitution_not_recorded(tmp_path): + """`<(helper)` (process substitution) must not be recorded as a call edge.""" + script = tmp_path / "process_substitution.sh" + script.write_text( + "#!/usr/bin/env bash\n" + "helper() { echo h; }\n" + "diff <(helper) <(helper)\n" + ) + result = extract_bash(script) + labels = {n["id"]: n["label"] for n in result["nodes"]} + call_pairs = [ + (labels.get(e["source"], e["source"]), labels.get(e["target"], e["target"])) + for e in result["edges"] + if e["relation"] == "calls" + ] + assert call_pairs == [], f"Process substitution erroneously emitted call edges: {call_pairs}" + + +def test_extract_bash_shadowing_function_is_recorded(tmp_path): + """User-defined function shadowing an external command (install/find/etc.) must still produce a call edge.""" + script = tmp_path / "shadowing.sh" + script.write_text( + "#!/usr/bin/env bash\n" + "install() { echo install; }\n" + "deploy() { install; }\n" + ) + result = extract_bash(script) + labels = {n["id"]: n["label"] for n in result["nodes"]} + call_pairs = [ + (labels.get(e["source"], e["source"]), labels.get(e["target"], e["target"])) + for e in result["edges"] + if e["relation"] == "calls" + ] + assert ("deploy()", "install()") in call_pairs, ( + f"Shadowing function call not recorded; got: {call_pairs}" + ) + + +def test_extract_bash_creates_entrypoint_node(tmp_path): + """Every bash file produces a `bash_entrypoint` node distinct from the file node, joined by a `contains` edge.""" + script = tmp_path / "with_entrypoint.sh" + script.write_text("#!/usr/bin/env bash\nfoo() { :; }\n") + result = extract_bash(script) + kinds = [n.get("metadata", {}).get("kind") for n in result["nodes"]] + assert "bash_entrypoint" in kinds, f"No bash_entrypoint node; kinds={kinds}" + assert "file" in kinds, f"No file node; kinds={kinds}" + file_node = next(n for n in result["nodes"] if n.get("metadata", {}).get("kind") == "file") + entry_node = next(n for n in result["nodes"] if n.get("metadata", {}).get("kind") == "bash_entrypoint") + contains_edges = [ + e for e in result["edges"] + if e["relation"] == "contains" and e["source"] == file_node["id"] and e["target"] == entry_node["id"] + ] + assert contains_edges, "Missing contains edge from file → bash_entrypoint" + + +def test_extract_bash_top_level_call_attributes_to_entrypoint(tmp_path): + """Top-level function call attaches to the entrypoint node, not orphaned.""" + script = tmp_path / "top_level_call.sh" + script.write_text( + "#!/usr/bin/env bash\n" + "build() { echo build; }\n" + "build\n" + ) + result = extract_bash(script) + entry_node = next( + (n for n in result["nodes"] if n.get("metadata", {}).get("kind") == "bash_entrypoint"), + None, + ) + assert entry_node is not None, "No entrypoint node created" + call_pairs = [ + (e["source"], e["target"]) + for e in result["edges"] + if e["relation"] == "calls" + ] + target_ids = {tgt for _, tgt in call_pairs if any(n["id"] == tgt and n["label"] == "build()" for n in result["nodes"])} + source_ids_to_build = {src for src, tgt in call_pairs if tgt in target_ids} + assert entry_node["id"] in source_ids_to_build, ( + f"Top-level call to build not attributed to entrypoint; calls={call_pairs}" + ) + + +# --------------------------------------------------------------------------- +# PR #893 regression tests — bash extractor Copilot review findings +# --------------------------------------------------------------------------- + + +def test_extract_bash_entrypoint_no_collision_with_function_named_script(tmp_path): + """Entrypoint node must have a distinct ID from a function also named 'script'. + + _make_id strips leading/trailing '_.' from each part, so + _make_id(stem, "__script__") strips to _make_id(stem, "script"), which is + identical to _make_id(stem, "script") for a function named 'script'. + """ + script = tmp_path / "deploy.sh" + script.write_text("#!/usr/bin/env bash\nfunction script() { echo hi; }\n") + result = extract_bash(script) + entry_nodes = [n for n in result["nodes"] if n.get("metadata", {}).get("kind") == "bash_entrypoint"] + func_nodes = [n for n in result["nodes"] if n.get("metadata", {}).get("kind") == "bash_function"] + assert entry_nodes, "Must have a bash_entrypoint node" + assert func_nodes, "Must have a bash_function node for 'script'" + entry_id = entry_nodes[0]["id"] + func_id = func_nodes[0]["id"] + assert entry_id != func_id, ( + f"Entrypoint ID must not collide with function 'script' ID; both are '{entry_id}'" + ) + + +def test_extract_bash_nested_function_calls_recorded(tmp_path): + """Calls made inside a nested (inner) function body must be collected.""" + script = tmp_path / "nested.sh" + script.write_text( + "#!/usr/bin/env bash\n" + "function do_work() { :; }\n" + "function outer() {\n" + " function inner() {\n" + " do_work\n" + " }\n" + " inner\n" + "}\n" + ) + result = extract_bash(script) + node_id_by_label = {n["label"].rstrip("()"): n["id"] for n in result["nodes"]} + assert "inner" in node_id_by_label, f"inner function must be discovered; labels={list(node_id_by_label)}" + assert "do_work" in node_id_by_label, f"do_work function must be discovered; labels={list(node_id_by_label)}" + calls = {(e["source"], e["target"]) for e in result["edges"] if e.get("relation") == "calls"} + inner_id = node_id_by_label["inner"] + do_work_id = node_id_by_label["do_work"] + assert (inner_id, do_work_id) in calls, ( + f"inner→do_work call edge must be recorded; got calls={calls}" + ) + + +def test_extract_bash_source_user_defined_emits_calls_not_imports_from(tmp_path): + """When 'source' is a user-defined function, 'source ./file.sh' must emit a + calls edge, not an imports_from edge. The user-defined function shadows the + built-in source command.""" + helpers = tmp_path / "helpers.sh" + helpers.write_text("#!/bin/bash\n") + script = tmp_path / "run.sh" + script.write_text( + "#!/usr/bin/env bash\n" + "function source() { echo 'custom source'; }\n" + "source ./helpers.sh\n" + ) + result = extract_bash(script) + import_edges = [e for e in result["edges"] if e.get("relation") == "imports_from"] + assert not import_edges, ( + f"'source' is a user-defined function; 'source ./helpers.sh' must not emit imports_from; got: {import_edges}" + ) + + # --------------------------------------------------------------------------- # JSON extractor tests (#866) # --------------------------------------------------------------------------- @@ -592,3 +761,19 @@ def test_extract_bash_via_dispatch(): def test_extract_json_via_dispatch(): from graphify.extract import _get_extractor assert _get_extractor(Path("foo.json")) is extract_json + + +def test_extract_bash_node_metadata_is_sanitized(): + """Bash extractor must route node metadata through sanitize_metadata so + HTML-sensitive characters cannot reach downstream graph viewers raw.""" + result = extract_bash(FIXTURES / "sample.sh") + assert "error" not in result + for node in result["nodes"]: + meta = node.get("metadata", {}) + # Static bash metadata is currently {"language": "bash", "kind": "code"}; + # both pass through sanitisation unchanged, but the values must be the + # post-sanitisation strings (not raw objects). + for value in meta.values(): + if isinstance(value, str): + assert "<" not in value + assert "\x00" not in value diff --git a/tests/test_global_graph.py b/tests/test_global_graph.py index 3e84fdf..f40d9c6 100644 --- a/tests/test_global_graph.py +++ b/tests/test_global_graph.py @@ -277,3 +277,22 @@ def test_merge_graphs_prefixes_ids(tmp_path): assert "repo1::userservice" in merged.nodes assert "repo2::userservice" in merged.nodes assert merged.number_of_nodes() == 2 # no silent collapse + + +def test_global_add_rejects_oversized_source_graph(monkeypatch, tmp_path): + """#F4: global_add must refuse to read a source graph.json that + exceeds the size cap, rather than json.loads-ing it into memory.""" + import pytest + + src_graph = tmp_path / "graph.json" + G = _make_graph([{"id": "x", "label": "X", "source_file": "src/x.py"}]) + _graph_to_json(G, src_graph) + + global_dir = tmp_path / ".graphify" + monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) + with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ + patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ + patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): + from graphify.global_graph import global_add + with pytest.raises(ValueError, match="exceeds"): + global_add(src_graph, "repoA") diff --git a/tests/test_multigraph_compat.py b/tests/test_multigraph_compat.py new file mode 100644 index 0000000..36902e6 --- /dev/null +++ b/tests/test_multigraph_compat.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import networkx as nx + +from graphify.multigraph_compat import ( + CapabilityCheck, + MultigraphCapabilityResult, + probe_multigraph_capabilities, + require_multigraph_capabilities, +) + + +def test_probe_multigraph_capabilities_passes_current_runtime() -> None: + result = probe_multigraph_capabilities() + + assert result.ok, result.error_message() + assert result.python_version + assert result.networkx_version + assert {check.name for check in result.checks} == { + "keyed_parallel_edges", + "node_link_edges_links_round_trip", + "duplicate_key_overwrite_semantics", + "reserved_key_attr_rejected", + "remove_edges_from_two_tuple_semantics", + "to_undirected_preserves_multigraph_type", + } + + +def test_require_multigraph_capabilities_returns_result() -> None: + result = require_multigraph_capabilities() + + assert result.ok + + +def test_failure_message_is_actionable() -> None: + result = MultigraphCapabilityResult( + python_version="3.10.0", + networkx_version="0.0", + checks=(CapabilityCheck("node_link_edges_links_round_trip", False, "boom"),), + ) + + message = result.error_message() + + assert "--multigraph requires NetworkX keyed MultiDiGraph node-link" in message + assert "Default simple graph mode remains available" in message + assert "node_link_edges_links_round_trip: boom" in message + + +def test_networkx_duplicate_key_overwrite_trap_is_real() -> None: + graph = nx.MultiDiGraph() + + graph.add_edge("a", "b", key="same", relation="first") + graph.add_edge("a", "b", key="same", relation="second") + + assert graph.number_of_edges("a", "b") == 1 + assert graph["a"]["b"]["same"]["relation"] == "second" diff --git a/tests/test_multigraph_diagnostics.py b/tests/test_multigraph_diagnostics.py new file mode 100644 index 0000000..8c39b8e --- /dev/null +++ b/tests/test_multigraph_diagnostics.py @@ -0,0 +1,460 @@ +from __future__ import annotations + +from copy import deepcopy +import json +from pathlib import Path + +import pytest + +import graphify.__main__ as mainmod +from graphify.diagnostics import ( + diagnose_extraction, + diagnose_file, + format_diagnostic_json, + format_diagnostic_report, + scan_producer_suppression_sites, +) + + +def _diagnostic_fixture() -> dict: + return { + "nodes": [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, + {"id": "c", "label": "C", "file_type": "code", "source_file": "c.py"}, + ], + "edges": [ + { + "source": "a", + "target": "b", + "relation": "calls", + "confidence": "EXTRACTED", + "source_file": "a.py", + "source_location": "L1", + "context": "call", + }, + { + "source": "a", + "target": "b", + "relation": "imports", + "confidence": "EXTRACTED", + "source_file": "a.py", + "source_location": "L2", + "context": "import", + }, + { + "source": "a", + "target": "b", + "relation": "calls", + "confidence": "INFERRED", + "source_file": "a.py", + "source_location": "L3", + "context": "call", + }, + { + "source": "a", + "target": "b", + "relation": "calls", + "confidence": "EXTRACTED", + "source_file": "a.py", + "source_location": "L1", + "context": "call", + }, + { + "source": "a", + "target": "missing", + "relation": "calls", + "confidence": "EXTRACTED", + "source_file": "a.py", + }, + { + "source": "a", + "relation": "calls", + "confidence": "EXTRACTED", + "source_file": "a.py", + }, + { + "source": "c", + "target": "c", + "relation": "references", + "confidence": "EXTRACTED", + "source_file": "c.py", + }, + ], + } + + +def test_diagnose_extraction_categorizes_same_endpoint_collapse() -> None: + summary = diagnose_extraction(_diagnostic_fixture(), directed=True) + + assert summary["node_count"] == 3 + assert summary["raw_edge_count"] == 7 + assert summary["valid_candidate_edges"] == 5 + assert summary["missing_endpoint_edges"] == 1 + assert summary["dangling_endpoint_edges"] == 1 + assert summary["self_loop_edges"] == 1 + assert summary["exact_duplicate_edges"] == 1 + assert summary["directed_unique_endpoint_pairs"] == 2 + assert summary["directed_same_endpoint_collapsed_edges"] == 3 + assert summary["same_endpoint_group_count"] == 1 + assert summary["relation_variant_groups"] == 1 + assert summary["source_location_variant_groups"] == 1 + assert summary["post_build_graph_type"] == "DiGraph" + assert summary["post_build_edge_count"] == 2 + + +def test_diagnose_extraction_accepts_node_link_links_key() -> None: + extraction = _diagnostic_fixture() + extraction["links"] = extraction.pop("edges") + + summary = diagnose_extraction(extraction, directed=True) + + assert summary["raw_edge_count"] == 7 + assert summary["directed_same_endpoint_collapsed_edges"] == 3 + + +def test_diagnose_extraction_does_not_mutate_input() -> None: + extraction = _diagnostic_fixture() + original = deepcopy(extraction) + + diagnose_extraction(extraction, directed=True) + + assert extraction == original + + +def test_diagnose_extraction_handles_malformed_shapes_without_crashing() -> None: + extraction = { + "nodes": [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, + ["not", "a", "node"], + {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, + ], + "edges": [ + None, + ["not", "an", "edge"], + {"from": "a", "to": "b", "relation": "legacy_from_to"}, + {"source": "a", "target": {"unhashable": "target"}, "relation": "bad-target"}, + {"source": "a", "target": "missing", "relation": "dangling"}, + {"source": "", "target": "b", "relation": "missing-source"}, + ], + } + + summary = diagnose_extraction(extraction, directed=True) + + assert summary["node_count"] == 2 + assert summary["raw_edge_count"] == 6 + assert summary["non_object_edges"] == 2 + assert summary["missing_endpoint_edges"] == 1 + assert summary["dangling_endpoint_edges"] == 2 + assert summary["valid_candidate_edges"] == 1 + assert summary["post_build_error"].startswith("TypeError:") + + +def test_diagnose_extraction_handles_non_list_nodes_and_edges() -> None: + summary = diagnose_extraction( + {"nodes": {"id": "a"}, "edges": {"source": "a", "target": "b"}}, + directed=True, + ) + + assert summary["node_count"] == 0 + assert summary["raw_edge_count"] == 0 + assert summary["valid_candidate_edges"] == 0 + + +def test_diagnose_extraction_bounds_examples() -> None: + summary = diagnose_extraction(_diagnostic_fixture(), directed=True, max_examples=0) + + assert summary["directed_same_endpoint_collapsed_edges"] == 3 + assert summary["examples"] == [] + + +def test_diagnose_extraction_stops_examples_at_requested_limit() -> None: + extraction = _diagnostic_fixture() + extraction["nodes"].append( + {"id": "d", "label": "D", "file_type": "code", "source_file": "d.py"} + ) + extraction["edges"].extend( + [ + {"source": "b", "target": "d", "relation": "imports", "source_file": "b.py"}, + {"source": "b", "target": "d", "relation": "calls", "source_file": "b.py"}, + ] + ) + + summary = diagnose_extraction(extraction, directed=True, max_examples=1) + + assert summary["same_endpoint_group_count"] == 2 + assert len(summary["examples"]) == 1 + + +def test_diagnose_extraction_defaults_raw_inputs_to_directed(tmp_path: Path) -> None: + graph_path = tmp_path / "raw-extraction.json" + graph_path.write_text(json.dumps(_diagnostic_fixture()), encoding="utf-8") + + summary = diagnose_file(graph_path) + + assert summary["effective_directed"] is True + assert summary["post_build_graph_type"] == "DiGraph" + + +def test_diagnose_file_reads_json_and_formats_report(tmp_path: Path) -> None: + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(_diagnostic_fixture()), encoding="utf-8") + + summary = diagnose_file(graph_path, directed=True, max_examples=2) + report = format_diagnostic_report(summary) + + assert summary["input_path"] == str(graph_path) + assert "[graphify] MultiDiGraph edge-collapse diagnostic" in report + assert "directed_same_endpoint_collapsed_edges: 3" in report + assert "relation_variant_groups: 1" in report + assert "producer_suppression_sites:" in report + assert "examples:" in report + assert "a -> b" in report + + +def test_format_diagnostic_report_includes_build_and_suppression_errors( + tmp_path: Path, +) -> None: + summary = diagnose_extraction( + { + "nodes": [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, + ["not", "a", "node"], + ], + "edges": [], + }, + extract_path=tmp_path / "missing-extract.py", + ) + + report = format_diagnostic_report(summary) + + assert "post_build_error: TypeError:" in report + assert "producer_suppression_error: file not found" in report + + +def test_diagnostic_json_report_is_serializable(tmp_path: Path) -> None: + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(_diagnostic_fixture()), encoding="utf-8") + + summary = diagnose_file(graph_path, directed=True) + payload = format_diagnostic_json(summary) + + assert payload["schema_version"] == 1 + assert payload["summary"]["raw_edge_count"] == 7 + assert "producer_suppression" in payload + json.dumps(payload) + + +def test_scan_producer_suppression_sites_finds_seen_sets(tmp_path: Path) -> None: + source = tmp_path / "extract.py" + source.write_text( + "\n".join( + [ + "seen_call_pairs: set[tuple[str, str]] = set()", + "seen_static_ref_pairs: set[tuple[str, str, str]] = set()", + "other = set()", + ] + ), + encoding="utf-8", + ) + + result = scan_producer_suppression_sites(source) + + assert result["total_sites"] == 2 + assert result["sites"][0]["name"] == "seen_call_pairs" + assert result["sites"][0]["tuple_arity"] == 2 + assert result["sites"][1]["tuple_arity"] == 3 + + +def test_scan_producer_suppression_sites_handles_unknown_tuple_arity(tmp_path: Path) -> None: + source = tmp_path / "extract.py" + source.write_text("seen_blank: set[tuple[ ]] = set()\n", encoding="utf-8") + + result = scan_producer_suppression_sites(source) + + assert result["total_sites"] == 1 + assert result["sites"][0]["tuple_arity"] == 0 + + +def test_diagnose_file_rejects_oversized_graph(monkeypatch, tmp_path: Path) -> None: + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(_diagnostic_fixture()), encoding="utf-8") + monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 16) + + with pytest.raises(ValueError, match="exceeds"): + diagnose_file(graph_path) + + +def test_diagnose_file_rejects_non_object_json(tmp_path: Path) -> None: + graph_path = tmp_path / "graph.json" + graph_path.write_text("[]", encoding="utf-8") + + with pytest.raises(ValueError, match="JSON object"): + diagnose_file(graph_path) + + +def test_diagnose_file_defaults_to_json_directed_flag(tmp_path: Path) -> None: + graph_path = tmp_path / "graph.json" + payload = _diagnostic_fixture() + payload["directed"] = False + graph_path.write_text(json.dumps(payload), encoding="utf-8") + + summary = diagnose_file(graph_path) + + assert summary["effective_directed"] is False + assert summary["post_build_graph_type"] == "Graph" + + +def test_diagnose_file_explicit_directed_override(tmp_path: Path) -> None: + graph_path = tmp_path / "graph.json" + payload = _diagnostic_fixture() + payload["directed"] = False + graph_path.write_text(json.dumps(payload), encoding="utf-8") + + summary = diagnose_file(graph_path, directed=True) + + assert summary["effective_directed"] is True + assert summary["post_build_graph_type"] == "DiGraph" + + +def test_scan_producer_suppression_sites_reports_missing_file(tmp_path: Path) -> None: + result = scan_producer_suppression_sites(tmp_path / "missing-extract.py") + + assert result["total_sites"] == 0 + assert result["sites"] == [] + assert result["error"] == "file not found" + + +def test_diagnose_multigraph_cli_human_output(monkeypatch, tmp_path: Path, capsys) -> None: + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(_diagnostic_fixture()), encoding="utf-8") + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "diagnose", "multigraph", "--graph", str(graph_path)], + ) + + mainmod.main() + + out = capsys.readouterr().out + assert "[graphify] MultiDiGraph edge-collapse diagnostic" in out + assert "raw_edges: 7" in out + assert "effective_directed: True" in out + assert "directed_same_endpoint_collapsed_edges: 3" in out + + +def test_diagnose_multigraph_cli_undirected_override(monkeypatch, tmp_path: Path, capsys) -> None: + graph_path = tmp_path / "graph.json" + payload = _diagnostic_fixture() + payload["directed"] = True + graph_path.write_text(json.dumps(payload), encoding="utf-8") + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "diagnose", "multigraph", "--graph", str(graph_path), "--undirected"], + ) + + mainmod.main() + + out = capsys.readouterr().out + assert "effective_directed: False" in out + assert "post_build_graph_type: Graph" in out + + +def test_diagnose_multigraph_cli_max_examples_zero(monkeypatch, tmp_path: Path, capsys) -> None: + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(_diagnostic_fixture()), encoding="utf-8") + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + [ + "graphify", + "diagnose", + "multigraph", + "--graph", + str(graph_path), + "--max-examples", + "0", + ], + ) + + mainmod.main() + + assert "\nexamples:" not in capsys.readouterr().out + + +def test_diagnose_multigraph_cli_json_output(monkeypatch, tmp_path: Path, capsys) -> None: + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(_diagnostic_fixture()), encoding="utf-8") + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "diagnose", "multigraph", "--graph", str(graph_path), "--json"], + ) + + mainmod.main() + + payload = json.loads(capsys.readouterr().out) + assert payload["schema_version"] == 1 + assert payload["summary"]["directed_same_endpoint_collapsed_edges"] == 3 + + +@pytest.mark.parametrize( + ("argv_tail", "expected"), + [ + ([], "Usage: graphify diagnose multigraph"), + (["wrong"], "Usage: graphify diagnose multigraph"), + (["multigraph", "--graph"], "error: --graph requires a path"), + (["multigraph", "--max-examples"], "error: --max-examples requires an integer"), + (["multigraph", "--max-examples", "many"], "error: --max-examples requires an integer"), + (["multigraph", "--max-examples", "-1"], "error: --max-examples must be >= 0"), + (["multigraph", "--unknown"], "error: unknown diagnose option --unknown"), + ], +) +def test_diagnose_multigraph_cli_usage_errors( + monkeypatch, + capsys, + argv_tail: list[str], + expected: str, +) -> None: + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", ["graphify", "diagnose", *argv_tail]) + + with pytest.raises(SystemExit) as exc_info: + mainmod.main() + + assert exc_info.value.code == 1 + assert expected in capsys.readouterr().err + + +def test_diagnose_multigraph_cli_rejects_conflicting_direction_flags( + monkeypatch, + tmp_path: Path, + capsys, +) -> None: + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(_diagnostic_fixture()), encoding="utf-8") + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + [ + "graphify", + "diagnose", + "multigraph", + "--graph", + str(graph_path), + "--directed", + "--undirected", + ], + ) + + with pytest.raises(SystemExit) as exc_info: + mainmod.main() + + assert exc_info.value.code == 1 + assert "--directed and --undirected are mutually exclusive" in capsys.readouterr().err diff --git a/tests/test_query_cli.py b/tests/test_query_cli.py index 39d016f..cf8eb6e 100644 --- a/tests/test_query_cli.py +++ b/tests/test_query_cli.py @@ -49,3 +49,22 @@ def test_query_cli_heuristic_context_filter(monkeypatch, tmp_path, capsys): assert "Context: call (heuristic)" in out assert "cluster" in out assert "build" not in out + + +def test_query_cli_rejects_oversized_graph(monkeypatch, tmp_path, capsys): + """#F4: query CLI must refuse to parse a graph.json that exceeds the cap.""" + import pytest + + graph_path = _write_graph(tmp_path) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 16) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "query", "extract", "--graph", str(graph_path)], + ) + with pytest.raises(SystemExit): + mainmod.main() + err = capsys.readouterr().err + assert "exceeds" in err + assert "byte cap" in err diff --git a/tests/test_scip_ingest.py b/tests/test_scip_ingest.py new file mode 100644 index 0000000..7129de2 --- /dev/null +++ b/tests/test_scip_ingest.py @@ -0,0 +1,1670 @@ +"""Comprehensive tests for graphify.scip_ingest.""" + +from __future__ import annotations + +import pytest + +from graphify.scip_ingest import ( + _build_scip_metadata, + _make_scip_node_id, + _scip_kind_to_file_type, + ingest_scip_json, +) + + +# --------------------------------------------------------------------------- +# Valid JSON parsing — full-document smoke tests +# --------------------------------------------------------------------------- + + +def test_ingest_empty_doc_returns_empty_lists() -> None: + """Empty dict input produces empty nodes and edges.""" + result = ingest_scip_json({}) + assert result == {"nodes": [], "edges": []} + + +def test_ingest_dict_without_documents_key() -> None: + """documents key not present → no processing → empty result.""" + result = ingest_scip_json({"metadata": "some meta"}) + assert result == {"nodes": [], "edges": []} + + +def test_ingest_documents_not_a_list_is_skipped() -> None: + """When documents is not a list, ingestion stops and returns empty.""" + result = ingest_scip_json({"documents": "not_a_list"}) + assert result == {"nodes": [], "edges": []} + + +def test_ingest_documents_empty_list() -> None: + """Empty documents list produces empty nodes and edges.""" + result = ingest_scip_json({"documents": []}) + assert result == {"nodes": [], "edges": []} + + +def test_ingest_single_symbol_no_relationships() -> None: + """A single symbol with no relationships yields one node and zero edges.""" + doc = { + "documents": [ + { + "relative_path": "src/main.py", + "language": "python", + "symbols": [ + { + "symbol": "python/main.py:MainClass#", + "kind": "class", + "display_name": "MainClass", + "documentation": ["The main class"], + "relationships": [], + "occurrences": [ + {"range": [5, 0, 5, 9], "symbol": "python/main.py:MainClass#"} + ], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + assert len(result["edges"]) == 0 + + node = result["nodes"][0] + assert node["label"] == "MainClass" + assert node["file_type"] == "code" + assert node["source_file"] == "src/main.py" + assert node["source_location"] == "L5" + assert node["metadata"]["scip_symbol"] == "python/main.py:MainClass#" + assert node["metadata"]["scip_kind"] == "class" + assert node["metadata"]["scip_description"] == "The main class" + + +def test_ingest_symbol_without_display_name_uses_suffix() -> None: + """When display_name is missing, label falls back to the portion after #.""" + doc = { + "documents": [ + { + "relative_path": "lib/helper.py", + "symbols": [ + { + "symbol": "python/helper.py:compute#run()", + "kind": "function", + "occurrences": [], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["label"] == "run()" + + +def test_ingest_symbol_trailing_hash_no_display_name_has_non_empty_label() -> None: + """Symbol ending with '#' and no display_name must produce a non-empty label. + + symbol.split('#')[-1] is '' when the symbol ends with '#', so + label = display_name or suffix evaluates to '' when display_name is also + absent. The fix must fall back to the full symbol_id. + """ + doc = { + "documents": [ + { + "relative_path": "src/Foo.java", + "symbols": [ + { + "symbol": "java/src/Foo.java:Foo#", + "kind": "class", + "occurrences": [], + "relationships": [], + # no display_name + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + assert result["nodes"][0]["label"], ( + "label must not be empty when symbol ends with '#' and display_name is absent" + ) + + +def test_ingest_symbol_without_hash_uses_full_symbol_as_label() -> None: + """When symbol has no #, the label is the full symbol id.""" + doc = { + "documents": [ + { + "relative_path": "lib/helper.py", + "symbols": [ + { + "symbol": "SimpleFunction", + "kind": "function", + "occurrences": [], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["label"] == "SimpleFunction" + + +def test_ingest_symbol_without_occurrences_has_empty_source_location() -> None: + """When occurrences list is empty, source_location is empty string.""" + doc = { + "documents": [ + { + "relative_path": "lib/a.py", + "symbols": [ + { + "symbol": "python/lib/a.py:Foo#", + "kind": "class", + "occurrences": [], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["source_location"] == "" + + +def test_ingest_symbol_without_occurrences_key() -> None: + """When occurrences key is missing entirely, falls back to empty source_location.""" + doc = { + "documents": [ + { + "relative_path": "lib/a.py", + "symbols": [ + { + "symbol": "python/lib/a.py:Foo#", + "kind": "class", + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["source_location"] == "" + + +def test_ingest_multiple_symbols_in_one_document() -> None: + """Multiple symbols in a single document all become nodes.""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "python/mod.py:A#", + "kind": "class", + "display_name": "A", + "occurrences": [], + "relationships": [], + }, + { + "symbol": "python/mod.py:B#", + "kind": "function", + "display_name": "B", + "occurrences": [], + "relationships": [], + }, + { + "symbol": "python/mod.py:C#", + "kind": "variable", + "display_name": "C", + "occurrences": [], + "relationships": [], + }, + ], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 3 + labels = {n["label"] for n in result["nodes"]} + assert labels == {"A", "B", "C"} + + +def test_ingest_multiple_documents() -> None: + """Symbols from multiple documents all become nodes.""" + doc = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [ + {"symbol": "A#", "kind": "class", "occurrences": [], "relationships": []}, + ], + }, + { + "relative_path": "b.py", + "symbols": [ + {"symbol": "B#", "kind": "function", "occurrences": [], "relationships": []}, + ], + }, + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 2 + + paths = {n["source_file"] for n in result["nodes"]} + assert paths == {"a.py", "b.py"} + + +# --------------------------------------------------------------------------- +# Reference/definition resolution — relationship → edge mapping +# --------------------------------------------------------------------------- + + +def _make_symbol_doc(symbol_id: str, kind: str, rels: list[object]) -> dict[str, object]: + """Helper to build a minimal SCIP document with one symbol.""" + return { + "documents": [ + { + "relative_path": "src/main.py", + "symbols": [ + { + "symbol": symbol_id, + "kind": kind, + "display_name": symbol_id.split("#")[-1].strip("()"), + "occurrences": [{"range": [10, 0, 10, 20], "symbol": symbol_id}], + "relationships": rels, + } + ], + } + ] + } + + +def test_ingest_is_reference_emits_scip_ref_edge() -> None: + """is_reference → relation 'scip_ref'.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [{"symbol": "python/main.py:Helper#help()", "is_reference": True}], + ) + result = ingest_scip_json(doc) + assert len(result["edges"]) == 1 + assert result["edges"][0]["relation"] == "scip_ref" + + +def test_ingest_is_definition_emits_scip_def_edge() -> None: + """is_definition → relation 'scip_def'.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [{"symbol": "python/main.py:Base#run()", "is_definition": True}], + ) + result = ingest_scip_json(doc) + assert result["edges"][0]["relation"] == "scip_def" + + +def test_ingest_is_implementation_emits_scip_impl_edge() -> None: + """is_implementation → relation 'scip_impl' (takes priority over is_definition).""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [{"symbol": "python/main.py:Base#run()", "is_implementation": True, "is_definition": True}], + ) + result = ingest_scip_json(doc) + assert result["edges"][0]["relation"] == "scip_impl" + + +def test_ingest_is_type_definition_emits_scip_typed_edge() -> None: + """is_type_definition → relation 'scip_typed'.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [{"symbol": "python/main.py:Base#run()", "is_type_definition": True}], + ) + result = ingest_scip_json(doc) + assert result["edges"][0]["relation"] == "scip_typed" + + +def test_ingest_relationship_priority_order() -> None: + """Implementation > TypeDefinition > Definition > Reference.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [ + { + "symbol": "python/main.py:Base#run()", + "is_implementation": True, + "is_type_definition": True, + "is_definition": True, + "is_reference": True, + } + ], + ) + result = ingest_scip_json(doc) + assert result["edges"][0]["relation"] == "scip_impl" + + +def test_ingest_relationship_no_boolean_flags_defaults_to_ref() -> None: + """When none of is_* flags are set, relation defaults to 'scip_ref'.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [{"symbol": "python/main.py:Other#"}], + ) + result = ingest_scip_json(doc) + assert result["edges"][0]["relation"] == "scip_ref" + + +def test_ingest_multiple_relationships_on_one_symbol() -> None: + """A symbol with multiple relationships emits one edge per relationship.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [ + {"symbol": "python/main.py:Base#run()", "is_definition": True}, + {"symbol": "python/main.py:Helper#help()", "is_reference": True}, + ], + ) + result = ingest_scip_json(doc) + assert len(result["edges"]) == 2 + relations = {e["relation"] for e in result["edges"]} + assert relations == {"scip_def", "scip_ref"} + + +def test_ingest_relationship_without_target_symbol_is_skipped() -> None: + """Relationship with empty or missing symbol field is ignored.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [ + {"symbol": "", "is_reference": True}, + {"is_reference": True}, + ], + ) + result = ingest_scip_json(doc) + assert len(result["edges"]) == 0 + + +def test_ingest_duplicate_edges_are_deduplicated() -> None: + """The same source→target→relation→location edge is only emitted once.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [ + {"symbol": "python/main.py:Helper#help()", "is_reference": True}, + {"symbol": "python/main.py:Helper#help()", "is_reference": True}, + ], + ) + result = ingest_scip_json(doc) + assert len(result["edges"]) == 1 + + +# --------------------------------------------------------------------------- +# Edge emission — edge dict structure +# --------------------------------------------------------------------------- + + +def test_ingest_edge_structure_complete() -> None: + """Verify every field in the emitted edge dict.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [{"symbol": "python/main.py:Helper#help()", "is_reference": True}], + ) + result = ingest_scip_json(doc) + edge = result["edges"][0] + assert edge["confidence"] == "EXTRACTED" + assert edge["confidence_score"] == 1.0 + assert edge["weight"] == 1.0 + assert edge["context"] == "scip" + assert edge["source_file"] == "src/main.py" + assert edge["source_location"] == "L10" + assert "scip_relationship" in edge["metadata"] + + +def test_ingest_edge_source_location_from_first_occurrence() -> None: + """source_location on edges uses the line from the first occurrence range[0].""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "python/mod.py:Foo#bar()", + "kind": "function", + "occurrences": [ + {"range": [42, 0, 42, 10], "symbol": "python/mod.py:Foo#bar()"}, + {"range": [99, 0, 99, 10], "symbol": "python/mod.py:Foo#bar()"}, + ], + "relationships": [{"symbol": "python/mod.py:Baz#", "is_reference": True}], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["edges"][0]["source_location"] == "L42" + assert result["nodes"][0]["source_location"] == "L42" + + +def test_ingest_node_id_contains_source_file_and_symbol_suffix() -> None: + """Node id is derived from source_file and symbol suffix.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [], + ) + result = ingest_scip_json(doc) + node_id = result["nodes"][0]["id"] + # Should start with scip_ and contain the suffix + assert node_id.startswith("scip_") + assert "run" in node_id + + +def test_ingest_node_id_is_deterministic() -> None: + """Same input produces the same node id.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [], + ) + result1 = ingest_scip_json(doc) + result2 = ingest_scip_json(doc) + assert result1["nodes"][0]["id"] == result2["nodes"][0]["id"] + + +def test_ingest_node_id_differs_by_source_file() -> None: + """Same symbol in different files produces different node ids.""" + doc1 = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [ + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []} + ], + } + ] + } + doc2 = { + "documents": [ + { + "relative_path": "b.py", + "symbols": [ + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []} + ], + } + ] + } + id1 = ingest_scip_json(doc1)["nodes"][0]["id"] + id2 = ingest_scip_json(doc2)["nodes"][0]["id"] + assert id1 != id2 + + +def test_ingest_duplicate_symbols_in_same_file_are_deduplicated() -> None: + """The same symbol appearing twice in a document yields only one node.""" + doc = { + "documents": [ + { + "relative_path": "src/main.py", + "symbols": [ + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []}, + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []}, + ], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + + +# --------------------------------------------------------------------------- +# Invalid JSON / non-dict input +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "bad_input", + [ + None, + "a string", + 42, + 3.14, + True, + [], + [1, 2, 3], + ], +) +def test_ingest_non_dict_input_returns_empty(bad_input: object) -> None: + """Non-dict inputs are guarded and return empty nodes/edges.""" + result = ingest_scip_json(bad_input) + assert result == {"nodes": [], "edges": []} + + +def test_ingest_document_item_not_a_dict_is_skipped() -> None: + """Non-dict entries in the documents list are silently skipped.""" + doc = { + "documents": [ + "not_a_dict", + 123, + None, + { + "relative_path": "valid.py", + "symbols": [ + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []} + ], + }, + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + + +def test_ingest_symbol_item_not_a_dict_is_skipped() -> None: + """Non-dict entries in the symbols list are silently skipped.""" + doc = { + "documents": [ + { + "relative_path": "src/main.py", + "symbols": [ + "not_a_dict", + 42, + None, + { + "symbol": "python/main.py:Valid#", + "kind": "class", + "display_name": "Valid", + "occurrences": [], + "relationships": [], + }, + ], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + assert result["nodes"][0]["label"] == "Valid" + + +def test_ingest_symbol_without_symbol_id_is_skipped() -> None: + """A symbol dict with empty or missing 'symbol' field produces no node.""" + doc = { + "documents": [ + { + "relative_path": "src/main.py", + "symbols": [ + {"kind": "class", "occurrences": [], "relationships": []}, + {"symbol": "", "kind": "class", "occurrences": [], "relationships": []}, + ], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 0 + + +def test_ingest_relationship_item_not_a_dict_is_skipped() -> None: + """Non-dict entries in the relationships list are silently skipped.""" + doc = _make_symbol_doc( + "python/main.py:MyClass#run()", + "function", + [ + "not_a_dict", + 42, + None, + {"symbol": "python/main.py:Helper#help()", "is_reference": True}, + ], + ) + result = ingest_scip_json(doc) + assert len(result["edges"]) == 1 + + +# --------------------------------------------------------------------------- +# Empty documents / missing keys +# --------------------------------------------------------------------------- + + +def test_ingest_document_without_symbols_key() -> None: + """Document dict without 'symbols' key is treated as empty list.""" + doc = {"documents": [{"relative_path": "src/main.py", "language": "python"}]} + result = ingest_scip_json(doc) + assert result == {"nodes": [], "edges": []} + + +def test_ingest_document_with_symbols_not_a_list() -> None: + """When symbols is not a list, that document is skipped.""" + doc = {"documents": [{"relative_path": "src/main.py", "symbols": "not_a_list"}]} + result = ingest_scip_json(doc) + assert result == {"nodes": [], "edges": []} + + +def test_ingest_symbol_without_kind_defaults_to_unknown() -> None: + """When kind is missing, metadata uses 'unknown'.""" + doc = { + "documents": [ + { + "relative_path": "src/main.py", + "symbols": [{"symbol": "F#", "occurrences": [], "relationships": []}], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["metadata"]["scip_kind"] == "unknown" + + +# --------------------------------------------------------------------------- +# Path validation / edge cases +# --------------------------------------------------------------------------- + + +def test_ingest_default_source_file_is_empty_string() -> None: + """When no relative_path is given on document, source_file defaults to ''.""" + doc = { + "documents": [ + { + "symbols": [ + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []} + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["source_file"] == "" + + +def test_ingest_source_file_falls_back_to_function_param() -> None: + """The source_file param provides a fallback when doc has no relative_path.""" + doc = { + "documents": [ + { + "symbols": [ + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []} + ], + } + ] + } + result = ingest_scip_json(doc, source_file="fallback.scip") + assert result["nodes"][0]["source_file"] == "fallback.scip" + + +def test_ingest_document_relative_path_overrides_source_file_param() -> None: + """Document relative_path takes precedence over the source_file parameter.""" + doc = { + "documents": [ + { + "relative_path": "explicit.py", + "symbols": [ + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []} + ], + } + ] + } + result = ingest_scip_json(doc, source_file="fallback.scip") + assert result["nodes"][0]["source_file"] == "explicit.py" + + +def test_ingest_document_without_language_defaults_to_function_param() -> None: + """When doc has no language field, uses the language function parameter.""" + doc = { + "documents": [ + { + "relative_path": "src/main.ts", + "symbols": [ + {"symbol": "F#", "kind": "class", "occurrences": [], "relationships": []} + ], + } + ] + } + result = ingest_scip_json(doc, language="typescript") + # language is passed to _ingest_symbol but not directly exposed on nodes. + # Verify that the node was still created (language defaults don't break ingestion). + assert len(result["nodes"]) == 1 + + +def test_ingest_symbol_with_short_range_uses_first_element_as_line() -> None: + """A range list with exactly 2 elements (minimum required) sets sourceline from range[0].""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "python/mod.py:F#", + "kind": "class", + "occurrences": [{"range": [7, 0], "symbol": "python/mod.py:F#"}], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["source_location"] == "L7" + + +def test_ingest_symbol_with_non_dict_occurrence_is_skipped() -> None: + """Only the first occurrence is used; if it is not a dict, sourceline stays 0.""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "python/mod.py:F#", + "kind": "class", + "occurrences": [ + "bad", + 123, + None, + {"range": [15, 0, 15, 5], "symbol": "python/mod.py:F#"}, + ], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + # The first occurrence "bad" is not a dict → range parsing skipped → source_location stays empty + assert result["nodes"][0]["source_location"] == "" + + +def test_ingest_symbol_with_non_list_range_falls_back_to_zero() -> None: + """When range is not a list, sourceline stays 0 (empty source_location).""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "F#", + "kind": "class", + "occurrences": [{"range": "not_a_list", "symbol": "F#"}], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["source_location"] == "" + + +def test_ingest_symbol_with_documentation_becomes_description() -> None: + """The first element of documentation[] becomes scip_description metadata.""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "F#", + "kind": "class", + "documentation": ["First line", "Second line"], + "occurrences": [], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["metadata"]["scip_description"] == "First line" + + +def test_ingest_symbol_with_empty_documentation_skips_description() -> None: + """When documentation[0] is empty string, scip_description is omitted.""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "F#", + "kind": "class", + "documentation": [""], + "occurrences": [], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert "scip_description" not in result["nodes"][0]["metadata"] + + +def test_ingest_symbol_without_documentation_omits_description() -> None: + """When documentation key is missing, scip_description is not in metadata.""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "F#", + "kind": "class", + "occurrences": [], + "relationships": [], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert "scip_description" not in result["nodes"][0]["metadata"] + + +def test_ingest_symbol_without_relationships_key_still_creates_node() -> None: + """Missing relationships key — symbol still becomes a node.""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [{"symbol": "F#", "kind": "class", "occurrences": []}], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + assert len(result["edges"]) == 0 + + +# --------------------------------------------------------------------------- +# _make_scip_node_id — node id generation +# --------------------------------------------------------------------------- + + +def test_make_scip_node_id_with_hash_separator() -> None: + """Symbol with # uses suffix after last #.""" + node_id = _make_scip_node_id("python/main.py:MyClass#run()", "src/main.py") + assert node_id.startswith("scip_") + assert "run" in node_id + # Should NOT contain raw parentheses + assert "(" not in node_id + assert ")" not in node_id + + +def test_make_scip_node_id_without_hash() -> None: + """Symbol without # uses the full symbol (sanitised) as suffix.""" + node_id = _make_scip_node_id("SimpleSymbol", "src/mod.py") + assert node_id.startswith("scip_") + assert "simplesymbol" in node_id.lower() + + +def test_make_scip_node_id_special_characters_are_sanitised() -> None: + """Non-alphanumeric characters are replaced with underscores.""" + node_id = _make_scip_node_id("foo.bar#baz!@qux", "test.py") + # Everything after last # becomes: baz!@qux → baz__qux + assert "scip_baz__qux" in node_id + + +def test_make_scip_node_id_deterministic() -> None: + """Same inputs always produce the same id.""" + a = _make_scip_node_id("python/main.py:Foo#bar", "src/a.py") + b = _make_scip_node_id("python/main.py:Foo#bar", "src/a.py") + assert a == b + + +def test_make_scip_node_id_source_file_affects_hash() -> None: + """Different source_file produces different hash.""" + a = _make_scip_node_id("F#", "a.py") + b = _make_scip_node_id("F#", "b.py") + assert a != b + + +def test_make_scip_node_id_symbol_affects_hash() -> None: + """Different symbol produces different hash.""" + a = _make_scip_node_id("A#", "f.py") + b = _make_scip_node_id("B#", "f.py") + assert a != b + + +def test_make_scip_node_id_empty_after_sanitisation_falls_back() -> None: + """If sanitised suffix is empty, uses just the hash.""" + node_id = _make_scip_node_id("#", "src/f.py") + # The suffix after # is empty string, so node_id should be scip_ + assert node_id.startswith("scip_") + # Verify it's just scip_ + 12 hex chars + import re + + assert re.match(r"^scip_[0-9a-f]{12}$", node_id) + + +# --------------------------------------------------------------------------- +# _scip_kind_to_file_type — always returns "code" +# --------------------------------------------------------------------------- + + +def test_scip_kind_to_file_type_always_code() -> None: + """Any kind string maps to 'code'.""" + assert _scip_kind_to_file_type("class") == "code" + assert _scip_kind_to_file_type("function") == "code" + assert _scip_kind_to_file_type("variable") == "code" + assert _scip_kind_to_file_type("") == "code" + assert _scip_kind_to_file_type("arbitrary_string") == "code" + + +# --------------------------------------------------------------------------- +# _build_scip_metadata — metadata dict construction +# --------------------------------------------------------------------------- + + +def test_build_scip_metadata_with_description() -> None: + """All three fields present when description is non-empty.""" + meta = _build_scip_metadata("sym_id", "class", "A sample description") + assert meta == { + "scip_symbol": "sym_id", + "scip_kind": "class", + "scip_description": "A sample description", + } + + +def test_build_scip_metadata_without_description() -> None: + """scip_description is omitted when description is empty string.""" + meta = _build_scip_metadata("sym_id", "class", "") + assert meta == { + "scip_symbol": "sym_id", + "scip_kind": "class", + } + assert "scip_description" not in meta + + +# --------------------------------------------------------------------------- +# Edge-case: very large symbol count +# --------------------------------------------------------------------------- + + +def test_ingest_many_symbols() -> None: + """Ingestion handles a large number of symbols gracefully.""" + symbols = [ + {"symbol": f"S{i}#", "kind": "class", "occurrences": [], "relationships": []} + for i in range(100) + ] + doc = {"documents": [{"relative_path": "big.py", "symbols": symbols}]} + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 100 + assert len(result["edges"]) == 0 + + +# --------------------------------------------------------------------------- +# Edge-case: relationship with missing source_location (line 0) +# --------------------------------------------------------------------------- + + +def test_ingest_edge_with_zero_sourceline_has_empty_location() -> None: + """When sourceline is 0, source_location on edge is empty string.""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "A#", + "kind": "class", + "occurrences": [], # no occurrences → sourceline 0 + "relationships": [{"symbol": "B#", "is_reference": True}], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["edges"][0]["source_location"] == "" + + +# --------------------------------------------------------------------------- +# Cycle 2.4 v2: endpoint-safe edges + build_from_json round-trip (F1) +# --------------------------------------------------------------------------- + + +def test_relationship_target_in_same_document_resolves_via_index(): + """Cross-symbol relationship within ONE document resolves via the symbol index.""" + doc = { + "documents": [ + { + "relative_path": "src/mod.py", + "symbols": [ + { + "symbol": "Caller#", + "kind": "function", + "relationships": [{"symbol": "Callee#", "is_reference": True}], + }, + {"symbol": "Callee#", "kind": "function"}, + ], + } + ] + } + result = ingest_scip_json(doc) + ids = {n["id"] for n in result["nodes"]} + assert len(result["edges"]) == 1 + edge = result["edges"][0] + # Both endpoints exist in nodes + assert edge["source"] in ids + assert edge["target"] in ids + + +def test_relationship_target_across_documents_resolves_via_index(): + """Cross-document relationship resolves to the target document's node id.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [ + { + "symbol": "Caller#", + "kind": "function", + "relationships": [{"symbol": "Callee#", "is_reference": True}], + }, + ], + }, + { + "relative_path": "src/b.py", + "symbols": [{"symbol": "Callee#", "kind": "function"}], + }, + ] + } + result = ingest_scip_json(doc) + by_symbol = {n["metadata"]["scip_symbol"]: n["id"] for n in result["nodes"]} + assert "Caller#" in by_symbol + assert "Callee#" in by_symbol + edge = result["edges"][0] + assert edge["source"] == by_symbol["Caller#"] + assert edge["target"] == by_symbol["Callee#"] + # The target node was emitted with src/b.py as source_file (its real home) + callee_node = next(n for n in result["nodes"] if n["id"] == by_symbol["Callee#"]) + assert callee_node["source_file"] == "src/b.py" + + +def test_relationship_target_unknown_emits_stub_node(): + """A relationship targeting a symbol NOT in any document creates a stub external node.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [ + { + "symbol": "Caller#", + "kind": "function", + "relationships": [{"symbol": "ExternalLib#fn", "is_reference": True}], + }, + ], + } + ] + } + result = ingest_scip_json(doc) + by_symbol = {n["metadata"]["scip_symbol"]: n for n in result["nodes"]} + assert "ExternalLib#fn" in by_symbol + stub = by_symbol["ExternalLib#fn"] + # Stub has scip_kind=external in metadata + assert stub["metadata"]["scip_kind"] == "external" + # Edge endpoints both resolve to existing nodes + ids = {n["id"] for n in result["nodes"]} + edge = result["edges"][0] + assert edge["source"] in ids + assert edge["target"] in ids + + +def test_relationship_edges_survive_validate_extraction_and_build(): + """Result passes Graphify's validate_extraction and build_from_json keeps the edges.""" + from graphify.build import build_from_json + from graphify.validate import validate_extraction + + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [ + { + "symbol": "Caller#", + "kind": "function", + "occurrences": [{"range": [10, 0, 10, 6]}], + "relationships": [ + {"symbol": "Callee#", "is_reference": True}, + {"symbol": "External#fn", "is_implementation": True}, + ], + }, + {"symbol": "Callee#", "kind": "function"}, + ], + } + ] + } + result = ingest_scip_json(doc) + errors = validate_extraction(result) + assert errors == [], f"validate_extraction failures: {errors}" + graph = build_from_json(result) + # Two edges should survive into the graph + edge_count = sum(1 for _ in graph.edges()) + assert edge_count == 2, f"expected 2 edges in graph, got {edge_count}" + + +# --------------------------------------------------------------------------- +# Cycle 2.4 v2: nested untrusted input guards (F2) +# --------------------------------------------------------------------------- + + +def test_non_string_relative_path_falls_back_to_default(): + """`relative_path` as a non-string falls back to the function's source_file default.""" + doc = { + "documents": [ + { + "relative_path": ["unexpected", "list"], + "symbols": [{"symbol": "Foo#", "kind": "function"}], + } + ] + } + result = ingest_scip_json(doc, source_file="fallback.py") + assert result["nodes"][0]["source_file"] == "fallback.py" + + +def test_non_string_language_falls_back(): + """`language` as a non-string falls back to the function default.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "language": 42, + "symbols": [{"symbol": "Foo#", "kind": "function"}], + } + ] + } + # Should not raise + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + + +def test_non_string_symbol_id_is_skipped(): + """A symbol entry with `symbol: ` is silently skipped.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [ + {"symbol": 123, "kind": "function"}, # invalid + {"symbol": "Valid#", "kind": "function"}, + ], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + assert result["nodes"][0]["metadata"]["scip_symbol"] == "Valid#" + + +def test_relationships_none_is_treated_as_empty(): + """A symbol with `relationships: None` ingests without error and emits no edges.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [{"symbol": "Foo#", "kind": "function", "relationships": None}], + } + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + assert result["edges"] == [] + + +def test_relationship_symbol_non_string_is_skipped(): + """A relationship entry whose `symbol` is a non-string is silently skipped.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [ + { + "symbol": "Foo#", + "kind": "function", + "relationships": [ + {"symbol": 123, "is_reference": True}, # invalid + {"symbol": "RealTarget#", "is_reference": True}, + ], + } + ], + } + ] + } + result = ingest_scip_json(doc) + # One real edge survives; the int-symbol relationship is dropped + assert len(result["edges"]) == 1 + assert result["edges"][0]["metadata"]["scip_relationship"]["symbol"] == "RealTarget#" + + +def test_non_string_kind_falls_back_to_unknown(): + """A symbol with `kind` as a non-string falls back to 'unknown'.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [{"symbol": "Foo#", "kind": ["not", "a", "string"]}], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["metadata"]["scip_kind"] == "unknown" + + +def test_non_string_display_name_falls_back(): + """`display_name` as a non-string falls back to the symbol suffix.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [{"symbol": "Foo#bar", "kind": "function", "display_name": 42}], + } + ] + } + result = ingest_scip_json(doc) + # Label falls back to the suffix after '#' + assert result["nodes"][0]["label"] == "bar" + + +def test_documentation_with_non_string_entries_is_ignored(): + """`documentation` first entry that isn't a string yields empty description (not crash).""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [{"symbol": "Foo#", "kind": "function", "documentation": [42, "later"]}], + } + ] + } + result = ingest_scip_json(doc) + # Only string first-elements become descriptions + assert "scip_description" not in result["nodes"][0]["metadata"] + + +def test_unrecognized_top_level_structure_returns_empty(): + """Top-level non-dict shapes still return the empty result.""" + assert ingest_scip_json("not a dict") == {"nodes": [], "edges": []} + assert ingest_scip_json([{"documents": []}]) == {"nodes": [], "edges": []} + assert ingest_scip_json(None) == {"nodes": [], "edges": []} + + +def test_documents_field_non_list_returns_empty(): + """`documents` as a non-list returns the empty result.""" + assert ingest_scip_json({"documents": "not a list"}) == {"nodes": [], "edges": []} + + +def test_document_entry_non_dict_is_skipped(): + """A non-dict entry in `documents` is silently skipped.""" + doc = { + "documents": [ + "not a dict", + {"relative_path": "src/a.py", "symbols": [{"symbol": "Foo#", "kind": "function"}]}, + ] + } + result = ingest_scip_json(doc) + assert len(result["nodes"]) == 1 + + +def test_occurrence_negative_line_falls_back_to_zero(): + """An occurrence with a negative line number resolves source_location to empty.""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [ + { + "symbol": "Foo#", + "kind": "function", + "occurrences": [{"range": [-1, 0, -1, 6]}], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["nodes"][0]["source_location"] == "" + + +# --------------------------------------------------------------------------- +# Cycle 2.4 v3: document-aware relationship resolution (F1) +# --------------------------------------------------------------------------- + + +def test_duplicate_local_symbol_resolves_to_same_document(): + """When two docs both have `F#`, a relationship from b.py's F# to F# must + resolve to b.py's own F# node, not a.py's.""" + doc = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [{"symbol": "F#", "kind": "function"}], + }, + { + "relative_path": "b.py", + "symbols": [ + { + "symbol": "F#", + "kind": "function", + "relationships": [{"symbol": "F#", "is_reference": True}], + } + ], + }, + ] + } + result = ingest_scip_json(doc) + # Find the two F# nodes + f_nodes = [n for n in result["nodes"] if n["metadata"]["scip_symbol"] == "F#"] + assert len(f_nodes) == 2 + b_f_node = next(n for n in f_nodes if n["source_file"] == "b.py") + a_f_node = next(n for n in f_nodes if n["source_file"] == "a.py") + assert b_f_node["id"] != a_f_node["id"] + # The edge: source must be b.py's F#, target must ALSO be b.py's F# (same-doc precedence) + assert len(result["edges"]) == 1 + edge = result["edges"][0] + assert edge["source"] == b_f_node["id"] + assert edge["target"] == b_f_node["id"] + + +def test_unique_cross_document_symbol_still_resolves(): + """When a target symbol is defined in exactly ONE other document, the edge + still routes to that document (unique-global rule).""" + doc = { + "documents": [ + { + "relative_path": "src/a.py", + "symbols": [ + { + "symbol": "Caller#", + "kind": "function", + "relationships": [{"symbol": "UniqueCallee#", "is_reference": True}], + }, + ], + }, + { + "relative_path": "src/b.py", + "symbols": [{"symbol": "UniqueCallee#", "kind": "function"}], + }, + ] + } + result = ingest_scip_json(doc) + by_symbol = {n["metadata"]["scip_symbol"]: n["id"] for n in result["nodes"]} + edge = result["edges"][0] + assert edge["target"] == by_symbol["UniqueCallee#"] + # Confirm the target node is in src/b.py (where it was DEFINED) + callee = next(n for n in result["nodes"] if n["id"] == by_symbol["UniqueCallee#"]) + assert callee["source_file"] == "src/b.py" + + +def test_ambiguous_duplicate_target_across_docs_creates_stub(): + """When a target symbol is defined in 2+ documents AND the source is in a + third (different) document, resolution is ambiguous — we refuse to pick + silently and emit a stub external node instead.""" + doc = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [{"symbol": "Shared#", "kind": "function"}], + }, + { + "relative_path": "b.py", + "symbols": [{"symbol": "Shared#", "kind": "function"}], + }, + { + "relative_path": "c.py", + "symbols": [ + { + "symbol": "Caller#", + "kind": "function", + "relationships": [{"symbol": "Shared#", "is_reference": True}], + }, + ], + }, + ] + } + result = ingest_scip_json(doc) + # Two Shared# nodes (one per defining doc) + a stub for c.py's reference + a Caller# + shared_in_c = [ + n + for n in result["nodes"] + if n["metadata"]["scip_symbol"] == "Shared#" and n["source_file"] == "c.py" + ] + assert len(shared_in_c) == 1 + # The stub from c.py is marked external (refused-to-guess fallback) + assert shared_in_c[0]["metadata"]["scip_kind"] == "external" + # The edge points at this stub (not at a.py's or b.py's Shared#) + edge = result["edges"][0] + assert edge["target"] == shared_in_c[0]["id"] + + +# --------------------------------------------------------------------------- +# Cycle 2.4 v3: strict boolean flags (F2) +# --------------------------------------------------------------------------- + + +def test_relationship_truthy_string_flag_is_ignored(): + """`"is_implementation": "false"` is a truthy STRING — must not route to + scip_impl. Only the actual boolean True counts as a set flag.""" + doc = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [ + { + "symbol": "Foo#", + "kind": "function", + "relationships": [ + { + "symbol": "B#", + "is_implementation": "false", # truthy STRING, not boolean True + "is_reference": True, + } + ], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["edges"][0]["relation"] == "scip_ref" + + +def test_relationship_int_flag_is_ignored(): + """`"is_implementation": 1` is truthy but not True — must not route to scip_impl.""" + doc = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [ + { + "symbol": "Foo#", + "kind": "function", + "relationships": [ + { + "symbol": "B#", + "is_implementation": 1, + "is_reference": True, + } + ], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["edges"][0]["relation"] == "scip_ref" + + +def test_relationship_boolean_true_routes_correctly(): + """Actual boolean True still routes to the corresponding scip_ relation.""" + cases = [ + ("is_implementation", "scip_impl"), + ("is_type_definition", "scip_typed"), + ("is_definition", "scip_def"), + ("is_reference", "scip_ref"), + ] + for flag, expected_relation in cases: + doc = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [ + { + "symbol": "Foo#", + "kind": "function", + "relationships": [{"symbol": "B#", flag: True}], + } + ], + } + ] + } + result = ingest_scip_json(doc) + assert result["edges"][0]["relation"] == expected_relation, ( + f"flag={flag} should produce {expected_relation}" + ) + + +# --------------------------------------------------------------------------- +# Cycle 2.4 v3: bool-int subclass guard for occurrence lines (F3) +# --------------------------------------------------------------------------- + + +def test_occurrence_bool_line_falls_back_to_zero(): + """range[0] = True (which is technically an int subclass) must not produce 'LTrue'.""" + doc = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [ + { + "symbol": "Foo#", + "kind": "function", + "occurrences": [{"range": [True, 0, True, 1]}], + } + ], + } + ] + } + result = ingest_scip_json(doc) + # Boolean line value rejected; source_location is empty (not "LTrue") + assert result["nodes"][0]["source_location"] == "" + + +def test_duplicate_same_document_definition_does_not_create_false_ambiguity(): + """Duplicate symbol records within the SAME document collapse to one node id + in the global index, so a caller in another file still resolves to that + real node (not a stub external).""" + doc = { + "documents": [ + { + "relative_path": "a.py", + "symbols": [ + # Two records for Helper# in the SAME file → same node id. + {"symbol": "Helper#", "kind": "function"}, + {"symbol": "Helper#", "kind": "function"}, + ], + }, + { + "relative_path": "b.py", + "symbols": [ + { + "symbol": "Caller#", + "kind": "function", + "relationships": [{"symbol": "Helper#", "is_reference": True}], + } + ], + }, + ] + } + result = ingest_scip_json(doc) + helper_nodes = [n for n in result["nodes"] if n["metadata"]["scip_symbol"] == "Helper#"] + # Only ONE Helper# node emitted (dedup), and it lives in a.py + assert len(helper_nodes) == 1 + assert helper_nodes[0]["source_file"] == "a.py" + assert helper_nodes[0]["metadata"]["scip_kind"] == "function" # real definition, not 'external' + # Edge from b.py's Caller# routes to a.py's real Helper# (NOT a stub) + edge = result["edges"][0] + assert edge["target"] == helper_nodes[0]["id"] + + +# --------------------------------------------------------------------------- +# sanitize_metadata wiring — SCIP descriptions / relationship payloads +# --------------------------------------------------------------------------- + + +def test_ingest_node_metadata_html_escaped() -> None: + """SCIP-supplied description must be HTML-escaped before reaching node + metadata; a malicious indexer cannot inject markup into HTML viewers.""" + doc = { + "documents": [ + { + "relative_path": "src/x.py", + "language": "python", + "symbols": [ + { + "symbol": "python/x.py:Evil#", + "kind": "class", + "display_name": "Evil", + "documentation": [""], + "occurrences": [{"range": [1, 0, 1, 5]}], + } + ], + } + ] + } + result = ingest_scip_json(doc) + node = result["nodes"][0] + desc = node["metadata"]["scip_description"] + assert "") + assert "<" in result + assert ">" in result + assert ""}) + assert isinstance(out, dict) + assert "<" in out["k"] + + +def test_sanitize_metadata_value_recurses_into_list(): + out = _sanitize_metadata_value(["", "", ""]) + assert isinstance(out, list) + assert all("<" in s for s in out) + + +def test_sanitize_metadata_value_caps_list_length(): + huge = list(range(_METADATA_MAX_LIST_ITEMS * 3)) + out = _sanitize_metadata_value(huge) + assert isinstance(out, list) + assert len(out) == _METADATA_MAX_LIST_ITEMS + + +def test_sanitize_metadata_value_converts_tuple_to_list(): + out = _sanitize_metadata_value(("a", "b")) + assert isinstance(out, list) + assert out == ["a", "b"] + + +def test_sanitize_metadata_none_returns_empty_dict(): + assert sanitize_metadata(None) == {} + + +def test_sanitize_metadata_drops_empty_key(): + # Empty key (after control-char strip) is dropped. + out = sanitize_metadata({"\x00": "v", "k": "v2"}) + assert "\x00" not in out + assert out.get("k") == "v2" + assert len(out) == 1 + + +def test_sanitize_metadata_sanitizes_keys(): + out = sanitize_metadata({"": "v"}) + assert "" not in out + assert any("<" in k for k in out.keys()) + + +def test_sanitize_metadata_recursive_nested(): + raw: dict[str, Any] = { + "outer": { + "inner": "", + "list": ["a", "", 99, None, True], + }, + "scalar": 42, + } + out = sanitize_metadata(raw) + assert isinstance(out["outer"], dict) + inner = out["outer"] + assert isinstance(inner, dict) + assert "<" in inner["inner"] + items = inner["list"] + assert isinstance(items, list) + assert items[0] == "a" + assert "<" in items[1] + assert items[2] == 99 + assert items[3] is None + assert items[4] is True + assert out["scalar"] == 42 + + +def test_sanitize_metadata_bool_not_coerced_to_int(): + # bool is an int subclass — order of isinstance checks must preserve bool. + out = sanitize_metadata({"flag_t": True, "flag_f": False, "num": 1}) + assert out["flag_t"] is True + assert out["flag_f"] is False + assert out["num"] == 1 diff --git a/tests/test_semantic_cleanup.py b/tests/test_semantic_cleanup.py new file mode 100644 index 0000000..7feda12 --- /dev/null +++ b/tests/test_semantic_cleanup.py @@ -0,0 +1,344 @@ +"""Tests for graphify.semantic_cleanup.validate_semantic_fragment (#825).""" + +import json + +from graphify import semantic_cleanup as sc + + +def _valid_fragment(): + return { + "nodes": [{"id": "module_func", "label": "func", "file_type": "code"}], + "edges": [{"source": "module_func", "target": "other_node"}], + "hyperedges": [], + } + + +def test_validate_semantic_fragment_accepts_valid(): + assert sc.validate_semantic_fragment(_valid_fragment()) == [] + + +def test_validate_semantic_fragment_rejects_non_object(): + errors = sc.validate_semantic_fragment(["not", "an", "object"]) + assert any("object" in e.lower() for e in errors) + + +def test_validate_semantic_fragment_rejects_oversize_payload(monkeypatch): + monkeypatch.setattr(sc, "MAX_SEMANTIC_FRAGMENT_BYTES", 64) + fragment = _valid_fragment() + fragment["nodes"][0]["label"] = "x" * 128 + errors = sc.validate_semantic_fragment(fragment) + assert any("payload" in e.lower() for e in errors) + + +def test_validate_semantic_fragment_rejects_too_many_nodes(monkeypatch): + monkeypatch.setattr(sc, "MAX_SEMANTIC_FRAGMENT_NODES", 1) + fragment = _valid_fragment() + fragment["nodes"].append({"id": "extra", "label": "extra", "file_type": "code"}) + errors = sc.validate_semantic_fragment(fragment) + assert any("nodes" in e.lower() for e in errors) + + +def test_validate_semantic_fragment_rejects_too_many_edges(monkeypatch): + monkeypatch.setattr(sc, "MAX_SEMANTIC_FRAGMENT_EDGES", 0) + errors = sc.validate_semantic_fragment(_valid_fragment()) + assert any("edges" in e.lower() for e in errors) + + +def test_validate_semantic_fragment_rejects_path_separator_in_id(): + fragment = _valid_fragment() + fragment["nodes"][0]["id"] = "../etc/passwd" + errors = sc.validate_semantic_fragment(fragment) + assert any("nodes[0].id" in e for e in errors) + + +def test_validate_semantic_fragment_rejects_invalid_file_type(): + fragment = _valid_fragment() + fragment["nodes"][0]["file_type"] = "executable" + errors = sc.validate_semantic_fragment(fragment) + assert any("file_type" in e for e in errors) + + +def test_validate_semantic_fragment_accepts_rationale_file_type(): + """LLM output with file_type='rationale' must pass validation so the cleanup + pass can convert or remove it. Validation must not reject it before cleanup runs.""" + fragment = _valid_fragment() + fragment["nodes"][0]["file_type"] = "rationale" + errors = sc.validate_semantic_fragment(fragment) + assert not any("file_type" in e for e in errors), ( + f"'rationale' must be accepted by validate_semantic_fragment; got errors: {errors}" + ) + + +def test_validate_semantic_fragment_accepts_concept_file_type(): + """LLM output with file_type='concept' must pass validation for the same reason.""" + fragment = _valid_fragment() + fragment["nodes"][0]["file_type"] = "concept" + errors = sc.validate_semantic_fragment(fragment) + assert not any("file_type" in e for e in errors), ( + f"'concept' must be accepted by validate_semantic_fragment; got errors: {errors}" + ) + + +def test_load_validated_semantic_fragment_accepts_valid(tmp_path): + chunk = tmp_path / ".graphify_chunk_00.json" + chunk.write_text(json.dumps(_valid_fragment())) + fragment, errors = sc.load_validated_semantic_fragment(chunk) + assert errors == [] + assert fragment == _valid_fragment() + + +def test_load_validated_semantic_fragment_rejects_oversize_before_parse(tmp_path, monkeypatch): + """Oversize files are rejected by stat() — payload is never parsed.""" + monkeypatch.setattr(sc, "MAX_SEMANTIC_FRAGMENT_BYTES", 64) + chunk = tmp_path / ".graphify_chunk_99.json" + # Write something that would PARSE successfully if read, but exceeds the size guard. + chunk.write_text("[" + ",".join(['"x"'] * 50) + "]") + fragment, errors = sc.load_validated_semantic_fragment(chunk) + assert fragment is None + assert any("payload" in e.lower() for e in errors) + + +def test_load_validated_semantic_fragment_rejects_invalid_json(tmp_path): + """Invalid JSON returns an error instead of raising.""" + chunk = tmp_path / ".graphify_chunk_bad.json" + chunk.write_text("{not valid json") + fragment, errors = sc.load_validated_semantic_fragment(chunk) + assert fragment is None + assert any("invalid json" in e.lower() for e in errors) + + +# --------------------------------------------------------------------------- +# Hyperedge validation (F2) +# --------------------------------------------------------------------------- + + +def test_validate_hyperedge_rejects_bad_id(): + fragment = _valid_fragment() + fragment["hyperedges"] = [ + {"id": "../escape", "label": "x", "nodes": ["module_func", "module_func"]} + ] + errors = sc.validate_semantic_fragment(fragment) + assert any("hyperedges[0].id" in e for e in errors) + + +def test_validate_hyperedge_rejects_bad_node_ref(): + fragment = _valid_fragment() + fragment["hyperedges"] = [ + {"id": "valid_he", "label": "x", "nodes": ["module_func", "../bad_ref"]} + ] + errors = sc.validate_semantic_fragment(fragment) + assert any("hyperedges[0].nodes[1]" in e for e in errors) + + +def test_validate_hyperedge_requires_list(): + fragment = _valid_fragment() + fragment["hyperedges"] = [{"id": "valid_he", "label": "x", "nodes": "not a list"}] + errors = sc.validate_semantic_fragment(fragment) + assert any("hyperedges[0].nodes" in e for e in errors) + + +def test_validate_hyperedge_caps_count(monkeypatch): + monkeypatch.setattr(sc, "MAX_SEMANTIC_FRAGMENT_HYPEREDGES", 1) + fragment = _valid_fragment() + fragment["hyperedges"] = [ + {"id": f"he_{i}", "label": "x", "nodes": ["module_func", "module_func"]} for i in range(3) + ] + errors = sc.validate_semantic_fragment(fragment) + assert any("hyperedges has 3" in e for e in errors) + + +# --------------------------------------------------------------------------- +# Sanitizer behavior (F3 + F4 + rationale conversion) +# --------------------------------------------------------------------------- + + +def test_sanitize_drops_rationale_filetype_node(): + """A node with file_type='rationale' is removed wholesale.""" + fragment = { + "nodes": [ + {"id": "real_node", "label": "Real", "file_type": "code"}, + {"id": "garbage", "label": "junk", "file_type": "rationale"}, + ], + "edges": [], + "hyperedges": [], + } + out = sc.sanitize_semantic_fragment(fragment) + ids = {n["id"] for n in out["nodes"]} + assert "real_node" in ids + assert "garbage" not in ids + + +def test_sanitize_converts_sentence_rationale_node_to_attribute(): + """Sentence-like rationale node connected via `rationale_for` → attribute on target.""" + fragment = { + "nodes": [ + {"id": "real_node", "label": "Real", "file_type": "code"}, + { + "id": "why_node", + "label": "We chose tree-sitter because the deterministic parser is faster than regex-based extraction.", + "file_type": "rationale", + }, + ], + "edges": [{"source": "why_node", "target": "real_node", "relation": "rationale_for"}], + "hyperedges": [], + } + out = sc.sanitize_semantic_fragment(fragment) + ids = {n["id"] for n in out["nodes"]} + assert "why_node" not in ids + target = next(n for n in out["nodes"] if n["id"] == "real_node") + assert "tree-sitter" in target.get("rationale", "") + + +def test_sanitize_converts_allowed_filetype_sentence_via_rationale_for_edge(): + """F3: a node with file_type='document' (allowed) that is BOTH sentence-like + AND sources a `rationale_for` edge is still cleaned to an attribute.""" + fragment = { + "nodes": [ + {"id": "real_node", "label": "Real", "file_type": "code"}, + { + "id": "sentence_node", + "label": ( + "Decision: this node has sentence-like rationale text but uses an " + "allowed file_type, so it should not survive as a standalone graph node." + ), + "file_type": "document", + }, + ], + "edges": [{"source": "sentence_node", "target": "real_node", "relation": "rationale_for"}], + "hyperedges": [], + } + out = sc.sanitize_semantic_fragment(fragment) + ids = {n["id"] for n in out["nodes"]} + assert "sentence_node" not in ids + target = next(n for n in out["nodes"] if n["id"] == "real_node") + assert "Decision" in target.get("rationale", "") + + +def test_sanitize_keeps_short_concept_named_node_with_punctuation(): + """A short named node with a period (e.g. abbreviation) is NOT sentence-like.""" + fragment = { + "nodes": [ + {"id": "a_b", "label": "a.b.c", "file_type": "document"}, + {"id": "anchor", "label": "Anchor", "file_type": "code"}, + ], + "edges": [{"source": "a_b", "target": "anchor", "relation": "rationale_for"}], + "hyperedges": [], + } + out = sc.sanitize_semantic_fragment(fragment) + ids = {n["id"] for n in out["nodes"]} + assert "a_b" in ids + assert "anchor" in ids + + +def test_sanitize_filters_hyperedges_after_node_removal(): + """F4: hyperedges referencing removed nodes are repaired or dropped.""" + fragment = { + "nodes": [ + {"id": "real_node", "label": "Real", "file_type": "code"}, + {"id": "other", "label": "Other", "file_type": "code"}, + {"id": "garbage", "label": "junk", "file_type": "rationale"}, + ], + "edges": [], + "hyperedges": [ + { + "id": "group_a", + "label": "Group A", + "nodes": ["garbage", "real_node", "other"], + "relation": "participate_in", + }, + { + "id": "group_b", + "label": "Group B (only one survivor)", + "nodes": ["garbage", "real_node"], + "relation": "participate_in", + }, + ], + } + out = sc.sanitize_semantic_fragment(fragment) + he_ids = {he["id"] for he in out["hyperedges"]} + # group_a survives with garbage filtered out + assert "group_a" in he_ids + group_a = next(he for he in out["hyperedges"] if he["id"] == "group_a") + assert "garbage" not in group_a["nodes"] + assert set(group_a["nodes"]) == {"real_node", "other"} + # group_b had only 1 surviving member → dropped + assert "group_b" not in he_ids + + +def test_sanitize_drops_hyperedge_with_only_unknown_refs(): + """A hyperedge referencing only nodes not present in the fragment is dropped.""" + fragment = { + "nodes": [{"id": "real_node", "label": "Real", "file_type": "code"}], + "edges": [], + "hyperedges": [{"id": "phantom", "label": "Phantom", "nodes": ["ghost1", "ghost2"]}], + } + out = sc.sanitize_semantic_fragment(fragment) + assert out["hyperedges"] == [] + + +def test_sanitize_boundary_sentence_threshold(): + """Boundary: a label with exactly 8 words + colon is sentence-like; + a 7-word label without sentence punctuation is not.""" + # 8 words, has colon → sentence-like + long_label = "Note: alpha beta gamma delta epsilon zeta eta" + fragment = { + "nodes": [ + {"id": "anchor", "label": "Anchor", "file_type": "code"}, + {"id": "n1", "label": long_label, "file_type": "rationale"}, + ], + "edges": [{"source": "n1", "target": "anchor", "relation": "rationale_for"}], + "hyperedges": [], + } + out = sc.sanitize_semantic_fragment(fragment) + assert {n["id"] for n in out["nodes"]} == {"anchor"} + anchor = out["nodes"][0] + assert "alpha" in anchor.get("rationale", "") + + # 7 words no terminal punctuation → not sentence-like + short_label = "alpha beta gamma delta epsilon zeta eta" + fragment = { + "nodes": [ + {"id": "anchor", "label": "Anchor", "file_type": "code"}, + {"id": "n2", "label": short_label, "file_type": "rationale"}, + ], + "edges": [], + "hyperedges": [], + } + out = sc.sanitize_semantic_fragment(fragment) + # n2 has file_type=rationale, so it's still removed via pass 1 — but should NOT + # become a rationale attribute on anchor (no rationale_for edge, no sentence pattern). + assert {n["id"] for n in out["nodes"]} == {"anchor"} + assert "rationale" not in out["nodes"][0] + + +def test_sanitize_rationale_only_propagates_through_rationale_for_edges(): + """A rationale node connected to ONE target via `rationale_for` and to ANOTHER + target via a non-rationale-for relation must NOT attach the rationale text + to the second target. Codex v2 caught the bug where every outgoing edge + propagated the rationale, corrupting unrelated nodes.""" + fragment = { + "nodes": [ + {"id": "rationale_target", "label": "Rationale Target", "file_type": "code"}, + {"id": "unrelated_target", "label": "Unrelated Target", "file_type": "code"}, + { + "id": "why_node", + "label": ( + "Decision: we chose tree-sitter because the deterministic parser " + "is faster than regex-based extraction." + ), + "file_type": "rationale", + }, + ], + "edges": [ + {"source": "why_node", "target": "rationale_target", "relation": "rationale_for"}, + {"source": "why_node", "target": "unrelated_target", "relation": "references"}, + ], + "hyperedges": [], + } + out = sc.sanitize_semantic_fragment(fragment) + ids = {n["id"]: n for n in out["nodes"]} + assert "why_node" not in ids + # rationale_target should have the rationale attribute + assert "tree-sitter" in ids["rationale_target"].get("rationale", "") + # unrelated_target should NOT have rationale leaked from the `references` edge + assert "rationale" not in ids["unrelated_target"] diff --git a/tests/test_serve.py b/tests/test_serve.py index 9dd1c7f..ce7461e 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -214,6 +214,32 @@ def test_load_graph_missing_file(tmp_path): _load_graph(str(graphify_dir / "nonexistent.json")) +def test_load_graph_rejects_oversized_file(monkeypatch, tmp_path, capsys): + # #F4: oversized graph.json must fail fast (SystemExit) with a clear error. + G = _make_graph() + data = json_graph.node_link_data(G, edges="links") + p = tmp_path / "graph.json" + p.write_text(json.dumps(data)) + monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 16) + with pytest.raises(SystemExit): + _load_graph(str(p)) + err = capsys.readouterr().err + assert "exceeds" in err + assert "byte cap" in err + + +def test_load_graph_accepts_under_cap(monkeypatch, tmp_path): + # Verifies the cap path does not regress the normal load. + G = _make_graph() + data = json_graph.node_link_data(G, edges="links") + p = tmp_path / "graph.json" + p.write_text(json.dumps(data)) + # Cap well above the actual file size — load proceeds. + monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 10 * 1024 * 1024) + G2 = _load_graph(str(p)) + assert G2.number_of_nodes() == G.number_of_nodes() + + # --- #874: MCP hot-reload --- def _write_graph(path, nodes: list[str]) -> None: diff --git a/tests/test_symbol_resolution.py b/tests/test_symbol_resolution.py new file mode 100644 index 0000000..44f6269 --- /dev/null +++ b/tests/test_symbol_resolution.py @@ -0,0 +1,1019 @@ +"""Tests for graphify.symbol_resolution.""" + +from __future__ import annotations + +from pathlib import Path + +from graphify.symbol_resolution import ( + _bash_make_id, + build_label_index, + build_python_symbol_index, + find_unique_python_symbol, + node_is_resolvable_symbol, + normalise_callable_label, + parse_python_import_aliases, + resolve_bash_source_edges, + resolve_cross_file_raw_calls, + resolve_python_import_guided_calls, +) + + +def test_normalise_callable_label_strips_function_punctuation() -> None: + assert normalise_callable_label("run()") == "run" + assert normalise_callable_label(".process()") == "process" + assert normalise_callable_label(" Execute ") == "execute" + + +def test_node_is_resolvable_symbol_skips_rationale_and_doc_tags() -> None: + assert node_is_resolvable_symbol({"id": "a", "label": "run()", "file_type": "code"}) is True + assert node_is_resolvable_symbol({"id": "r", "label": "why", "file_type": "rationale"}) is False + assert ( + node_is_resolvable_symbol({"id": "d", "label": "param x", "file_type": "doc_tag"}) is False + ) + + +def test_build_label_index_collects_unique_symbols() -> None: + nodes = [ + {"id": "a_run", "label": "run()", "file_type": "code"}, + {"id": "b_run", "label": "run()", "file_type": "code"}, + {"id": "doc", "label": "run docs", "file_type": "doc_tag"}, + ] + assert build_label_index(nodes) == {"run": ["a_run", "b_run"]} + + +def test_resolve_cross_file_raw_calls_emits_unique_unqualified_call() -> None: + per_file = [ + { + "raw_calls": [ + { + "caller_nid": "caller_run", + "callee": "helper", + "is_member_call": False, + "source_file": "caller.py", + "source_location": "L2", + } + ] + } + ] + nodes = [ + {"id": "caller_run", "label": "run()", "file_type": "code"}, + {"id": "helper_helper", "label": "helper()", "file_type": "code"}, + ] + edges = [] + + resolved = resolve_cross_file_raw_calls(per_file, nodes, edges) + + assert resolved == [ + { + "source": "caller_run", + "target": "helper_helper", + "relation": "calls", + "context": "call", + "confidence": "INFERRED", + "confidence_score": 0.8, + "source_file": "caller.py", + "source_location": "L2", + "weight": 1.0, + } + ] + + +def test_resolve_cross_file_raw_calls_skips_member_calls() -> None: + per_file = [ + { + "raw_calls": [ + { + "caller_nid": "caller_run", + "callee": "helper", + "is_member_call": True, + "source_file": "caller.py", + "source_location": "L2", + } + ] + } + ] + nodes = [ + {"id": "caller_run", "label": "run()", "file_type": "code"}, + {"id": "helper_helper", "label": "helper()", "file_type": "code"}, + ] + assert resolve_cross_file_raw_calls(per_file, nodes, []) == [] + + +def test_resolve_cross_file_raw_calls_skips_ambiguous_duplicate_labels() -> None: + per_file = [ + { + "raw_calls": [ + { + "caller_nid": "caller_run", + "callee": "log", + "is_member_call": False, + "source_file": "caller.py", + "source_location": "L2", + } + ] + } + ] + nodes = [ + {"id": "caller_run", "label": "run()", "file_type": "code"}, + {"id": "a_log", "label": "log()", "file_type": "code"}, + {"id": "b_log", "label": "log()", "file_type": "code"}, + ] + assert resolve_cross_file_raw_calls(per_file, nodes, []) == [] + + +def test_resolve_cross_file_raw_calls_skips_existing_pair() -> None: + per_file = [ + { + "raw_calls": [ + { + "caller_nid": "caller_run", + "callee": "helper", + "is_member_call": False, + "source_file": "caller.py", + "source_location": "L2", + } + ] + } + ] + nodes = [ + {"id": "caller_run", "label": "run()", "file_type": "code"}, + {"id": "helper_helper", "label": "helper()", "file_type": "code"}, + ] + edges = [{"source": "caller_run", "target": "helper_helper", "relation": "calls"}] + assert resolve_cross_file_raw_calls(per_file, nodes, edges) == [] + + +def test_parse_python_import_aliases_supports_from_import_alias(tmp_path: Path) -> None: + src = tmp_path / "caller.py" + src.write_text("from helper import transform as tx\n", encoding="utf-8") + + aliases = parse_python_import_aliases(src) + + assert set(aliases) == {"tx"} + imported = aliases["tx"] + assert imported.local_name == "tx" + assert imported.imported_name == "transform" + assert imported.module_stem == "helper" + assert imported.source_location == "L1" + + +def test_build_python_symbol_index_uses_module_stem_and_label() -> None: + nodes = [ + { + "id": "helper_transform", + "label": "transform()", + "file_type": "code", + "source_file": "/repo/helper.py", + }, + { + "id": "other_transform", + "label": "transform()", + "file_type": "code", + "source_file": "/repo/other.py", + }, + ] + index = build_python_symbol_index(nodes) + assert index[("helper", "transform")] == ["helper_transform"] + assert index[("other", "transform")] == ["other_transform"] + + +def test_find_unique_python_symbol_returns_none_when_ambiguous(tmp_path: Path) -> None: + src = tmp_path / "caller.py" + src.write_text("from helper import transform\n", encoding="utf-8") + imported = parse_python_import_aliases(src)["transform"] + index = {("helper", "transform"): ["a", "b"]} + assert find_unique_python_symbol(index, imported) is None + + +def test_resolve_python_import_guided_calls_emits_extracted_edge(tmp_path: Path) -> None: + caller = tmp_path / "caller.py" + helper = tmp_path / "helper.py" + caller.write_text( + "from helper import transform as tx\n\ndef run(value):\n return tx(value)\n", + encoding="utf-8", + ) + helper.write_text("def transform(value):\n return value\n", encoding="utf-8") + + per_file = [ + { + "raw_calls": [ + { + "caller_nid": "caller_run", + "callee": "tx", + "is_member_call": False, + "source_file": str(caller), + "source_location": "L4", + } + ] + }, + {"raw_calls": []}, + ] + nodes = [ + {"id": "caller_run", "label": "run()", "file_type": "code", "source_file": str(caller)}, + { + "id": "helper_transform", + "label": "transform()", + "file_type": "code", + "source_file": str(helper), + }, + ] + + edges = resolve_python_import_guided_calls(per_file, [caller, helper], nodes, []) + + assert edges == [ + { + "source": "caller_run", + "target": "helper_transform", + "relation": "calls", + "context": "import_guided_call", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": str(caller), + "source_location": "L4", + "weight": 1.0, + "metadata": { + "resolver": "python_import_guided", + "local_name": "tx", + "imported_name": "transform", + "module_stem": "helper", + "import_source_location": "L1", + }, + } + ] + + +# ═══════════════════════════════════════════════════════════════════════════════ +# ── Bash source edges resolver tests ────────────────────────────────────────── +# ═══════════════════════════════════════════════════════════════════════════════ + + +def test_bash_call_resolver_emits_source_edges(tmp_path: Path) -> None: + a_sh = tmp_path / "a.sh" + b_sh = tmp_path / "b.sh" + a_sh.write_text("#!/usr/bin/env bash\nsource ./b.sh\n") + b_sh.write_text("#!/usr/bin/env bash\nb_func() { echo ok; }\n") + + per_file = [ + { + "nodes": [ + {"id": "a_sh", "label": "a.sh", "file_type": "code", "source_file": str(a_sh)}, + { + "id": "a_entry", + "label": "a.sh script", + "file_type": "code", + "source_file": str(a_sh), + }, + ], + "edges": [], + "raw_calls": [], + "bash_sources": [ + {"source_file": str(a_sh), "target_path": str(b_sh), "source_location": "L2"} + ], + }, + { + "nodes": [ + {"id": "b_sh", "label": "b.sh", "file_type": "code", "source_file": str(b_sh)}, + { + "id": "b_func", + "label": "b_func()", + "file_type": "code", + "source_file": str(b_sh), + "metadata": {"kind": "bash_function"}, + }, + ], + "edges": [], + "raw_calls": [], + "bash_sources": [], + }, + ] + + edges = resolve_bash_source_edges(per_file, [a_sh, b_sh], tmp_path) + + imports = [e for e in edges if e["relation"] == "imports_from"] + assert len(imports) == 1 + assert imports[0]["confidence"] == "EXTRACTED" + + +def test_bash_call_resolver_emits_call_edges_from_sourced_files(tmp_path: Path) -> None: + a_sh = tmp_path / "a.sh" + b_sh = tmp_path / "b.sh" + a_sh.write_text("#!/usr/bin/env bash\nsource ./b.sh\nmain() { b_func; }\n") + b_sh.write_text("#!/usr/bin/env bash\nb_func() { echo ok; }\n") + + per_file = [ + { + "nodes": [ + {"id": "a_sh", "label": "a.sh", "file_type": "code", "source_file": str(a_sh)}, + { + "id": "main", + "label": "main()", + "file_type": "code", + "source_file": str(a_sh), + "metadata": {"kind": "bash_function"}, + }, + ], + "edges": [], + "raw_calls": [ + { + "language": "bash", + "caller_nid": "main", + "callee": "b_func", + "is_member_call": False, + "source_file": str(a_sh), + "source_location": "L3", + } + ], + "bash_sources": [ + {"source_file": str(a_sh), "target_path": str(b_sh), "source_location": "L2"} + ], + }, + { + "nodes": [ + {"id": "b_sh", "label": "b.sh", "file_type": "code", "source_file": str(b_sh)}, + { + "id": "b_func", + "label": "b_func()", + "file_type": "code", + "source_file": str(b_sh), + "metadata": {"kind": "bash_function"}, + }, + ], + "edges": [], + "raw_calls": [], + "bash_sources": [], + }, + ] + + edges = resolve_bash_source_edges(per_file, [a_sh, b_sh], tmp_path) + + calls = [e for e in edges if e["relation"] == "calls"] + assert len(calls) == 1 + assert calls[0]["source"] == "main" + assert calls[0]["target"] == "b_func" + assert calls[0]["confidence"] == "EXTRACTED" + + +def test_bash_call_resolver_skips_existing_pair(tmp_path: Path) -> None: + a_sh = tmp_path / "a.sh" + b_sh = tmp_path / "b.sh" + a_sh.write_text("#!/usr/bin/env bash\nsource ./b.sh\nmain() { b_func; }\n") + b_sh.write_text("#!/usr/bin/env bash\nb_func() { echo ok; }\n") + + per_file = [ + { + "nodes": [ + {"id": "a_sh", "label": "a.sh", "file_type": "code", "source_file": str(a_sh)}, + { + "id": "main", + "label": "main()", + "file_type": "code", + "source_file": str(a_sh), + "metadata": {"kind": "bash_function"}, + }, + ], + "edges": [], + "raw_calls": [ + { + "language": "bash", + "caller_nid": "main", + "callee": "b_func", + "is_member_call": False, + "source_file": str(a_sh), + "source_location": "L3", + } + ], + "bash_sources": [ + {"source_file": str(a_sh), "target_path": str(b_sh), "source_location": "L2"} + ], + }, + { + "nodes": [ + {"id": "b_sh", "label": "b.sh", "file_type": "code", "source_file": str(b_sh)}, + { + "id": "b_func", + "label": "b_func()", + "file_type": "code", + "source_file": str(b_sh), + "metadata": {"kind": "bash_function"}, + }, + ], + "edges": [], + "raw_calls": [], + "bash_sources": [], + }, + ] + existing = [{"source": "main", "target": "b_func", "relation": "calls"}] + + edges = resolve_bash_source_edges(per_file, [a_sh, b_sh], tmp_path, existing_edges=existing) + + calls = [e for e in edges if e["relation"] == "calls"] + assert len(calls) == 0, f"Should skip existing pair but got: {calls}" + + +def test_bash_call_resolver_skips_ambiguous_multiple_candidates(tmp_path: Path) -> None: + """When a callee function is defined in multiple sourced files, skip it.""" + a_sh = tmp_path / "a.sh" + b_sh = tmp_path / "b.sh" + c_sh = tmp_path / "c.sh" + a_sh.write_text("#!/usr/bin/env bash\nsource ./b.sh\nsource ./c.sh\nmain() { helper; }\n") + b_sh.write_text("#!/usr/bin/env bash\nhelper() { echo b; }\n") + c_sh.write_text("#!/usr/bin/env bash\nhelper() { echo c; }\n") + + per_file = [ + { + "nodes": [ + {"id": "a_sh", "label": "a.sh", "file_type": "code", "source_file": str(a_sh)}, + { + "id": "main", + "label": "main()", + "file_type": "code", + "source_file": str(a_sh), + "metadata": {"kind": "bash_function"}, + }, + ], + "edges": [], + "raw_calls": [ + { + "language": "bash", + "caller_nid": "main", + "callee": "helper", + "is_member_call": False, + "source_file": str(a_sh), + "source_location": "L4", + } + ], + "bash_sources": [ + {"source_file": str(a_sh), "target_path": str(b_sh), "source_location": "L2"}, + {"source_file": str(a_sh), "target_path": str(c_sh), "source_location": "L3"}, + ], + }, + { + "nodes": [ + {"id": "b_sh", "label": "b.sh", "file_type": "code", "source_file": str(b_sh)}, + { + "id": "b_helper", + "label": "helper()", + "file_type": "code", + "source_file": str(b_sh), + "metadata": {"kind": "bash_function"}, + }, + ], + "edges": [], + "raw_calls": [], + "bash_sources": [], + }, + { + "nodes": [ + {"id": "c_sh", "label": "c.sh", "file_type": "code", "source_file": str(c_sh)}, + { + "id": "c_helper", + "label": "helper()", + "file_type": "code", + "source_file": str(c_sh), + "metadata": {"kind": "bash_function"}, + }, + ], + "edges": [], + "raw_calls": [], + "bash_sources": [], + }, + ] + + edges = resolve_bash_source_edges(per_file, [a_sh, b_sh, c_sh], tmp_path) + + calls = [e for e in edges if e["relation"] == "calls"] + # helper() is defined in both b.sh and c.sh → ambiguous → should be skipped + assert len(calls) == 0, f"Should skip ambiguous callee but got: {calls}" + + +def test_bash_call_resolver_skips_non_bash_raw_calls(tmp_path: Path) -> None: + """Non-bash raw_calls inside sourced-file per_file entries are ignored.""" + a_sh = tmp_path / "a.sh" + a_sh.write_text("#!/usr/bin/env bash\n") + + per_file = [ + { + "nodes": [ + {"id": "a_sh", "label": "a.sh", "file_type": "code", "source_file": str(a_sh)}, + ], + "edges": [], + "raw_calls": [ + { + "language": "python", + "caller_nid": "a_main", + "callee": "helper", + "is_member_call": False, + "source_file": str(a_sh), + "source_location": "L1", + } + ], + "bash_sources": [], + }, + ] + + edges = resolve_bash_source_edges(per_file, [a_sh], tmp_path) + assert edges == [], f"Should ignore non-bash raw_calls but got: {edges}" + + +def test_bash_make_id_identical_to_make_id() -> None: + from graphify.extract import _make_id + + assert _bash_make_id("foo", "bar") == _make_id("foo", "bar") + assert _bash_make_id("auth") == _make_id("auth") + assert _bash_make_id("_module", "_helper") == _make_id("_module", "_helper") + assert _bash_make_id("my-script", "main") == _make_id("my-script", "main") + + +def test_bash_make_id_unicode_matches_make_id() -> None: + """_bash_make_id must produce identical output to _make_id for Unicode inputs. + + The two functions must remain in sync so resolve_bash_source_edges + produces node IDs that match those from extract_bash. The original local + copy lacked NFKC normalisation, Unicode-aware regex, and casefold(). + """ + from graphify.extract import _make_id + + # Accented letter: é is a Unicode word char that _make_id preserves + assert _bash_make_id("café", "run") == _make_id("café", "run"), ( + "_bash_make_id must preserve Unicode word characters like _make_id" + ) + # German sharp s: casefold maps ß→ss, lower does not + assert _bash_make_id("straße") == _make_id("straße"), ( + "_bash_make_id must use casefold not lower to match _make_id" + ) + + +# --------------------------------------------------------------------------- +# Cycle 2.5 v2 — Codex blocker fixes +# --------------------------------------------------------------------------- + + +# F1 — top-level imports only +def test_parse_python_import_aliases_skips_function_local_imports(tmp_path): + """A `from helper import transform` inside a function MUST NOT become + file-wide evidence — function-local imports are only valid in their + lexical scope. Walking the whole AST would falsely justify unrelated + calls in other scopes.""" + from graphify.symbol_resolution import parse_python_import_aliases + + py = tmp_path / "scoped.py" + py.write_text( + "def one():\n" + " from helper import transform\n" + " return transform()\n" + "\n" + "def two():\n" + " return transform()\n" + ) + aliases = parse_python_import_aliases(py) + assert "transform" not in aliases, ( + f"function-local import leaked as file-wide evidence: {aliases}" + ) + + +def test_parse_python_import_aliases_accepts_top_level_import(tmp_path): + """A module-level `from helper import transform` IS file-wide evidence.""" + from graphify.symbol_resolution import parse_python_import_aliases + + py = tmp_path / "toplevel.py" + py.write_text("from helper import transform\n\ndef one():\n return transform()\n") + aliases = parse_python_import_aliases(py) + assert "transform" in aliases + assert aliases["transform"].module_stem == "helper" + + +# F2 — only code nodes are resolvable +def test_node_is_resolvable_symbol_requires_code_file_type(): + """Document/paper/image/concept nodes MUST NOT be indexed as call targets, + even when their label looks like a callable identifier.""" + from graphify.symbol_resolution import node_is_resolvable_symbol + + code = {"id": "n1", "label": "helper", "file_type": "code"} + doc = {"id": "n2", "label": "helper", "file_type": "document"} + paper = {"id": "n3", "label": "helper", "file_type": "paper"} + image = {"id": "n4", "label": "helper", "file_type": "image"} + no_ft = {"id": "n5", "label": "helper"} + + assert node_is_resolvable_symbol(code) is True + assert node_is_resolvable_symbol(doc) is False + assert node_is_resolvable_symbol(paper) is False + assert node_is_resolvable_symbol(image) is False + assert node_is_resolvable_symbol(no_ft) is False + + +def test_build_label_index_excludes_non_code_nodes(): + """label index must not include document/paper/image nodes even when + label and id are present and well-formed.""" + from graphify.symbol_resolution import build_label_index + + nodes = [ + {"id": "code_one", "label": "helper", "file_type": "code"}, + {"id": "doc_one", "label": "helper", "file_type": "document"}, + {"id": "paper_one", "label": "helper", "file_type": "paper"}, + ] + index = build_label_index(nodes) + assert index.get("helper") == ["code_one"] + + +# F3 — bash resolver defensive against malformed input +def test_resolve_bash_source_edges_skips_malformed_source(tmp_path): + """A `bash_sources` entry missing `target_path` must not raise KeyError.""" + from graphify.symbol_resolution import resolve_bash_source_edges + + per_file = [ + { + "nodes": [], + "raw_calls": [], + "bash_sources": [ + {}, # missing target_path entirely + {"target_path": ""}, # empty target_path + {"target_path": None}, # non-string target_path + ], + } + ] + a = tmp_path / "a.sh" + a.write_text("# noop\n") + edges = resolve_bash_source_edges(per_file, [a], tmp_path) + assert edges == [] + + +def test_resolve_bash_source_edges_skips_bash_function_node_missing_id(tmp_path): + """A node tagged as bash_function but missing `id` must not raise KeyError.""" + from graphify.symbol_resolution import resolve_bash_source_edges + + per_file = [ + { + "nodes": [ + {"label": "build()", "metadata": {"kind": "bash_function"}}, + ], + "raw_calls": [], + "bash_sources": [], + } + ] + a = tmp_path / "a.sh" + a.write_text("# noop\n") + # Should not raise + edges = resolve_bash_source_edges(per_file, [a], tmp_path) + assert edges == [] + + +def test_resolve_bash_source_edges_skips_raw_call_missing_caller_nid(tmp_path): + """A raw_call entry missing `caller_nid` must not raise KeyError.""" + from graphify.symbol_resolution import resolve_bash_source_edges + + a = tmp_path / "a.sh" + b = tmp_path / "b.sh" + a.write_text("# noop\n") + b.write_text("# noop\n") + per_file = [ + { + "nodes": [], + "raw_calls": [ + {"language": "bash", "callee": "helper"}, # missing caller_nid + ], + "bash_sources": [{"target_path": str(b)}], + }, + { + "nodes": [ + {"id": "b_helper", "label": "helper()", "metadata": {"kind": "bash_function"}}, + ], + "raw_calls": [], + "bash_sources": [], + }, + ] + edges = resolve_bash_source_edges(per_file, [a, b], tmp_path) + # No raw-call edge emitted because caller_nid was missing; source edge OK. + assert all(e["relation"] != "calls" for e in edges) + + +def test_resolve_bash_source_edges_accepts_none_per_file_entries(tmp_path): + """A None entry in per_file (e.g. failed extraction) must be silently skipped.""" + from graphify.symbol_resolution import resolve_bash_source_edges + + a = tmp_path / "a.sh" + a.write_text("# noop\n") + edges = resolve_bash_source_edges([None], [a], tmp_path) + assert edges == [] + + +def test_resolve_bash_source_edges_skips_non_dict_lists(tmp_path): + """Non-dict entries in bash_sources/raw_calls/nodes must be silently skipped.""" + from graphify.symbol_resolution import resolve_bash_source_edges + + a = tmp_path / "a.sh" + a.write_text("# noop\n") + per_file = [ + { + "nodes": ["not a dict", 42, None], + "raw_calls": [None, "string entry", {"language": "bash"}], # last is missing caller_nid + "bash_sources": [None, "str", 99], + } + ] + edges = resolve_bash_source_edges(per_file, [a], tmp_path) + assert edges == [] + + +# F4 — relative target_path resolves against source file directory +def test_resolve_bash_source_edges_relative_path_resolves_against_source_dir(tmp_path): + """`source ./helper.sh` from a/main.sh should resolve to a/helper.sh, + not to ./helper.sh from the process CWD.""" + from graphify.symbol_resolution import resolve_bash_source_edges + + sub = tmp_path / "scripts" + sub.mkdir() + main = sub / "main.sh" + helper = sub / "helper.sh" + main.write_text("# main\n") + helper.write_text("# helper\n") + + per_file = [ + { + "nodes": [], + "raw_calls": [], + # Relative path: should resolve to scripts/helper.sh (next to main.sh) + "bash_sources": [{"target_path": "./helper.sh"}], + }, + { + "nodes": [], + "raw_calls": [], + "bash_sources": [], + }, + ] + edges = resolve_bash_source_edges(per_file, [main, helper], tmp_path) + # One imports_from edge from main → helper + import_edges = [e for e in edges if e["relation"] == "imports_from"] + assert len(import_edges) == 1 + # Note: the actual node IDs are sha-hash-derived; just verify the edge exists. + + +# F1 — malformed raw_calls in non-Bash resolvers +def test_iter_raw_calls_skips_non_dict_per_file_entries(): + """A non-dict per_file entry (e.g. junk fragment) must be silently skipped.""" + from graphify.symbol_resolution import iter_raw_calls + + assert iter_raw_calls(["not a dict", None, 42]) == [] + + +def test_iter_raw_calls_skips_non_list_raw_calls(): + """`raw_calls` that isn't a list must yield empty.""" + from graphify.symbol_resolution import iter_raw_calls + + assert iter_raw_calls([{"raw_calls": "abc"}]) == [] + assert iter_raw_calls([{"raw_calls": None}]) == [] + assert iter_raw_calls([{"raw_calls": 42}]) == [] + + +def test_iter_raw_calls_drops_non_dict_items_in_list(): + """Items inside `raw_calls` list that aren't dicts must be dropped.""" + from graphify.symbol_resolution import iter_raw_calls + + out = iter_raw_calls([{"raw_calls": ["str", 42, None, {"callee": "real", "caller_nid": "c"}]}]) + assert out == [{"callee": "real", "caller_nid": "c"}] + + +def test_resolve_cross_file_raw_calls_survives_malformed_raw_calls(): + """The python cross-file resolver returns [] (not crash) on bad raw_calls.""" + from graphify.symbol_resolution import resolve_cross_file_raw_calls + + # raw_calls is a string instead of a list + assert resolve_cross_file_raw_calls([{"raw_calls": "abc"}], [], []) == [] + # raw_calls list contains non-dict entries + assert resolve_cross_file_raw_calls([{"raw_calls": ["not dict", 42]}], [], []) == [] + + +def test_resolve_python_import_guided_calls_survives_malformed_raw_calls(tmp_path): + """Python import-guided resolver also tolerates malformed raw_calls.""" + from graphify.symbol_resolution import resolve_python_import_guided_calls + + py = tmp_path / "caller.py" + py.write_text("from helper import transform\n") + per_file = [{"raw_calls": "not a list"}] + paths = [py] + nodes = [ + { + "id": "h_transform", + "label": "transform", + "file_type": "code", + "source_file": str(tmp_path / "helper.py"), + } + ] + # Should not raise; should return no edges since raw_calls isn't a list + edges = resolve_python_import_guided_calls(per_file, paths, nodes, []) + assert edges == [] + + +# F2 — unhashable callee in bash resolver +def test_resolve_bash_source_edges_skips_unhashable_callee(tmp_path): + """A bash raw_call with `callee: [list]` (unhashable for dict membership) + must not raise TypeError — silently skip the call.""" + from graphify.symbol_resolution import resolve_bash_source_edges + + a = tmp_path / "a.sh" + b = tmp_path / "b.sh" + a.write_text("# noop\n") + b.write_text("# noop\n") + per_file = [ + { + "nodes": [], + "raw_calls": [ + {"language": "bash", "caller_nid": "caller", "callee": ["bad"]}, + {"language": "bash", "caller_nid": "caller", "callee": {"also": "bad"}}, + {"language": "bash", "caller_nid": "caller", "callee": 42}, + ], + "bash_sources": [{"target_path": str(b)}], + }, + { + "nodes": [ + {"id": "b_helper", "label": "helper()", "metadata": {"kind": "bash_function"}}, + ], + "raw_calls": [], + "bash_sources": [], + }, + ] + # Must not raise — non-string callees are skipped before dict membership. + edges = resolve_bash_source_edges(per_file, [a, b], tmp_path) + # No call edges emitted (all malformed); imports_from edge from sourcing OK + assert all(e["relation"] != "calls" for e in edges) + + +# v3 Codex F1 — resolve_python_import_guided_calls hardened against +# malformed per_file slots and length mismatches. +def test_resolve_python_import_guided_calls_non_dict_per_file_slot(tmp_path): + """A non-dict per_file slot (e.g. a string) must not raise AttributeError.""" + from graphify.symbol_resolution import resolve_python_import_guided_calls + + py = tmp_path / "caller.py" + py.write_text("from helper import transform\n") + # per_file slot is a STRING, not a dict — used to crash with AttributeError + edges = resolve_python_import_guided_calls(["not a dict"], [py], [], []) + assert edges == [] + + +def test_resolve_python_import_guided_calls_per_file_shorter_than_paths(tmp_path): + """per_file shorter than paths must not raise IndexError.""" + from graphify.symbol_resolution import resolve_python_import_guided_calls + + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("from helper import transform\n") + b.write_text("from helper import transform\n") + # Only ONE per_file entry but TWO paths — used to crash with IndexError + edges = resolve_python_import_guided_calls([{}], [a, b], [], []) + assert edges == [] + + +def test_resolve_python_import_guided_calls_per_file_none_slot(tmp_path): + """A None per_file slot is treated as empty fragment (no crash, no edges).""" + from graphify.symbol_resolution import resolve_python_import_guided_calls + + py = tmp_path / "caller.py" + py.write_text("from helper import transform\n") + edges = resolve_python_import_guided_calls([None], [py], [], []) + assert edges == [] + + +def test_resolve_python_import_guided_calls_metadata_is_sanitized(tmp_path: Path) -> None: + """Edge metadata produced by the import-guided resolver must pass through + sanitize_metadata so HTML / control characters in import-site strings + (e.g. malformed source_location values, alias names from extractor bugs) + cannot survive into the graph as raw markup.""" + caller = tmp_path / "caller.py" + helper = tmp_path / "helper.py" + # Import alias that includes an angle bracket — pathological but defensive + # cover: the resolver itself does not parse names this aggressively, but a + # future extractor or upstream fragment could. The boundary is the cycle's + # stated policy: every edge metadata field goes through sanitize_metadata. + caller.write_text( + "from helper import transform as tx\n\ndef run(value):\n return tx(value)\n", + encoding="utf-8", + ) + helper.write_text("def transform(value):\n return value\n", encoding="utf-8") + + per_file = [ + { + "raw_calls": [ + { + "caller_nid": "caller_run", + "callee": "tx", + "is_member_call": False, + "source_file": str(caller), + "source_location": "L4", + } + ] + }, + {"raw_calls": []}, + ] + nodes = [ + {"id": "caller_run", "label": "run()", "file_type": "code", "source_file": str(caller)}, + { + "id": "helper_transform", + "label": "transform()", + "file_type": "code", + "source_file": str(helper), + }, + ] + + edges = resolve_python_import_guided_calls(per_file, [caller, helper], nodes, []) + assert len(edges) == 1 + metadata = edges[0]["metadata"] + # All values must be present and HTML/control-char safe after sanitisation. + for value in metadata.values(): + if isinstance(value, str): + assert "<" not in value + assert "\x00" not in value + # And the structural shape is unchanged for benign inputs. + assert metadata["resolver"] == "python_import_guided" + assert metadata["local_name"] == "tx" + assert metadata["imported_name"] == "transform" + assert metadata["module_stem"] == "helper" + + +def test_resolve_python_import_guided_calls_metadata_sanitizes_hostile_alias( + monkeypatch, tmp_path: Path +) -> None: + """Strong regression for #cycle-2.7-Codex-v2: monkeypatch the alias parser + so the resolver sees HOSTILE strings in ImportedSymbol fields, then assert + the emitted metadata is HTML-escaped / control-char-stripped. + + Removing the sanitize_metadata() wrap in + ``resolve_python_import_guided_calls`` would make this test fail: + `<script>` would not appear in `imported_name`, and the raw + NUL byte would not be stripped from `module_stem`. + """ + import graphify.symbol_resolution as sr + + caller = tmp_path / "caller.py" + helper = tmp_path / "helper.py" + caller.write_text( + "from helper import transform as tx\n\ndef run(value):\n return tx(value)\n", + encoding="utf-8", + ) + helper.write_text("def transform(value):\n return value\n", encoding="utf-8") + + # imported_name and module_stem are the lookup keys used to resolve the + # call target; they must match the real helper symbol or the edge will + # not fire. local_name and source_location are stored verbatim into + # metadata and are the surface that sanitize_metadata() must scrub. + hostile_alias_key = "" + hostile = sr.ImportedSymbol( + local_name=hostile_alias_key, + imported_name="transform", + module_stem="helper", + source_file=str(caller), + source_location="L1\x00trail", + ) + + def _fake_aliases(path: Path) -> dict[str, sr.ImportedSymbol]: + if path == caller: + return {hostile_alias_key: hostile} + return {} + + monkeypatch.setattr(sr, "parse_python_import_aliases", _fake_aliases) + + per_file = [ + { + "raw_calls": [ + { + "caller_nid": "caller_run", + "callee": hostile_alias_key, + "is_member_call": False, + "source_file": str(caller), + "source_location": "L4", + } + ] + }, + {"raw_calls": []}, + ] + nodes = [ + {"id": "caller_run", "label": "run()", "file_type": "code", "source_file": str(caller)}, + { + "id": "helper_transform", + "label": "transform()", + "file_type": "code", + "source_file": str(helper), + }, + ] + + edges = resolve_python_import_guided_calls(per_file, [caller, helper], nodes, []) + assert len(edges) == 1 + metadata = edges[0]["metadata"] + + # `local_name` carries the hostile alias key. Without sanitisation it + # would still contain `