Christian Winther
0dfc26e57f
When both a file (foo.ts) and a directory (foo/) exist at the same path, both TypeScript and Vite prefer the file. The previous ordering checked directory first and fell through unchanged when the directory had no index, silently dropping every import like 'from ./auth' when an auth/ subdirectory existed alongside auth.ts.
4492 lines
189 KiB
Python
4492 lines
189 KiB
Python
"""Deterministic structural extraction from source code using tree-sitter. Outputs nodes+edges dicts."""
|
|
from __future__ import annotations
|
|
import importlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Callable, Any
|
|
from .cache import load_cached, save_cached
|
|
|
|
_RECURSION_LIMIT = 10_000
|
|
|
|
|
|
def _raise_recursion_limit() -> None:
|
|
if sys.getrecursionlimit() < _RECURSION_LIMIT:
|
|
sys.setrecursionlimit(_RECURSION_LIMIT)
|
|
|
|
|
|
def _safe_extract(extractor: Callable, path: Path) -> dict:
|
|
try:
|
|
return extractor(path)
|
|
except RecursionError:
|
|
print(f" warning: skipped {path} (recursion limit exceeded)", file=sys.stderr, flush=True)
|
|
return {"nodes": [], "edges": [], "error": "recursion_limit_exceeded"}
|
|
except Exception as e:
|
|
print(f" warning: skipped {path} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
|
|
return {"nodes": [], "edges": [], "error": f"{type(e).__name__}: {e}"}
|
|
|
|
|
|
def _make_id(*parts: str) -> str:
|
|
"""Build a stable node ID from one or more name parts."""
|
|
combined = "_".join(p.strip("_.") for p in parts if p)
|
|
cleaned = re.sub(r"[^a-zA-Z0-9]+", "_", combined)
|
|
return cleaned.strip("_").lower()
|
|
|
|
|
|
def _file_stem(path: Path) -> str:
|
|
"""Return a stem qualified with the parent directory name to avoid ID collisions
|
|
when multiple files share the same filename in different directories (#550)."""
|
|
parent = path.parent.name
|
|
if parent and parent not in (".", ""):
|
|
return f"{parent}.{path.stem}"
|
|
return path.stem
|
|
|
|
|
|
_TSCONFIG_ALIAS_CACHE: dict[str, dict[str, str]] = {}
|
|
|
|
|
|
def _strip_jsonc(text: str) -> str:
|
|
"""Strip // line comments, /* */ block comments, and trailing commas from JSONC.
|
|
|
|
Preserves string contents (including // and /* inside strings) by skipping over
|
|
quoted spans first. Required for tsconfig.json files generated by SvelteKit,
|
|
NestJS, Vite, T3, Astro, etc., which use JSONC by default (#700).
|
|
"""
|
|
# Remove block and line comments while leaving string literals untouched.
|
|
pattern = re.compile(
|
|
r'"(?:\\.|[^"\\])*"' # double-quoted string (with escapes)
|
|
r"|/\*.*?\*/" # /* block comment */
|
|
r"|//[^\n]*", # // line comment
|
|
re.DOTALL,
|
|
)
|
|
|
|
def _replace(match: re.Match) -> str:
|
|
token = match.group(0)
|
|
if token.startswith('"'):
|
|
return token
|
|
return ""
|
|
|
|
stripped = pattern.sub(_replace, text)
|
|
# Remove trailing commas before } or ] (allowing whitespace between).
|
|
stripped = re.sub(r",(\s*[}\]])", r"\1", stripped)
|
|
return stripped
|
|
|
|
|
|
def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[str, str]:
|
|
"""Recursively read path aliases from a tsconfig, following extends chains.
|
|
|
|
Child config paths override parent. Circular extends are detected via seen set.
|
|
npm package configs (e.g. @tsconfig/svelte) are skipped since they're not on disk.
|
|
Handles JSONC (comments + trailing commas) which is the default tsconfig format
|
|
for SvelteKit, NestJS, Vite, T3, Astro, etc. (#700).
|
|
"""
|
|
if str(tsconfig) in seen:
|
|
return {}
|
|
seen.add(str(tsconfig))
|
|
try:
|
|
raw = tsconfig.read_text(encoding="utf-8")
|
|
except Exception as e:
|
|
print(f" warning: could not read {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
|
|
return {}
|
|
try:
|
|
data = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
try:
|
|
data = json.loads(_strip_jsonc(raw))
|
|
except json.JSONDecodeError as e:
|
|
print(f" warning: failed to parse {tsconfig} as JSON/JSONC ({e.msg} at line {e.lineno} col {e.colno})", file=sys.stderr, flush=True)
|
|
return {}
|
|
except Exception as e:
|
|
print(f" warning: failed to parse {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
|
|
return {}
|
|
|
|
aliases: dict[str, str] = {}
|
|
extends = data.get("extends")
|
|
if extends and not extends.startswith("@"):
|
|
extended_path = (base_dir / extends).resolve()
|
|
if not extended_path.suffix:
|
|
extended_path = extended_path.with_suffix(".json")
|
|
if extended_path.exists():
|
|
aliases.update(_read_tsconfig_aliases(extended_path, extended_path.parent, seen))
|
|
|
|
paths = data.get("compilerOptions", {}).get("paths", {})
|
|
for alias, targets in paths.items():
|
|
if not targets:
|
|
continue
|
|
alias_prefix = alias.rstrip("/*")
|
|
target_base = targets[0].rstrip("/*")
|
|
aliases[alias_prefix] = str(base_dir / target_base)
|
|
|
|
return aliases
|
|
|
|
|
|
def _load_tsconfig_aliases(start_dir: Path) -> dict[str, str]:
|
|
"""Walk up from start_dir to find tsconfig.json and return compilerOptions.paths aliases.
|
|
|
|
Follows extends chains so SvelteKit/Nuxt/NestJS inherited aliases are included.
|
|
Returns a dict mapping alias prefix (e.g. "@/") to resolved base dir (e.g. "src/").
|
|
Result is cached by tsconfig path string.
|
|
"""
|
|
current = start_dir.resolve()
|
|
for candidate in [current, *current.parents]:
|
|
tsconfig = candidate / "tsconfig.json"
|
|
if tsconfig.exists():
|
|
key = str(tsconfig)
|
|
if key not in _TSCONFIG_ALIAS_CACHE:
|
|
_TSCONFIG_ALIAS_CACHE[key] = _read_tsconfig_aliases(tsconfig, candidate, seen=set())
|
|
return _TSCONFIG_ALIAS_CACHE[key]
|
|
return {}
|
|
|
|
|
|
# ── LanguageConfig dataclass ─────────────────────────────────────────────────
|
|
|
|
@dataclass
|
|
class LanguageConfig:
|
|
ts_module: str # e.g. "tree_sitter_python"
|
|
ts_language_fn: str = "language" # attr to call: e.g. tslang.language()
|
|
|
|
class_types: frozenset = frozenset()
|
|
function_types: frozenset = frozenset()
|
|
import_types: frozenset = frozenset()
|
|
call_types: frozenset = frozenset()
|
|
static_prop_types: frozenset = frozenset()
|
|
helper_fn_names: frozenset = frozenset()
|
|
container_bind_methods: frozenset = frozenset()
|
|
event_listener_properties: frozenset = frozenset()
|
|
|
|
# Name extraction
|
|
name_field: str = "name"
|
|
name_fallback_child_types: tuple = ()
|
|
|
|
# Body detection
|
|
body_field: str = "body"
|
|
body_fallback_child_types: tuple = () # e.g. ("declaration_list", "compound_statement")
|
|
|
|
# Call name extraction
|
|
call_function_field: str = "function" # field on call node for callee
|
|
call_accessor_node_types: frozenset = frozenset() # member/attribute nodes
|
|
call_accessor_field: str = "attribute" # field on accessor for method name
|
|
|
|
# Stop recursion at these types in walk_calls
|
|
function_boundary_types: frozenset = frozenset()
|
|
|
|
# Import handler: called for import nodes instead of generic handling
|
|
import_handler: Callable | None = None
|
|
|
|
# Optional custom name resolver for functions (C, C++ declarator unwrapping)
|
|
resolve_function_name_fn: Callable | None = None
|
|
|
|
# Extra label formatting for functions: if True, functions get "name()" label
|
|
function_label_parens: bool = True
|
|
|
|
# Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.)
|
|
extra_walk_fn: Callable | None = None
|
|
|
|
|
|
# ── Generic helpers ───────────────────────────────────────────────────────────
|
|
|
|
# Vite / TypeScript resolver extensions. Used by _resolve_js_module_path()
|
|
# to map import specifiers onto real files on disk, so the resulting node
|
|
# id matches the one _extract_generic creates for the target file.
|
|
_JS_RESOLVE_EXTS = (".ts", ".tsx", ".svelte", ".js", ".jsx", ".mjs")
|
|
_JS_INDEX_FILES = ("index.ts", "index.tsx", "index.js", "index.jsx")
|
|
|
|
|
|
def _resolve_js_module_path(p: Path) -> Path:
|
|
"""Resolve a JS/TS-style import specifier path to an actual file on disk.
|
|
|
|
TypeScript / SvelteKit / Vite let you write imports without a file
|
|
extension and auto-resolve via a fixed extension order. The pre-existing
|
|
.js→.ts and .jsx→.tsx rewrites only covered the TS-ESM-via-.js convention;
|
|
every other shape produced a phantom node id and the edge was lost in
|
|
build_from_json.
|
|
|
|
Order, mirroring Vite's resolver:
|
|
|
|
1. exact path, when it's a real file on disk
|
|
2. directory → try index.{ts,tsx,js,jsx}
|
|
3. .js → .ts (TS ESM convention; written as .js, file is .ts)
|
|
.jsx → .tsx
|
|
4. append .ts/.tsx/.svelte/.js/.jsx/.mjs to the FULL filename — not
|
|
a suffix-swap. This handles, in one rule:
|
|
- bare paths: foo → foo.ts
|
|
- Svelte 5 rune files: foo.svelte → foo.svelte.ts
|
|
- multi-dot helper files: foo.shared → foo.shared.ts
|
|
- config files: foo.config → foo.config.ts
|
|
- test helper files: foo.spec → foo.spec.ts
|
|
5. directory variant: try ./<name>/index.{ts,tsx,js,jsx}
|
|
|
|
Falls back to the original path on no match — preserves pre-fix behaviour
|
|
for genuinely external modules (the edge gets dropped as external by
|
|
build_from_json).
|
|
"""
|
|
if p.is_file():
|
|
return p
|
|
# TS ESM convention: import path written with .js but the real file is .ts.
|
|
# Apply BEFORE the generic append loop so we don't accidentally match
|
|
# foo.js → foo.js.ts when the real file is foo.ts.
|
|
if p.suffix == ".js":
|
|
c = p.with_suffix(".ts")
|
|
if c.is_file():
|
|
return c
|
|
if p.suffix == ".jsx":
|
|
c = p.with_suffix(".tsx")
|
|
if c.is_file():
|
|
return c
|
|
# Try appending extensions to the FULL filename BEFORE checking for a
|
|
# directory import. Both TypeScript and Vite resolvers prefer a file
|
|
# match over a directory match — projects routinely have a `foo.ts`
|
|
# file living alongside a `foo/` directory of sub-modules (e.g.
|
|
# `auth.ts` next to `auth/`). If we checked the directory first, those
|
|
# file imports would silently lose to a directory with no `index.*`.
|
|
for ext in _JS_RESOLVE_EXTS:
|
|
c = p.parent / (p.name + ext)
|
|
if c.is_file():
|
|
return c
|
|
# Directory imports: try ./<name>/index.{ts,tsx,js,jsx}. Reached only
|
|
# after every file-extension candidate has been ruled out, matching the
|
|
# resolver fallback chain.
|
|
if p.is_dir():
|
|
for idx in _JS_INDEX_FILES:
|
|
c = p / idx
|
|
if c.is_file():
|
|
return c
|
|
return p
|
|
|
|
|
|
def _read_text(node, source: bytes) -> str:
|
|
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
|
|
|
|
def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None:
|
|
"""Get the name from a node using config.name_field, falling back to child types."""
|
|
if config.resolve_function_name_fn is not None:
|
|
# For C/C++ where the name is inside a declarator
|
|
return None # caller handles this separately
|
|
n = node.child_by_field_name(config.name_field)
|
|
if n:
|
|
return _read_text(n, source)
|
|
for child in node.children:
|
|
if child.type in config.name_fallback_child_types:
|
|
return _read_text(child, source)
|
|
return None
|
|
|
|
|
|
def _find_body(node, config: LanguageConfig):
|
|
"""Find the body node using config.body_field, falling back to child types."""
|
|
b = node.child_by_field_name(config.body_field)
|
|
if b:
|
|
return b
|
|
for child in node.children:
|
|
if child.type in config.body_fallback_child_types:
|
|
return child
|
|
return None
|
|
|
|
|
|
# ── Import handlers ───────────────────────────────────────────────────────────
|
|
|
|
def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
t = node.type
|
|
if t == "import_statement":
|
|
for child in node.children:
|
|
if child.type in ("dotted_name", "aliased_import"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split(" as ")[0].strip().lstrip(".")
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
elif t == "import_from_statement":
|
|
module_node = node.child_by_field_name("module_name")
|
|
if module_node:
|
|
raw = _read_text(module_node, source)
|
|
if raw.startswith("."):
|
|
# Relative import - resolve to full path so IDs match file node IDs
|
|
dots = len(raw) - len(raw.lstrip("."))
|
|
module_name = raw.lstrip(".")
|
|
base = Path(str_path).parent
|
|
for _ in range(dots - 1):
|
|
base = base.parent
|
|
rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py"
|
|
tgt_nid = _make_id(str(base / rel))
|
|
else:
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
resolved_path: "Path | None" = None
|
|
for child in node.children:
|
|
if child.type == "string":
|
|
raw = _read_text(child, source).strip("'\"` ")
|
|
if not raw:
|
|
break
|
|
if raw.startswith("."):
|
|
# Relative import - resolve to full path so IDs match file node IDs
|
|
# normpath removes ".." segments so the ID matches the target file's own node ID
|
|
resolved = Path(os.path.normpath(Path(str_path).parent / raw))
|
|
# TS / SvelteKit resolver: try .ts/.tsx/.svelte/.svelte.ts/index.{ts,…}
|
|
# so bare-path and Svelte-5-rune imports land on the right node id (#716)
|
|
resolved = _resolve_js_module_path(resolved)
|
|
tgt_nid = _make_id(str(resolved))
|
|
resolved_path = resolved
|
|
else:
|
|
# Check tsconfig.json path aliases (e.g. "@/" → "src/") before treating as external (#575)
|
|
aliases = _load_tsconfig_aliases(Path(str_path).parent)
|
|
resolved_alias = None
|
|
for alias_prefix, alias_base in aliases.items():
|
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
|
rest = raw[len(alias_prefix):].lstrip("/")
|
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
|
break
|
|
if resolved_alias is not None:
|
|
# Same resolver fixups as the relative branch — alias targets
|
|
# are equally likely to be bare paths / .svelte.ts / index.ts (#716)
|
|
resolved_alias = _resolve_js_module_path(resolved_alias)
|
|
tgt_nid = _make_id(str(resolved_alias))
|
|
resolved_path = resolved_alias
|
|
else:
|
|
# Bare/scoped import (node_modules) - use last segment; dropped as external
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
break
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
# Emit symbol-level edges for named imports from local/aliased files.
|
|
# e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED)
|
|
# Uses the same _make_id(target_stem, name) key that _extract_generic emits when
|
|
# defining the symbol, so these edges wire importers directly to existing symbol nodes.
|
|
if resolved_path is not None:
|
|
target_stem = _file_stem(resolved_path)
|
|
line = node.start_point[0] + 1
|
|
for child in node.children:
|
|
if child.type == "import_clause":
|
|
for sub in child.children:
|
|
if sub.type == "named_imports":
|
|
for spec in sub.children:
|
|
if spec.type == "import_specifier":
|
|
name_node = spec.child_by_field_name("name")
|
|
if name_node:
|
|
sym = _read_text(name_node, source)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": _make_id(target_stem, sym),
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _dynamic_import_js(node, source: bytes, caller_nid: str, str_path: str, edges: list,
|
|
seen_dyn_pairs: set) -> bool:
|
|
"""Detect dynamic import() calls in JS/TS and emit imports_from edges.
|
|
|
|
Handles patterns like:
|
|
await import('./foo.js')
|
|
import('./foo.js').then(...)
|
|
const m = await import(`./foo`)
|
|
|
|
Returns True if the node was a dynamic import (caller should skip normal call handling).
|
|
"""
|
|
# Dynamic import is a call_expression whose function child is the keyword "import".
|
|
# tree-sitter-typescript parses `import('...')` as call_expression with first child
|
|
# being an "import" token (type="import").
|
|
func_node = node.child_by_field_name("function")
|
|
if func_node is None:
|
|
# Fallback: check first child directly (some TS versions)
|
|
if node.children and _read_text(node.children[0], source) == "import":
|
|
func_node = node.children[0]
|
|
else:
|
|
return False
|
|
if _read_text(func_node, source) != "import":
|
|
return False
|
|
|
|
# Extract the module path from the arguments
|
|
args = node.child_by_field_name("arguments")
|
|
if args is None:
|
|
return True # It's an import() but no args — skip
|
|
for arg in args.children:
|
|
if arg.type == "template_string":
|
|
# Skip dynamic template literals — path can't be statically resolved
|
|
if any(c.type == "template_substitution" for c in arg.children):
|
|
break
|
|
raw = _read_text(arg, source).strip("`")
|
|
elif arg.type == "string":
|
|
raw = _read_text(arg, source).strip("'\" ")
|
|
else:
|
|
continue
|
|
if not raw:
|
|
break
|
|
# Resolve path using the same logic as static imports
|
|
if raw.startswith("."):
|
|
resolved = Path(os.path.normpath(Path(str_path).parent / raw))
|
|
if resolved.suffix == ".js":
|
|
resolved = resolved.with_suffix(".ts")
|
|
elif resolved.suffix == ".jsx":
|
|
resolved = resolved.with_suffix(".tsx")
|
|
tgt_nid = _make_id(str(resolved))
|
|
else:
|
|
aliases = _load_tsconfig_aliases(Path(str_path).parent)
|
|
resolved_alias = None
|
|
for alias_prefix, alias_base in aliases.items():
|
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
|
rest = raw[len(alias_prefix):].lstrip("/")
|
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
|
break
|
|
if resolved_alias is not None:
|
|
tgt_nid = _make_id(str(resolved_alias))
|
|
else:
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
break
|
|
tgt_nid = _make_id(module_name)
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_dyn_pairs:
|
|
seen_dyn_pairs.add(pair)
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
return True
|
|
|
|
|
|
def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
def _walk_scoped(n) -> str:
|
|
parts: list[str] = []
|
|
cur = n
|
|
while cur:
|
|
if cur.type == "scoped_identifier":
|
|
name_node = cur.child_by_field_name("name")
|
|
if name_node:
|
|
parts.append(_read_text(name_node, source))
|
|
cur = cur.child_by_field_name("scope")
|
|
elif cur.type == "identifier":
|
|
parts.append(_read_text(cur, source))
|
|
break
|
|
else:
|
|
break
|
|
parts.reverse()
|
|
return ".".join(parts)
|
|
|
|
for child in node.children:
|
|
if child.type in ("scoped_identifier", "identifier"):
|
|
path_str = _walk_scoped(child)
|
|
module_name = path_str.split(".")[-1].strip("*").strip(".") or (
|
|
path_str.split(".")[-2] if len(path_str.split(".")) > 1 else path_str
|
|
)
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type in ("string_literal", "system_lib_string", "string"):
|
|
raw = _read_text(child, source).strip('"<> ')
|
|
module_name = raw.split("/")[-1].split(".")[0]
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_csharp(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type in ("qualified_name", "identifier", "name_equals"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split(".")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_kotlin(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
path_node = node.child_by_field_name("path")
|
|
if path_node:
|
|
raw = _read_text(path_node, source)
|
|
module_name = raw.split(".")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
return
|
|
# Fallback: find identifier child
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
raw = _read_text(child, source)
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_scala(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type in ("stable_id", "identifier"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split(".")[-1].strip("{} ")
|
|
if module_name and module_name != "_":
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_php(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type in ("qualified_name", "name", "identifier"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split("\\")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
# ── C/C++ function name helpers ───────────────────────────────────────────────
|
|
|
|
def _get_c_func_name(node, source: bytes) -> str | None:
|
|
"""Recursively unwrap declarator to find the innermost identifier (C)."""
|
|
if node.type == "identifier":
|
|
return _read_text(node, source)
|
|
decl = node.child_by_field_name("declarator")
|
|
if decl:
|
|
return _get_c_func_name(decl, source)
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
return _read_text(child, source)
|
|
return None
|
|
|
|
|
|
def _get_cpp_func_name(node, source: bytes) -> str | None:
|
|
"""Recursively unwrap declarator to find the innermost identifier (C++)."""
|
|
if node.type == "identifier":
|
|
return _read_text(node, source)
|
|
if node.type == "qualified_identifier":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
return _read_text(name_node, source)
|
|
decl = node.child_by_field_name("declarator")
|
|
if decl:
|
|
return _get_cpp_func_name(decl, source)
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
return _read_text(child, source)
|
|
return None
|
|
|
|
|
|
# ── JS/TS extra walk for arrow functions ──────────────────────────────────────
|
|
|
|
def _js_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str,
|
|
nodes: list, edges: list, seen_ids: set, function_bodies: list,
|
|
parent_class_nid: str | None, add_node_fn, add_edge_fn) -> bool:
|
|
"""Handle lexical_declaration (arrow functions) for JS/TS. Returns True if handled."""
|
|
if node.type == "lexical_declaration":
|
|
for child in node.children:
|
|
if child.type == "variable_declarator":
|
|
value = child.child_by_field_name("value")
|
|
if value and value.type == "arrow_function":
|
|
name_node = child.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = child.start_point[0] + 1
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node_fn(func_nid, f"{func_name}()", line)
|
|
add_edge_fn(file_nid, func_nid, "contains", line)
|
|
body = value.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return True
|
|
return False
|
|
|
|
|
|
# ── C# extra walk for namespace declarations ──────────────────────────────────
|
|
|
|
def _csharp_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str,
|
|
nodes: list, edges: list, seen_ids: set, function_bodies: list,
|
|
parent_class_nid: str | None, add_node_fn, add_edge_fn,
|
|
walk_fn) -> bool:
|
|
"""Handle namespace_declaration for C#. Returns True if handled."""
|
|
if node.type == "namespace_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
ns_name = _read_text(name_node, source)
|
|
ns_nid = _make_id(stem, ns_name)
|
|
line = node.start_point[0] + 1
|
|
add_node_fn(ns_nid, ns_name, line)
|
|
add_edge_fn(file_nid, ns_nid, "contains", line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
for child in body.children:
|
|
walk_fn(child, parent_class_nid)
|
|
return True
|
|
return False
|
|
|
|
|
|
# ── Swift extra walk for enum cases ──────────────────────────────────────────
|
|
|
|
def _swift_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str,
|
|
nodes: list, edges: list, seen_ids: set, function_bodies: list,
|
|
parent_class_nid: str | None, add_node_fn, add_edge_fn) -> bool:
|
|
"""Handle enum_entry for Swift. Returns True if handled."""
|
|
if node.type == "enum_entry" and parent_class_nid:
|
|
for child in node.children:
|
|
if child.type == "simple_identifier":
|
|
case_name = _read_text(child, source)
|
|
case_nid = _make_id(parent_class_nid, case_name)
|
|
line = node.start_point[0] + 1
|
|
add_node_fn(case_nid, case_name, line)
|
|
add_edge_fn(parent_class_nid, case_nid, "case_of", line)
|
|
return True
|
|
return False
|
|
|
|
|
|
# ── Language configs ──────────────────────────────────────────────────────────
|
|
|
|
_PYTHON_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_python",
|
|
class_types=frozenset({"class_definition"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"import_statement", "import_from_statement"}),
|
|
call_types=frozenset({"call"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"attribute"}),
|
|
call_accessor_field="attribute",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_python,
|
|
)
|
|
|
|
_JS_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_javascript",
|
|
class_types=frozenset({"class_declaration"}),
|
|
function_types=frozenset({"function_declaration", "method_definition"}),
|
|
import_types=frozenset({"import_statement"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_expression"}),
|
|
call_accessor_field="property",
|
|
function_boundary_types=frozenset({"function_declaration", "arrow_function", "method_definition"}),
|
|
import_handler=_import_js,
|
|
)
|
|
|
|
_TS_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_typescript",
|
|
ts_language_fn="language_typescript",
|
|
class_types=frozenset({"class_declaration"}),
|
|
function_types=frozenset({"function_declaration", "method_definition"}),
|
|
import_types=frozenset({"import_statement"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_expression"}),
|
|
call_accessor_field="property",
|
|
function_boundary_types=frozenset({"function_declaration", "arrow_function", "method_definition"}),
|
|
import_handler=_import_js,
|
|
)
|
|
|
|
_JAVA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_java",
|
|
class_types=frozenset({"class_declaration", "interface_declaration"}),
|
|
function_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"method_invocation"}),
|
|
call_function_field="name",
|
|
call_accessor_node_types=frozenset(),
|
|
function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_handler=_import_java,
|
|
)
|
|
|
|
_C_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_c",
|
|
class_types=frozenset(),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"preproc_include"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"field_expression"}),
|
|
call_accessor_field="field",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_c,
|
|
resolve_function_name_fn=_get_c_func_name,
|
|
)
|
|
|
|
_CPP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_cpp",
|
|
class_types=frozenset({"class_specifier"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"preproc_include"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"field_expression", "qualified_identifier"}),
|
|
call_accessor_field="field",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_c,
|
|
resolve_function_name_fn=_get_cpp_func_name,
|
|
)
|
|
|
|
_RUBY_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_ruby",
|
|
class_types=frozenset({"class"}),
|
|
function_types=frozenset({"method", "singleton_method"}),
|
|
import_types=frozenset(),
|
|
call_types=frozenset({"call"}),
|
|
call_function_field="method",
|
|
call_accessor_node_types=frozenset(),
|
|
name_fallback_child_types=("constant", "scope_resolution", "identifier"),
|
|
body_fallback_child_types=("body_statement",),
|
|
function_boundary_types=frozenset({"method", "singleton_method"}),
|
|
)
|
|
|
|
_CSHARP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_c_sharp",
|
|
class_types=frozenset({"class_declaration", "interface_declaration"}),
|
|
function_types=frozenset({"method_declaration"}),
|
|
import_types=frozenset({"using_directive"}),
|
|
call_types=frozenset({"invocation_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_access_expression"}),
|
|
call_accessor_field="name",
|
|
body_fallback_child_types=("declaration_list",),
|
|
function_boundary_types=frozenset({"method_declaration"}),
|
|
import_handler=_import_csharp,
|
|
)
|
|
|
|
_KOTLIN_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_kotlin",
|
|
class_types=frozenset({"class_declaration", "object_declaration"}),
|
|
function_types=frozenset({"function_declaration"}),
|
|
import_types=frozenset({"import_header"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"navigation_expression"}),
|
|
call_accessor_field="",
|
|
# Different tree-sitter-kotlin grammar versions name plain identifier
|
|
# nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`,
|
|
# older forks use `simple_identifier`. Accept both so the extractor
|
|
# works across grammar generations.
|
|
name_fallback_child_types=("simple_identifier", "identifier"),
|
|
body_fallback_child_types=("function_body", "class_body"),
|
|
function_boundary_types=frozenset({"function_declaration"}),
|
|
import_handler=_import_kotlin,
|
|
)
|
|
|
|
_SCALA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_scala",
|
|
class_types=frozenset({"class_definition", "object_definition"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"field_expression"}),
|
|
call_accessor_field="field",
|
|
name_fallback_child_types=("identifier",),
|
|
body_fallback_child_types=("template_body",),
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_scala,
|
|
)
|
|
|
|
_PHP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_php",
|
|
ts_language_fn="language_php",
|
|
class_types=frozenset({"class_declaration"}),
|
|
function_types=frozenset({"function_definition", "method_declaration"}),
|
|
import_types=frozenset({"namespace_use_clause"}),
|
|
call_types=frozenset({"function_call_expression", "member_call_expression", "scoped_call_expression", "class_constant_access_expression"}),
|
|
static_prop_types=frozenset({"scoped_property_access_expression"}),
|
|
helper_fn_names=frozenset({"config"}),
|
|
container_bind_methods=frozenset({"bind", "singleton", "scoped", "instance"}),
|
|
event_listener_properties=frozenset({"listen", "subscribe"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_call_expression"}),
|
|
call_accessor_field="name",
|
|
name_fallback_child_types=("name",),
|
|
body_fallback_child_types=("declaration_list", "compound_statement"),
|
|
function_boundary_types=frozenset({"function_definition", "method_declaration"}),
|
|
import_handler=_import_php,
|
|
)
|
|
|
|
|
|
def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
"""Extract require('module') from Lua variable_declaration nodes."""
|
|
text = _read_text(node, source)
|
|
import re
|
|
m = re.search(r"""require\s*[\('"]\s*['"]?([^'")\s]+)""", text)
|
|
if m:
|
|
module_name = m.group(1).split(".")[-1]
|
|
if module_name:
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": module_name,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": str(node.start_point[0] + 1),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
_LUA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_lua",
|
|
ts_language_fn="language",
|
|
class_types=frozenset(),
|
|
function_types=frozenset({"function_declaration"}),
|
|
import_types=frozenset({"variable_declaration"}),
|
|
call_types=frozenset({"function_call"}),
|
|
call_function_field="name",
|
|
call_accessor_node_types=frozenset({"method_index_expression"}),
|
|
call_accessor_field="name",
|
|
name_fallback_child_types=("identifier", "method_index_expression"),
|
|
body_fallback_child_types=("block",),
|
|
function_boundary_types=frozenset({"function_declaration"}),
|
|
import_handler=_import_lua,
|
|
)
|
|
|
|
|
|
def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
raw = _read_text(child, source)
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _read_csharp_type_name(node, source: bytes) -> str | None:
|
|
"""Resolve a readable C# type name from a field/type node."""
|
|
if node is None:
|
|
return None
|
|
if node.type in ("identifier", "predefined_type"):
|
|
return _read_text(node, source)
|
|
if node.type == "qualified_name":
|
|
return _read_text(node, source).split(".")[-1]
|
|
if node.type == "generic_name":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node is not None:
|
|
return _read_text(name_node, source)
|
|
for child in node.children:
|
|
if not child.is_named:
|
|
continue
|
|
name = _read_csharp_type_name(child, source)
|
|
if name:
|
|
return name
|
|
return None
|
|
|
|
|
|
_SWIFT_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_swift",
|
|
class_types=frozenset({"class_declaration", "protocol_declaration"}),
|
|
function_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"navigation_expression"}),
|
|
call_accessor_field="",
|
|
name_fallback_child_types=("simple_identifier", "type_identifier", "user_type"),
|
|
body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"),
|
|
function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}),
|
|
import_handler=_import_swift,
|
|
)
|
|
|
|
# ── Generic extractor ─────────────────────────────────────────────────────────
|
|
|
|
def _extract_generic(path: Path, config: LanguageConfig) -> dict:
|
|
"""Generic AST extractor driven by LanguageConfig."""
|
|
try:
|
|
mod = importlib.import_module(config.ts_module)
|
|
from tree_sitter import Language, Parser
|
|
lang_fn = getattr(mod, config.ts_language_fn, None)
|
|
if lang_fn is None:
|
|
# Fallback for PHP: try "language_php" then "language"
|
|
lang_fn = getattr(mod, "language", None)
|
|
if lang_fn is None:
|
|
return {"nodes": [], "edges": [], "error": f"No language function in {config.ts_module}"}
|
|
language = Language(lang_fn())
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": f"{config.ts_module} not installed"}
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
try:
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, object]] = []
|
|
pending_listen_edges: list[tuple[str, str, int]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
def ensure_named_node(name: str, line: int) -> str:
|
|
nid = _make_id(stem, name)
|
|
if nid in seen_ids:
|
|
return nid
|
|
nid = _make_id(name)
|
|
if nid not in seen_ids:
|
|
add_node(nid, name, line)
|
|
return nid
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def walk(node, parent_class_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
# Import types
|
|
if t in config.import_types:
|
|
if config.import_handler:
|
|
config.import_handler(node, source, file_nid, stem, edges, str_path)
|
|
return
|
|
|
|
# Class types
|
|
if t in config.class_types:
|
|
# Resolve class name
|
|
name_node = node.child_by_field_name(config.name_field)
|
|
if name_node is None:
|
|
for child in node.children:
|
|
if child.type in config.name_fallback_child_types:
|
|
name_node = child
|
|
break
|
|
if not name_node:
|
|
return
|
|
class_name = _read_text(name_node, source)
|
|
class_nid = _make_id(stem, class_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(class_nid, class_name, line)
|
|
add_edge(file_nid, class_nid, "contains", line)
|
|
|
|
# Python-specific: inheritance
|
|
if config.ts_module == "tree_sitter_python":
|
|
args = node.child_by_field_name("superclasses")
|
|
if args:
|
|
for arg in args.children:
|
|
if arg.type == "identifier":
|
|
base = _read_text(arg, source)
|
|
base_nid = _make_id(stem, base)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, "inherits", line)
|
|
|
|
# Swift-specific: conformance / inheritance
|
|
if config.ts_module == "tree_sitter_swift":
|
|
for child in node.children:
|
|
if child.type == "inheritance_specifier":
|
|
for sub in child.children:
|
|
if sub.type in ("user_type", "type_identifier"):
|
|
base = _read_text(sub, source)
|
|
base_nid = _make_id(stem, base)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, "inherits", line)
|
|
|
|
# C#-specific: inheritance / interface implementation via base_list
|
|
if config.ts_module == "tree_sitter_c_sharp":
|
|
for child in node.children:
|
|
if child.type == "base_list":
|
|
for sub in child.children:
|
|
if sub.type in ("identifier", "generic_name"):
|
|
if sub.type == "generic_name":
|
|
name_child = sub.child_by_field_name("name")
|
|
base = _read_text(name_child, source) if name_child else _read_text(sub.children[0], source)
|
|
else:
|
|
base = _read_text(sub, source)
|
|
base_nid = _make_id(stem, base)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, "inherits", line)
|
|
|
|
# Java-specific: extends (superclass) / implements (interfaces) / interface-extends
|
|
if config.ts_module == "tree_sitter_java":
|
|
def _emit_java_parent(base_name: str, rel: str, at_line: int) -> None:
|
|
if not base_name:
|
|
return
|
|
base_nid = _make_id(stem, base_name)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base_name)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base_name,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, rel, at_line)
|
|
|
|
sup = node.child_by_field_name("superclass")
|
|
if sup is not None:
|
|
for sub in sup.children:
|
|
if sub.type == "type_identifier":
|
|
_emit_java_parent(_read_text(sub, source), "extends", line)
|
|
break
|
|
|
|
ifs = node.child_by_field_name("interfaces")
|
|
if ifs is not None:
|
|
for sub in ifs.children:
|
|
if sub.type == "type_list":
|
|
for tid in sub.children:
|
|
if tid.type == "type_identifier":
|
|
_emit_java_parent(_read_text(tid, source), "implements", line)
|
|
|
|
if t == "interface_declaration":
|
|
for child in node.children:
|
|
if child.type == "extends_interfaces":
|
|
for sub in child.children:
|
|
if sub.type == "type_list":
|
|
for tid in sub.children:
|
|
if tid.type == "type_identifier":
|
|
_emit_java_parent(_read_text(tid, source), "extends", line)
|
|
|
|
# Find body and recurse
|
|
body = _find_body(node, config)
|
|
if body:
|
|
for child in body.children:
|
|
walk(child, parent_class_nid=class_nid)
|
|
return
|
|
|
|
# Event listener property arrays: $listen = [Event::class => [Listener::class]]
|
|
if (t == "property_declaration"
|
|
and parent_class_nid
|
|
and config.event_listener_properties):
|
|
for element in node.children:
|
|
if element.type != "property_element":
|
|
continue
|
|
prop_name: str | None = None
|
|
array_node = None
|
|
for c in element.children:
|
|
if c.type == "variable_name":
|
|
for sc in c.children:
|
|
if sc.type == "name":
|
|
prop_name = _read_text(sc, source)
|
|
break
|
|
elif c.type == "array_creation_expression":
|
|
array_node = c
|
|
if (prop_name is None
|
|
or prop_name not in config.event_listener_properties
|
|
or array_node is None):
|
|
continue
|
|
for entry in array_node.children:
|
|
if entry.type != "array_element_initializer":
|
|
continue
|
|
event_cls: str | None = None
|
|
listener_arr = None
|
|
for sub in entry.children:
|
|
if sub.type == "class_constant_access_expression" and event_cls is None:
|
|
for sc in sub.children:
|
|
if sc.is_named and sc.type in ("name", "qualified_name"):
|
|
event_cls = _read_text(sc, source)
|
|
break
|
|
elif sub.type == "array_creation_expression":
|
|
listener_arr = sub
|
|
if not event_cls or listener_arr is None:
|
|
continue
|
|
for listener_entry in listener_arr.children:
|
|
if listener_entry.type != "array_element_initializer":
|
|
continue
|
|
for item in listener_entry.children:
|
|
if item.type != "class_constant_access_expression":
|
|
continue
|
|
for sc in item.children:
|
|
if sc.is_named and sc.type in ("name", "qualified_name"):
|
|
listener_cls = _read_text(sc, source)
|
|
line_no = item.start_point[0] + 1
|
|
pending_listen_edges.append((event_cls, listener_cls, line_no))
|
|
break
|
|
break
|
|
return
|
|
|
|
if (config.ts_module == "tree_sitter_c_sharp"
|
|
and t == "field_declaration"
|
|
and parent_class_nid):
|
|
type_node = node.child_by_field_name("type")
|
|
if type_node is None:
|
|
for child in node.children:
|
|
if child.type == "variable_declaration":
|
|
type_node = child.child_by_field_name("type")
|
|
if type_node is not None:
|
|
break
|
|
type_name = _read_csharp_type_name(type_node, source)
|
|
if type_name:
|
|
line = node.start_point[0] + 1
|
|
add_edge(parent_class_nid, ensure_named_node(type_name, line),
|
|
"references", line, context="field")
|
|
return
|
|
|
|
# Function types
|
|
if t in config.function_types:
|
|
# Swift deinit/subscript have no name field — resolve before generic fallback
|
|
if t == "deinit_declaration":
|
|
func_name: str | None = "deinit"
|
|
elif t == "subscript_declaration":
|
|
func_name = "subscript"
|
|
elif config.resolve_function_name_fn is not None:
|
|
# C/C++ style: use declarator
|
|
declarator = node.child_by_field_name("declarator")
|
|
func_name = None
|
|
if declarator:
|
|
func_name = config.resolve_function_name_fn(declarator, source)
|
|
else:
|
|
name_node = node.child_by_field_name(config.name_field)
|
|
if name_node is None:
|
|
for child in node.children:
|
|
if child.type in config.name_fallback_child_types:
|
|
name_node = child
|
|
break
|
|
func_name = _read_text(name_node, source) if name_node else None
|
|
|
|
if not func_name:
|
|
return
|
|
|
|
line = node.start_point[0] + 1
|
|
if parent_class_nid:
|
|
func_nid = _make_id(parent_class_nid, func_name)
|
|
add_node(func_nid, f".{func_name}()", line)
|
|
add_edge(parent_class_nid, func_nid, "method", line)
|
|
else:
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
|
|
body = _find_body(node, config)
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
# JS/TS arrow functions and C# namespaces — language-specific extra handling
|
|
if config.ts_module in ("tree_sitter_javascript", "tree_sitter_typescript"):
|
|
if _js_extra_walk(node, source, file_nid, stem, str_path,
|
|
nodes, edges, seen_ids, function_bodies,
|
|
parent_class_nid, add_node, add_edge):
|
|
return
|
|
|
|
if config.ts_module == "tree_sitter_c_sharp":
|
|
if _csharp_extra_walk(node, source, file_nid, stem, str_path,
|
|
nodes, edges, seen_ids, function_bodies,
|
|
parent_class_nid, add_node, add_edge, walk):
|
|
return
|
|
|
|
if config.ts_module == "tree_sitter_swift":
|
|
if _swift_extra_walk(node, source, file_nid, stem, str_path,
|
|
nodes, edges, seen_ids, function_bodies,
|
|
parent_class_nid, add_node, add_edge):
|
|
return
|
|
|
|
# Default: recurse
|
|
for child in node.children:
|
|
walk(child, parent_class_nid=None)
|
|
|
|
walk(root)
|
|
|
|
# ── Call-graph pass ───────────────────────────────────────────────────────
|
|
label_to_nid: dict[str, str] = {}
|
|
for n in nodes:
|
|
raw = n["label"]
|
|
normalised = raw.strip("()").lstrip(".")
|
|
label_to_nid[normalised.lower()] = n["id"]
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
seen_dyn_import_pairs: set[tuple[str, str]] = set()
|
|
seen_static_ref_pairs: set[tuple[str, str, str]] = set()
|
|
seen_helper_ref_pairs: set[tuple[str, str, str]] = set()
|
|
seen_bind_pairs: set[tuple[str, str, str]] = set()
|
|
raw_calls: list[dict] = [] # unresolved calls for cross-file resolution in extract()
|
|
|
|
def _php_class_const_scope(n) -> str | None:
|
|
scope = n.child_by_field_name("scope")
|
|
if scope is None:
|
|
for c in n.children:
|
|
if c.is_named and c.type in ("name", "qualified_name", "identifier"):
|
|
scope = c
|
|
break
|
|
if scope is None:
|
|
return None
|
|
return _read_text(scope, source)
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type in config.function_boundary_types:
|
|
return
|
|
|
|
if node.type in config.call_types:
|
|
# JS/TS dynamic imports: await import('./foo.js')
|
|
if config.ts_module in ("tree_sitter_javascript", "tree_sitter_typescript"):
|
|
if _dynamic_import_js(node, source, caller_nid, str_path,
|
|
edges, seen_dyn_import_pairs):
|
|
# Still recurse into children (import().then(...) may have calls)
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
return
|
|
|
|
callee_name: str | None = None
|
|
is_member_call: bool = False
|
|
|
|
# Special handling per language
|
|
if config.ts_module == "tree_sitter_swift":
|
|
# Swift: first child may be simple_identifier or navigation_expression
|
|
first = node.children[0] if node.children else None
|
|
if first:
|
|
if first.type == "simple_identifier":
|
|
callee_name = _read_text(first, source)
|
|
elif first.type == "navigation_expression":
|
|
is_member_call = True
|
|
for child in first.children:
|
|
if child.type == "navigation_suffix":
|
|
for sc in child.children:
|
|
if sc.type == "simple_identifier":
|
|
callee_name = _read_text(sc, source)
|
|
elif config.ts_module == "tree_sitter_kotlin":
|
|
# Kotlin: first child may be simple_identifier/identifier or
|
|
# navigation_expression. PyPI's `tree_sitter_kotlin` produces
|
|
# `identifier` for plain identifier nodes; older grammar
|
|
# versions (including the JVM `io.github.bonede:tree-sitter-kotlin`
|
|
# binding) produce `simple_identifier`. Accept both.
|
|
first = node.children[0] if node.children else None
|
|
if first:
|
|
if first.type in ("simple_identifier", "identifier"):
|
|
callee_name = _read_text(first, source)
|
|
elif first.type == "navigation_expression":
|
|
is_member_call = True
|
|
for child in reversed(first.children):
|
|
if child.type in ("simple_identifier", "identifier"):
|
|
callee_name = _read_text(child, source)
|
|
break
|
|
elif config.ts_module == "tree_sitter_scala":
|
|
# Scala: first child
|
|
first = node.children[0] if node.children else None
|
|
if first:
|
|
if first.type == "identifier":
|
|
callee_name = _read_text(first, source)
|
|
elif first.type == "field_expression":
|
|
is_member_call = True
|
|
field = first.child_by_field_name("field")
|
|
if field:
|
|
callee_name = _read_text(field, source)
|
|
else:
|
|
for child in reversed(first.children):
|
|
if child.type == "identifier":
|
|
callee_name = _read_text(child, source)
|
|
break
|
|
elif config.ts_module == "tree_sitter_c_sharp" and node.type == "invocation_expression":
|
|
# C#: try name field, then first named child
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
callee_name = _read_text(name_node, source)
|
|
else:
|
|
for child in node.children:
|
|
if child.is_named:
|
|
raw = _read_text(child, source)
|
|
if "." in raw:
|
|
callee_name = raw.split(".")[-1]
|
|
is_member_call = True
|
|
else:
|
|
callee_name = raw
|
|
break
|
|
elif config.ts_module == "tree_sitter_php":
|
|
# PHP: distinguish call expression subtypes
|
|
if node.type == "function_call_expression":
|
|
func_node = node.child_by_field_name("function")
|
|
if func_node:
|
|
callee_name = _read_text(func_node, source)
|
|
elif node.type == "scoped_call_expression":
|
|
# Static method call: Helper::format() → callee = "Helper"
|
|
scope_node = node.child_by_field_name("scope")
|
|
if scope_node:
|
|
callee_name = _read_text(scope_node, source)
|
|
else:
|
|
# member_call_expression: $obj->method()
|
|
is_member_call = True
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
callee_name = _read_text(name_node, source)
|
|
elif config.ts_module == "tree_sitter_cpp":
|
|
# C++: function field, then field_expression/qualified_identifier
|
|
func_node = node.child_by_field_name(config.call_function_field) if config.call_function_field else None
|
|
if func_node:
|
|
if func_node.type == "identifier":
|
|
callee_name = _read_text(func_node, source)
|
|
elif func_node.type in ("field_expression", "qualified_identifier"):
|
|
is_member_call = True
|
|
name = func_node.child_by_field_name("field") or func_node.child_by_field_name("name")
|
|
if name:
|
|
callee_name = _read_text(name, source)
|
|
else:
|
|
# Generic: get callee from call_function_field
|
|
func_node = node.child_by_field_name(config.call_function_field) if config.call_function_field else None
|
|
if func_node:
|
|
if func_node.type == "identifier":
|
|
callee_name = _read_text(func_node, source)
|
|
elif func_node.type in config.call_accessor_node_types:
|
|
is_member_call = True
|
|
if config.call_accessor_field:
|
|
attr = func_node.child_by_field_name(config.call_accessor_field)
|
|
if attr:
|
|
callee_name = _read_text(attr, source)
|
|
else:
|
|
# Try reading the node directly (e.g. Java name field is the callee)
|
|
callee_name = _read_text(func_node, source)
|
|
|
|
if callee_name:
|
|
tgt_nid = label_to_nid.get(callee_name.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
elif callee_name and not tgt_nid:
|
|
# Callee not in this file — save for cross-file resolution in extract()
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee_name,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
|
|
# Helper function calls: config('foo.bar') → uses_config edge to "foo"
|
|
if (callee_name and callee_name in config.helper_fn_names):
|
|
args_node = node.child_by_field_name("arguments")
|
|
first_key: str | None = None
|
|
if args_node:
|
|
for arg in args_node.children:
|
|
if arg.type != "argument":
|
|
continue
|
|
for inner in arg.children:
|
|
if inner.type == "string":
|
|
for sc in inner.children:
|
|
if sc.type == "string_content":
|
|
first_key = _read_text(sc, source)
|
|
break
|
|
break
|
|
if first_key:
|
|
break
|
|
if first_key:
|
|
segment = first_key.split(".")[0]
|
|
tgt_nid = (label_to_nid.get(segment.lower())
|
|
or label_to_nid.get(f"{segment}.php".lower()))
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
relation = f"uses_{callee_name}"
|
|
pair3 = (caller_nid, tgt_nid, relation)
|
|
if pair3 not in seen_helper_ref_pairs:
|
|
seen_helper_ref_pairs.add(pair3)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": relation,
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Service container bindings: $this->app->bind(Foo::class, Bar::class)
|
|
if (node.type == "member_call_expression"
|
|
and callee_name
|
|
and callee_name in config.container_bind_methods):
|
|
args_node = node.child_by_field_name("arguments")
|
|
class_args: list[str] = []
|
|
if args_node:
|
|
for arg in args_node.children:
|
|
if arg.type != "argument":
|
|
continue
|
|
for inner in arg.children:
|
|
if inner.type == "class_constant_access_expression":
|
|
cls = _php_class_const_scope(inner)
|
|
if cls:
|
|
class_args.append(cls)
|
|
break
|
|
if len(class_args) >= 2:
|
|
break
|
|
if len(class_args) == 2:
|
|
contract_name, impl_name = class_args
|
|
contract_nid = label_to_nid.get(contract_name.lower())
|
|
impl_nid = label_to_nid.get(impl_name.lower())
|
|
if contract_nid and impl_nid and contract_nid != impl_nid:
|
|
pair3 = (contract_nid, impl_nid, "bound_to")
|
|
if pair3 not in seen_bind_pairs:
|
|
seen_bind_pairs.add(pair3)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": contract_nid,
|
|
"target": impl_nid,
|
|
"relation": "bound_to",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Static property access: Foo::$bar → uses_static_prop edge
|
|
if node.type in config.static_prop_types:
|
|
scope_node = node.child_by_field_name("scope")
|
|
if scope_node is None:
|
|
for child in node.children:
|
|
if child.is_named and child.type in ("name", "qualified_name", "identifier"):
|
|
scope_node = child
|
|
break
|
|
if scope_node is not None:
|
|
class_name = _read_text(scope_node, source)
|
|
tgt_nid = label_to_nid.get(class_name.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair3 = (caller_nid, tgt_nid, "uses_static_prop")
|
|
if pair3 not in seen_static_ref_pairs:
|
|
seen_static_ref_pairs.add(pair3)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "uses_static_prop",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# PHP class constant access: Foo::BAR → references_constant edge
|
|
if config.ts_module == "tree_sitter_php" and node.type == "class_constant_access_expression":
|
|
class_name = _php_class_const_scope(node)
|
|
if class_name:
|
|
tgt_nid = label_to_nid.get(class_name.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair3 = (caller_nid, tgt_nid, "references_constant")
|
|
if pair3 not in seen_static_ref_pairs:
|
|
seen_static_ref_pairs.add(pair3)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "references_constant",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
# ── Event listener pass ───────────────────────────────────────────────────
|
|
seen_listen_pairs: set[tuple[str, str]] = set()
|
|
for event_name, listener_name, line in pending_listen_edges:
|
|
event_nid = label_to_nid.get(event_name.lower())
|
|
listener_nid = label_to_nid.get(listener_name.lower())
|
|
if not event_nid or not listener_nid or event_nid == listener_nid:
|
|
continue
|
|
pair2 = (event_nid, listener_nid)
|
|
if pair2 in seen_listen_pairs:
|
|
continue
|
|
seen_listen_pairs.add(pair2)
|
|
edges.append({
|
|
"source": event_nid,
|
|
"target": listener_nid,
|
|
"relation": "listened_by",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# ── Clean edges ───────────────────────────────────────────────────────────
|
|
valid_ids = seen_ids
|
|
clean_edges = []
|
|
for edge in edges:
|
|
src, tgt = edge["source"], edge["target"]
|
|
if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):
|
|
clean_edges.append(edge)
|
|
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── Python rationale extraction ───────────────────────────────────────────────
|
|
|
|
_RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:")
|
|
|
|
|
|
def _extract_python_rationale(path: Path, result: dict) -> None:
|
|
"""Post-pass: extract docstrings and rationale comments from Python source.
|
|
Mutates result in-place by appending to result['nodes'] and result['edges'].
|
|
"""
|
|
try:
|
|
import tree_sitter_python as tspython
|
|
from tree_sitter import Language, Parser
|
|
language = Language(tspython.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception:
|
|
return
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes = result["nodes"]
|
|
edges = result["edges"]
|
|
seen_ids = {n["id"] for n in nodes}
|
|
file_nid = _make_id(str(path))
|
|
|
|
def _get_docstring(body_node) -> tuple[str, int] | None:
|
|
if not body_node:
|
|
return None
|
|
for child in body_node.children:
|
|
if child.type == "expression_statement":
|
|
for sub in child.children:
|
|
if sub.type in ("string", "concatenated_string"):
|
|
text = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
|
|
text = text.strip("\"'").strip('"""').strip("'''").strip()
|
|
if len(text) > 20:
|
|
return text, child.start_point[0] + 1
|
|
break
|
|
return None
|
|
|
|
def _add_rationale(text: str, line: int, parent_nid: str) -> None:
|
|
label = text[:80].replace("\r\n", " ").replace("\r", " ").replace("\n", " ").strip()
|
|
rid = _make_id(stem, "rationale", str(line))
|
|
if rid not in seen_ids:
|
|
seen_ids.add(rid)
|
|
nodes.append({
|
|
"id": rid,
|
|
"label": label,
|
|
"file_type": "rationale",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
edges.append({
|
|
"source": rid,
|
|
"target": parent_nid,
|
|
"relation": "rationale_for",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Module-level docstring
|
|
ds = _get_docstring(root)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], file_nid)
|
|
|
|
# Class and function docstrings
|
|
def walk_docstrings(node, parent_nid: str) -> None:
|
|
t = node.type
|
|
if t == "class_definition":
|
|
name_node = node.child_by_field_name("name")
|
|
body = node.child_by_field_name("body")
|
|
if name_node and body:
|
|
class_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
nid = _make_id(stem, class_name)
|
|
ds = _get_docstring(body)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], nid)
|
|
for child in body.children:
|
|
walk_docstrings(child, nid)
|
|
return
|
|
if t == "function_definition":
|
|
name_node = node.child_by_field_name("name")
|
|
body = node.child_by_field_name("body")
|
|
if name_node and body:
|
|
func_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
nid = _make_id(parent_nid, func_name) if parent_nid != file_nid else _make_id(stem, func_name)
|
|
ds = _get_docstring(body)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], nid)
|
|
return
|
|
for child in node.children:
|
|
walk_docstrings(child, parent_nid)
|
|
|
|
walk_docstrings(root, file_nid)
|
|
|
|
# Rationale comments (# NOTE:, # IMPORTANT:, etc.)
|
|
source_text = source.decode("utf-8", errors="replace")
|
|
for lineno, line_text in enumerate(source_text.splitlines(), start=1):
|
|
stripped = line_text.strip()
|
|
if any(stripped.startswith(p) for p in _RATIONALE_PREFIXES):
|
|
_add_rationale(stripped, lineno, file_nid)
|
|
|
|
|
|
# ── Public API ────────────────────────────────────────────────────────────────
|
|
|
|
def extract_python(path: Path) -> dict:
|
|
"""Extract classes, functions, and imports from a .py file via tree-sitter AST."""
|
|
result = _extract_generic(path, _PYTHON_CONFIG)
|
|
if "error" not in result:
|
|
_extract_python_rationale(path, result)
|
|
return result
|
|
|
|
|
|
def extract_js(path: Path) -> dict:
|
|
"""Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx file."""
|
|
config = _TS_CONFIG if path.suffix in (".ts", ".tsx") else _JS_CONFIG
|
|
return _extract_generic(path, config)
|
|
|
|
|
|
def extract_svelte(path: Path) -> dict:
|
|
"""Extract imports from .svelte files: script-block via JS AST + template regex fallback.
|
|
|
|
Tree-sitter only sees the <script> block. Svelte template syntax like
|
|
{#await import('./X.svelte')} lives in the markup layer and is invisible
|
|
to the JS parser, so a regex pass covers those dynamic imports.
|
|
"""
|
|
result = _extract_generic(path, _JS_CONFIG)
|
|
try:
|
|
import re as _re
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
existing_ids = {n["id"] for n in result.get("nodes", [])}
|
|
# Source file node ID must match the one _extract_generic creates:
|
|
# _make_id(str(path)) - single arg, no stem prefix. Otherwise the source
|
|
# endpoint is a phantom node and build_from_json drops the edge (#701).
|
|
file_node_id = _make_id(str(path))
|
|
aliases = _load_tsconfig_aliases(path.parent)
|
|
for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
# Relative import - resolve to full path so IDs match file node IDs.
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
# Apply same TS/Svelte resolver fixups as static imports so dynamic
|
|
# imports of bare paths and .svelte.ts rune files land on real
|
|
# file nodes instead of phantom ids (#716).
|
|
resolved = _resolve_js_module_path(resolved)
|
|
node_id = _make_id(str(resolved))
|
|
else:
|
|
# Check tsconfig.json path aliases (e.g. "$lib/" -> "src/lib/", "@/" -> "src/")
|
|
# before treating as external. Mirrors _import_js logic so SvelteKit alias
|
|
# imports resolve to the same file node IDs the extractor creates (#701).
|
|
resolved_alias = None
|
|
for alias_prefix, alias_base in aliases.items():
|
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
|
rest = raw[len(alias_prefix):].lstrip("/")
|
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
|
break
|
|
if resolved_alias is not None:
|
|
resolved_alias = _resolve_js_module_path(resolved_alias)
|
|
node_id = _make_id(str(resolved_alias))
|
|
else:
|
|
# Bare/scoped import (node_modules) - use last segment;
|
|
# build_from_json drops as external if no matching node exists.
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
if node_id in existing_ids:
|
|
# Edge target already a real node - just add the edge, don't add a node.
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": str(path),
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
except Exception:
|
|
pass
|
|
return result
|
|
|
|
|
|
def extract_java(path: Path) -> dict:
|
|
"""Extract classes, interfaces, methods, constructors, and imports from a .java file."""
|
|
return _extract_generic(path, _JAVA_CONFIG)
|
|
|
|
|
|
def extract_c(path: Path) -> dict:
|
|
"""Extract functions and includes from a .c/.h file."""
|
|
return _extract_generic(path, _C_CONFIG)
|
|
|
|
|
|
def extract_cpp(path: Path) -> dict:
|
|
"""Extract functions, classes, and includes from a .cpp/.cc/.cxx/.hpp file."""
|
|
return _extract_generic(path, _CPP_CONFIG)
|
|
|
|
|
|
def extract_ruby(path: Path) -> dict:
|
|
"""Extract classes, methods, singleton methods, and calls from a .rb file."""
|
|
return _extract_generic(path, _RUBY_CONFIG)
|
|
|
|
|
|
def extract_csharp(path: Path) -> dict:
|
|
"""Extract classes, interfaces, methods, namespaces, and usings from a .cs file."""
|
|
return _extract_generic(path, _CSHARP_CONFIG)
|
|
|
|
|
|
def extract_kotlin(path: Path) -> dict:
|
|
"""Extract classes, objects, functions, and imports from a .kt/.kts file."""
|
|
return _extract_generic(path, _KOTLIN_CONFIG)
|
|
|
|
|
|
def extract_scala(path: Path) -> dict:
|
|
"""Extract classes, objects, functions, and imports from a .scala file."""
|
|
return _extract_generic(path, _SCALA_CONFIG)
|
|
|
|
|
|
def extract_php(path: Path) -> dict:
|
|
"""Extract classes, functions, methods, namespace uses, and calls from a .php file."""
|
|
return _extract_generic(path, _PHP_CONFIG)
|
|
|
|
|
|
def extract_blade(path: Path) -> dict:
|
|
"""Extract @include, <livewire:> components, and wire:click bindings from Blade templates."""
|
|
import re
|
|
try:
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return {"error": f"cannot read {path}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
nodes = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str(path), "source_location": None}]
|
|
edges = []
|
|
|
|
# @include('path.to.partial') or @include("path.to.partial")
|
|
for m in re.finditer(r"@include\(['\"]([^'\"]+)['\"]", src):
|
|
tgt = m.group(1).replace(".", "/")
|
|
tgt_nid = _make_id(tgt)
|
|
if tgt_nid not in {n["id"] for n in nodes}:
|
|
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges.append({"source": file_nid, "target": tgt_nid, "relation": "includes",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
|
|
# <livewire:component.name /> or <livewire:component.name>
|
|
for m in re.finditer(r"<livewire:([\w.\-]+)", src):
|
|
tgt_nid = _make_id(m.group(1))
|
|
if tgt_nid not in {n["id"] for n in nodes}:
|
|
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges.append({"source": file_nid, "target": tgt_nid, "relation": "uses_component",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
|
|
# wire:click="methodName"
|
|
for m in re.finditer(r'wire:click=["\']([^"\']+)["\']', src):
|
|
tgt_nid = _make_id(m.group(1))
|
|
if tgt_nid not in {n["id"] for n in nodes}:
|
|
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges.append({"source": file_nid, "target": tgt_nid, "relation": "binds_method",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_dart(path: Path) -> dict:
|
|
"""Extract classes, mixins, functions, imports, and calls from a .dart file using regex."""
|
|
try:
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return {"error": f"cannot read {path}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
nodes = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str(path), "source_location": None}]
|
|
edges = []
|
|
defined: set[str] = set()
|
|
|
|
# Classes and mixins
|
|
for m in re.finditer(r"^\s*(?:abstract\s+)?(?:class|mixin)\s+(\w+)", src, re.MULTILINE):
|
|
nid = _make_id(str(path), m.group(1))
|
|
if nid not in defined:
|
|
nodes.append({"id": nid, "label": m.group(1), "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges.append({"source": file_nid, "target": nid, "relation": "defines",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
defined.add(nid)
|
|
|
|
# Top-level and member functions/methods
|
|
for m in re.finditer(r"^\s*(?:static\s+|async\s+)?(?:\w+\s+)+(\w+)\s*\(", src, re.MULTILINE):
|
|
name = m.group(1)
|
|
if name in {"if", "for", "while", "switch", "catch", "return"}:
|
|
continue
|
|
nid = _make_id(str(path), name)
|
|
if nid not in defined:
|
|
nodes.append({"id": nid, "label": name, "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges.append({"source": file_nid, "target": nid, "relation": "defines",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
defined.add(nid)
|
|
|
|
# import 'package:...' or import '...'
|
|
for m in re.finditer(r"""^import\s+['"]([^'"]+)['"]""", src, re.MULTILINE):
|
|
pkg = m.group(1)
|
|
tgt_nid = _make_id(pkg)
|
|
if tgt_nid not in defined:
|
|
nodes.append({"id": tgt_nid, "label": pkg, "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
defined.add(tgt_nid)
|
|
edges.append({"source": file_nid, "target": tgt_nid, "relation": "imports",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_verilog(path: Path) -> dict:
|
|
"""Extract modules, functions, tasks, package imports, and instantiations from .v/.sv files."""
|
|
try:
|
|
import tree_sitter_verilog as tsverilog
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_verilog not installed"}
|
|
|
|
try:
|
|
language = Language(tsverilog.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}",
|
|
"confidence_score": 1.0})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", score: float = 1.0) -> None:
|
|
edges.append({"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "confidence_score": score,
|
|
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0})
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def walk(node, module_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
if t == "module_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
mod_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
nid = _make_id(stem, mod_name)
|
|
add_node(nid, mod_name, line)
|
|
add_edge(file_nid, nid, "defines", line)
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
elif t in ("function_declaration", "function_prototype"):
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
parent = module_nid or file_nid
|
|
nid = _make_id(parent, func_name)
|
|
add_node(nid, f"{func_name}()", line)
|
|
add_edge(parent, nid, "contains", line)
|
|
|
|
elif t == "task_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
task_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
parent = module_nid or file_nid
|
|
nid = _make_id(parent, task_name)
|
|
add_node(nid, task_name, line)
|
|
add_edge(parent, nid, "contains", line)
|
|
|
|
elif t == "package_import_declaration":
|
|
for child in node.children:
|
|
if child.type == "package_import_item":
|
|
pkg_text = _read_text(child, source)
|
|
pkg_name = pkg_text.split("::")[0].strip()
|
|
if pkg_name:
|
|
line = node.start_point[0] + 1
|
|
tgt_nid = _make_id(pkg_name)
|
|
add_node(tgt_nid, pkg_name, line)
|
|
src = module_nid or file_nid
|
|
add_edge(src, tgt_nid, "imports_from", line)
|
|
|
|
elif t == "module_instantiation":
|
|
# module_type instantiates another module
|
|
type_node = node.child_by_field_name("module_type")
|
|
if type_node and module_nid:
|
|
inst_type = _read_text(type_node, source).strip()
|
|
if inst_type:
|
|
line = node.start_point[0] + 1
|
|
tgt_nid = _make_id(inst_type)
|
|
add_node(tgt_nid, inst_type, line)
|
|
add_edge(module_nid, tgt_nid, "instantiates", line)
|
|
|
|
for child in node.children:
|
|
walk(child, module_nid)
|
|
|
|
walk(root)
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_sql(path: Path) -> dict:
|
|
"""Extract tables, views, functions, and relationships from .sql files via tree-sitter."""
|
|
try:
|
|
import tree_sitter_sql as tssql
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"}
|
|
|
|
try:
|
|
language = Language(tssql.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = re.sub(r"[^a-z0-9]", "_", path.stem.lower())
|
|
str_path = str(path)
|
|
file_nid = _make_id(str_path)
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = {file_nid}
|
|
table_nids: dict[str, str] = {} # name → nid for reference resolution
|
|
|
|
def _read(n) -> str:
|
|
return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
|
|
|
def _obj_name(n) -> str | None:
|
|
for c in n.children:
|
|
if c.type == "object_reference":
|
|
for cc in c.children:
|
|
if cc.type == "identifier":
|
|
return _read(cc)
|
|
return None
|
|
|
|
def _add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0})
|
|
|
|
def _add_edge(src: str, tgt: str, relation: str, line: int) -> None:
|
|
edges.append({"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0})
|
|
|
|
def walk(node) -> None:
|
|
t = node.type
|
|
line = node.start_point[0] + 1
|
|
|
|
if t == "create_table":
|
|
name = _obj_name(node)
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
_add_node(nid, name, line)
|
|
table_nids[name.lower()] = nid
|
|
# Foreign key REFERENCES
|
|
for col in node.children:
|
|
if col.type == "column_definitions":
|
|
for cd in col.children:
|
|
if cd.type != "column_definition":
|
|
continue
|
|
ref_name: str | None = None
|
|
found_ref = False
|
|
for cc in cd.children:
|
|
if cc.type == "keyword_references":
|
|
found_ref = True
|
|
elif found_ref and cc.type == "object_reference":
|
|
for ccc in cc.children:
|
|
if ccc.type == "identifier":
|
|
ref_name = _read(ccc)
|
|
break
|
|
if ref_name:
|
|
ref_nid = _make_id(stem, ref_name)
|
|
_add_edge(nid, ref_nid, "references", line)
|
|
|
|
elif t == "create_view":
|
|
name = _obj_name(node)
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
_add_node(nid, name, line)
|
|
table_nids[name.lower()] = nid
|
|
# FROM/JOIN table references inside view body
|
|
_walk_from_refs(node, nid, line)
|
|
|
|
elif t == "create_function":
|
|
name = _obj_name(node)
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
_add_node(nid, f"{name}()", line)
|
|
_walk_from_refs(node, nid, line)
|
|
|
|
elif t == "create_procedure":
|
|
name = _obj_name(node)
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
_add_node(nid, f"{name}()", line)
|
|
_walk_from_refs(node, nid, line)
|
|
|
|
for child in node.children:
|
|
walk(child)
|
|
|
|
def _walk_from_refs(node, caller_nid: str, line: int) -> None:
|
|
"""Recursively find FROM/JOIN table references inside a node."""
|
|
if node.type in ("from", "join"):
|
|
for c in node.children:
|
|
if c.type == "relation":
|
|
for cc in c.children:
|
|
if cc.type == "object_reference":
|
|
for ccc in cc.children:
|
|
if ccc.type == "identifier":
|
|
tbl = _read(ccc)
|
|
tbl_nid = _make_id(stem, tbl)
|
|
_add_edge(caller_nid, tbl_nid, "reads_from",
|
|
c.start_point[0] + 1)
|
|
for child in node.children:
|
|
_walk_from_refs(child, caller_nid, line)
|
|
|
|
for stmt in root.children:
|
|
if stmt.type == "statement":
|
|
for child in stmt.children:
|
|
walk(child)
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_lua(path: Path) -> dict:
|
|
"""Extract functions, methods, require() imports, and calls from a .lua file."""
|
|
return _extract_generic(path, _LUA_CONFIG)
|
|
|
|
|
|
def extract_swift(path: Path) -> dict:
|
|
"""Extract classes, structs, protocols, functions, imports, and calls from a .swift file."""
|
|
return _extract_generic(path, _SWIFT_CONFIG)
|
|
|
|
|
|
# ── Julia extractor (custom walk) ────────────────────────────────────────────
|
|
|
|
def extract_julia(path: Path) -> dict:
|
|
"""Extract modules, structs, functions, imports, and calls from a .jl file."""
|
|
try:
|
|
import tree_sitter_julia as tsjulia
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"}
|
|
|
|
try:
|
|
language = Language(tsjulia.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, object]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _func_name_from_signature(sig_node) -> str | None:
|
|
"""Extract function name from a Julia signature node (call_expression > identifier)."""
|
|
for child in sig_node.children:
|
|
if child.type == "call_expression":
|
|
callee = child.children[0] if child.children else None
|
|
if callee and callee.type == "identifier":
|
|
return _read_text(callee, source)
|
|
return None
|
|
|
|
def walk_calls(body_node, func_nid: str) -> None:
|
|
if body_node is None:
|
|
return
|
|
t = body_node.type
|
|
if t in ("function_definition", "short_function_definition"):
|
|
return
|
|
if t == "call_expression" and body_node.children:
|
|
callee = body_node.children[0]
|
|
# Direct call: foo(...)
|
|
if callee.type == "identifier":
|
|
callee_name = _read_text(callee, source)
|
|
target_nid = _make_id(stem, callee_name)
|
|
add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
|
|
confidence="EXTRACTED", context="call")
|
|
# Method call: obj.method(...)
|
|
elif callee.type == "field_expression" and len(callee.children) >= 3:
|
|
method_node = callee.children[-1]
|
|
method_name = _read_text(method_node, source)
|
|
target_nid = _make_id(stem, method_name)
|
|
add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
|
|
confidence="EXTRACTED", context="call")
|
|
for child in body_node.children:
|
|
walk_calls(child, func_nid)
|
|
|
|
def walk(node, scope_nid: str) -> None:
|
|
t = node.type
|
|
|
|
# Module
|
|
if t == "module_definition":
|
|
name_node = next((c for c in node.children if c.type == "identifier"), None)
|
|
if name_node:
|
|
mod_name = _read_text(name_node, source)
|
|
mod_nid = _make_id(stem, mod_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(mod_nid, mod_name, line)
|
|
add_edge(file_nid, mod_nid, "defines", line)
|
|
for child in node.children:
|
|
walk(child, mod_nid)
|
|
return
|
|
|
|
# Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia)
|
|
if t == "struct_definition":
|
|
# type_head may contain: identifier (simple) or binary_expression (Foo <: Bar)
|
|
type_head = next((c for c in node.children if c.type == "type_head"), None)
|
|
if type_head:
|
|
bin_expr = next((c for c in type_head.children if c.type == "binary_expression"), None)
|
|
if bin_expr:
|
|
# First identifier is the struct name, last is the supertype
|
|
identifiers = [c for c in bin_expr.children if c.type == "identifier"]
|
|
if identifiers:
|
|
struct_name = _read_text(identifiers[0], source)
|
|
struct_nid = _make_id(stem, struct_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(struct_nid, struct_name, line)
|
|
add_edge(scope_nid, struct_nid, "defines", line)
|
|
if len(identifiers) >= 2:
|
|
super_name = _read_text(identifiers[-1], source)
|
|
add_edge(struct_nid, _make_id(stem, super_name), "inherits",
|
|
line, confidence="EXTRACTED")
|
|
else:
|
|
name_node = next((c for c in type_head.children if c.type == "identifier"), None)
|
|
if name_node:
|
|
struct_name = _read_text(name_node, source)
|
|
struct_nid = _make_id(stem, struct_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(struct_nid, struct_name, line)
|
|
add_edge(scope_nid, struct_nid, "defines", line)
|
|
return
|
|
|
|
# Abstract type
|
|
if t == "abstract_definition":
|
|
# type_head > identifier
|
|
type_head = next((c for c in node.children if c.type == "type_head"), None)
|
|
if type_head:
|
|
name_node = next((c for c in type_head.children if c.type == "identifier"), None)
|
|
if name_node:
|
|
abs_name = _read_text(name_node, source)
|
|
abs_nid = _make_id(stem, abs_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(abs_nid, abs_name, line)
|
|
add_edge(scope_nid, abs_nid, "defines", line)
|
|
return
|
|
|
|
# Function: function foo(...) ... end
|
|
if t == "function_definition":
|
|
sig_node = next((c for c in node.children if c.type == "signature"), None)
|
|
if sig_node:
|
|
func_name = _func_name_from_signature(sig_node)
|
|
if func_name:
|
|
func_nid = _make_id(stem, func_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(scope_nid, func_nid, "defines", line)
|
|
function_bodies.append((func_nid, node))
|
|
return
|
|
|
|
# Short function: foo(x) = expr
|
|
if t == "assignment":
|
|
lhs = node.children[0] if node.children else None
|
|
if lhs and lhs.type == "call_expression" and lhs.children:
|
|
callee = lhs.children[0]
|
|
if callee.type == "identifier":
|
|
func_name = _read_text(callee, source)
|
|
func_nid = _make_id(stem, func_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(scope_nid, func_nid, "defines", line)
|
|
# Only walk the RHS (index 2 after lhs and operator) to avoid self-loops
|
|
rhs = node.children[-1] if len(node.children) >= 3 else None
|
|
if rhs:
|
|
function_bodies.append((func_nid, rhs))
|
|
return
|
|
|
|
# Using / Import
|
|
if t in ("using_statement", "import_statement"):
|
|
line = node.start_point[0] + 1
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
mod_name = _read_text(child, source)
|
|
imp_nid = _make_id(mod_name)
|
|
add_node(imp_nid, mod_name, line)
|
|
add_edge(scope_nid, imp_nid, "imports", line, context="import")
|
|
elif child.type == "selected_import":
|
|
identifiers = [c for c in child.children if c.type == "identifier"]
|
|
if identifiers:
|
|
pkg_name = _read_text(identifiers[0], source)
|
|
pkg_nid = _make_id(pkg_name)
|
|
add_node(pkg_nid, pkg_name, line)
|
|
add_edge(scope_nid, pkg_nid, "imports", line, context="import")
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, scope_nid)
|
|
|
|
walk(root, file_nid)
|
|
|
|
for func_nid, body_node in function_bodies:
|
|
# For function_definition nodes, walk children directly to avoid
|
|
# the boundary check returning early on the top-level node itself.
|
|
# Skip the "signature" child — it contains the function's own call_expression
|
|
# which would create a self-loop.
|
|
if body_node.type == "function_definition":
|
|
for child in body_node.children:
|
|
if child.type != "signature":
|
|
walk_calls(child, func_nid)
|
|
else:
|
|
walk_calls(body_node, func_nid)
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
_FORTRAN_CPP_EXTS = {".F", ".F90", ".F95", ".F03", ".F08"}
|
|
|
|
|
|
def _cpp_preprocess(path: Path) -> bytes:
|
|
"""Run cpp -w -P on a capital-F Fortran file and return preprocessed bytes.
|
|
|
|
Falls back to raw file bytes if cpp is not available. Capital-F extensions
|
|
conventionally require C preprocessor expansion (#ifdef MPI, #define REAL8, etc.)
|
|
before parsing.
|
|
"""
|
|
import shutil
|
|
import subprocess
|
|
if not shutil.which("cpp"):
|
|
return path.read_bytes()
|
|
try:
|
|
result = subprocess.run(
|
|
["cpp", "-w", "-P", str(path)],
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
if result.returncode == 0 and result.stdout:
|
|
return result.stdout
|
|
except Exception:
|
|
pass
|
|
return path.read_bytes()
|
|
|
|
|
|
def extract_fortran(path: Path) -> dict:
|
|
"""Extract programs, modules, subroutines, functions, use statements, and calls from Fortran files.
|
|
|
|
Capital-F extensions (.F, .F90, etc.) are run through the C preprocessor before
|
|
parsing so #ifdef/#define macros are resolved.
|
|
"""
|
|
try:
|
|
import tree_sitter_fortran as tsfortran
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-fortran not installed"}
|
|
|
|
try:
|
|
language = Language(tsfortran.language())
|
|
parser = Parser(language)
|
|
source = _cpp_preprocess(path) if path.suffix in _FORTRAN_CPP_EXTS else path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
scope_bodies: list[tuple[str, object]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _fortran_name(stmt_node) -> str | None:
|
|
"""Extract name from a *_statement node. Fortran is case-insensitive; lowercase."""
|
|
for child in stmt_node.children:
|
|
if child.type in ("name", "identifier"):
|
|
return _read_text(child, source).lower()
|
|
return None
|
|
|
|
def walk_calls(node, scope_nid: str) -> None:
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t in ("subroutine", "function", "module", "program", "internal_procedures"):
|
|
return
|
|
# call FOO(args) — tree-sitter-fortran uses subroutine_call
|
|
if t == "subroutine_call":
|
|
name_node = next((c for c in node.children if c.type == "identifier"), None)
|
|
if name_node:
|
|
callee = _read_text(name_node, source).lower()
|
|
target_nid = _make_id(stem, callee)
|
|
add_edge(scope_nid, target_nid, "calls", node.start_point[0] + 1,
|
|
confidence="EXTRACTED", context="call")
|
|
for child in node.children:
|
|
walk_calls(child, scope_nid)
|
|
|
|
def walk(node, scope_nid: str) -> None:
|
|
t = node.type
|
|
|
|
if t == "program":
|
|
stmt = next((c for c in node.children if c.type == "program_statement"), None)
|
|
name = _fortran_name(stmt) if stmt else None
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(nid, name, line)
|
|
add_edge(file_nid, nid, "defines", line)
|
|
scope_bodies.append((nid, node))
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
if t == "module":
|
|
stmt = next((c for c in node.children if c.type == "module_statement"), None)
|
|
name = _fortran_name(stmt) if stmt else None
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(nid, name, line)
|
|
add_edge(file_nid, nid, "defines", line)
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
# subroutines/functions inside a module live under internal_procedures
|
|
if t == "internal_procedures":
|
|
for child in node.children:
|
|
walk(child, scope_nid)
|
|
return
|
|
|
|
if t == "subroutine":
|
|
stmt = next((c for c in node.children if c.type == "subroutine_statement"), None)
|
|
name = _fortran_name(stmt) if stmt else None
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(nid, f"{name}()", line)
|
|
add_edge(scope_nid, nid, "defines", line)
|
|
scope_bodies.append((nid, node))
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
if t == "function":
|
|
stmt = next((c for c in node.children if c.type == "function_statement"), None)
|
|
name = _fortran_name(stmt) if stmt else None
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(nid, f"{name}()", line)
|
|
add_edge(scope_nid, nid, "defines", line)
|
|
scope_bodies.append((nid, node))
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
if t == "use_statement":
|
|
line = node.start_point[0] + 1
|
|
# tree-sitter-fortran uses module_name node for the used module
|
|
name_node = next((c for c in node.children if c.type in ("module_name", "name", "identifier")), None)
|
|
if name_node:
|
|
mod_name = _read_text(name_node, source).lower()
|
|
imp_nid = _make_id(mod_name)
|
|
add_node(imp_nid, mod_name, line)
|
|
add_edge(scope_nid, imp_nid, "imports", line, context="use")
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, scope_nid)
|
|
|
|
walk(root, file_nid)
|
|
|
|
_stmt_headers = {
|
|
"subroutine_statement", "function_statement",
|
|
"program_statement", "module_statement",
|
|
}
|
|
for scope_nid, body_node in scope_bodies:
|
|
for child in body_node.children:
|
|
if child.type not in _stmt_headers:
|
|
walk_calls(child, scope_nid)
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# ── Go extractor (custom walk) ────────────────────────────────────────────────
|
|
|
|
def extract_go(path: Path) -> dict:
|
|
"""Extract functions, methods, type declarations, and imports from a .go file."""
|
|
try:
|
|
import tree_sitter_go as tsgo
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-go not installed"}
|
|
|
|
try:
|
|
language = Language(tsgo.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
# Use directory name as package scope so methods on the same type across
|
|
# multiple files in a package share one canonical type node.
|
|
pkg_scope = path.parent.name or stem
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, object]] = []
|
|
go_imported_pkgs: set[str] = set() # local names of imported packages
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def walk(node) -> None:
|
|
t = node.type
|
|
|
|
if t == "function_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
if t == "method_declaration":
|
|
receiver = node.child_by_field_name("receiver")
|
|
receiver_type: str | None = None
|
|
if receiver:
|
|
for param in receiver.children:
|
|
if param.type == "parameter_declaration":
|
|
type_node = param.child_by_field_name("type")
|
|
if type_node:
|
|
raw = _read_text(type_node, source).lstrip("*").strip()
|
|
receiver_type = raw
|
|
break
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
method_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
if receiver_type:
|
|
parent_nid = _make_id(pkg_scope, receiver_type)
|
|
add_node(parent_nid, receiver_type, line)
|
|
method_nid = _make_id(parent_nid, method_name)
|
|
add_node(method_nid, f".{method_name}()", line)
|
|
add_edge(parent_nid, method_nid, "method", line)
|
|
else:
|
|
method_nid = _make_id(stem, method_name)
|
|
add_node(method_nid, f"{method_name}()", line)
|
|
add_edge(file_nid, method_nid, "contains", line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((method_nid, body))
|
|
return
|
|
|
|
if t == "type_declaration":
|
|
for child in node.children:
|
|
if child.type == "type_spec":
|
|
name_node = child.child_by_field_name("name")
|
|
if name_node:
|
|
type_name = _read_text(name_node, source)
|
|
line = child.start_point[0] + 1
|
|
type_nid = _make_id(pkg_scope, type_name)
|
|
add_node(type_nid, type_name, line)
|
|
add_edge(file_nid, type_nid, "contains", line)
|
|
return
|
|
|
|
if t == "import_declaration":
|
|
for child in node.children:
|
|
if child.type == "import_spec_list":
|
|
for spec in child.children:
|
|
if spec.type == "import_spec":
|
|
path_node = spec.child_by_field_name("path")
|
|
if path_node:
|
|
raw = _read_text(path_node, source).strip('"')
|
|
# Prefix with go_pkg_ so stdlib names (e.g. "context")
|
|
# don't collide with local files of the same basename.
|
|
tgt_nid = _make_id("go", "pkg", raw)
|
|
add_edge(file_nid, tgt_nid, "imports_from", spec.start_point[0] + 1, context="import")
|
|
# Track local name (alias or last path segment)
|
|
alias = spec.child_by_field_name("name")
|
|
local_name = _read_text(alias, source) if alias else raw.split("/")[-1]
|
|
if local_name and local_name != "_" and local_name != ".":
|
|
go_imported_pkgs.add(local_name)
|
|
elif child.type == "import_spec":
|
|
path_node = child.child_by_field_name("path")
|
|
if path_node:
|
|
raw = _read_text(path_node, source).strip('"')
|
|
tgt_nid = _make_id("go", "pkg", raw)
|
|
add_edge(file_nid, tgt_nid, "imports_from", child.start_point[0] + 1, context="import")
|
|
alias = child.child_by_field_name("name")
|
|
local_name = _read_text(alias, source) if alias else raw.split("/")[-1]
|
|
if local_name and local_name != "_" and local_name != ".":
|
|
go_imported_pkgs.add(local_name)
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child)
|
|
|
|
walk(root)
|
|
|
|
label_to_nid: dict[str, str] = {}
|
|
for n in nodes:
|
|
raw = n["label"]
|
|
normalised = raw.strip("()").lstrip(".")
|
|
label_to_nid[normalised.lower()] = n["id"]
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type in ("function_declaration", "method_declaration"):
|
|
return
|
|
if node.type == "call_expression":
|
|
func_node = node.child_by_field_name("function")
|
|
callee_name: str | None = None
|
|
is_member_call: bool = False
|
|
if func_node:
|
|
if func_node.type == "identifier":
|
|
callee_name = _read_text(func_node, source)
|
|
elif func_node.type == "selector_expression":
|
|
field = func_node.child_by_field_name("field")
|
|
operand = func_node.child_by_field_name("operand")
|
|
receiver_name = _read_text(operand, source) if operand else ""
|
|
# Package-qualified call (e.g. fmt.Println) → allow cross-file resolution.
|
|
# Receiver method call (e.g. s.logger.Log) → skip, no import evidence.
|
|
is_member_call = receiver_name not in go_imported_pkgs
|
|
if field:
|
|
callee_name = _read_text(field, source)
|
|
if callee_name:
|
|
tgt_nid = label_to_nid.get(callee_name.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
elif callee_name:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee_name,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
valid_ids = seen_ids
|
|
clean_edges = []
|
|
for edge in edges:
|
|
src, tgt = edge["source"], edge["target"]
|
|
if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):
|
|
clean_edges.append(edge)
|
|
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── Rust extractor (custom walk) ──────────────────────────────────────────────
|
|
|
|
def extract_rust(path: Path) -> dict:
|
|
"""Extract functions, structs, enums, traits, impl methods, and use declarations from a .rs file."""
|
|
try:
|
|
import tree_sitter_rust as tsrust
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-rust not installed"}
|
|
|
|
try:
|
|
language = Language(tsrust.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, object]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def walk(node, parent_impl_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
if t == "function_item":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
if parent_impl_nid:
|
|
func_nid = _make_id(parent_impl_nid, func_name)
|
|
add_node(func_nid, f".{func_name}()", line)
|
|
add_edge(parent_impl_nid, func_nid, "method", line)
|
|
else:
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
if t in ("struct_item", "enum_item", "trait_item"):
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
item_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
item_nid = _make_id(stem, item_name)
|
|
add_node(item_nid, item_name, line)
|
|
add_edge(file_nid, item_nid, "contains", line)
|
|
return
|
|
|
|
if t == "impl_item":
|
|
type_node = node.child_by_field_name("type")
|
|
impl_nid: str | None = None
|
|
if type_node:
|
|
type_name = _read_text(type_node, source).strip()
|
|
impl_nid = _make_id(stem, type_name)
|
|
add_node(impl_nid, type_name, node.start_point[0] + 1)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
for child in body.children:
|
|
walk(child, parent_impl_nid=impl_nid)
|
|
return
|
|
|
|
if t == "use_declaration":
|
|
arg = node.child_by_field_name("argument")
|
|
if arg:
|
|
raw = _read_text(arg, source)
|
|
clean = raw.split("{")[0].rstrip(":").rstrip("*").rstrip(":")
|
|
module_name = clean.split("::")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1, context="import")
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_impl_nid=None)
|
|
|
|
walk(root)
|
|
|
|
label_to_nid: dict[str, str] = {}
|
|
for n in nodes:
|
|
raw = n["label"]
|
|
normalised = raw.strip("()").lstrip(".")
|
|
label_to_nid[normalised.lower()] = n["id"]
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type == "function_item":
|
|
return
|
|
if node.type == "call_expression":
|
|
func_node = node.child_by_field_name("function")
|
|
callee_name: str | None = None
|
|
is_member_call: bool = False
|
|
if func_node:
|
|
if func_node.type == "identifier":
|
|
callee_name = _read_text(func_node, source)
|
|
elif func_node.type == "field_expression":
|
|
is_member_call = True
|
|
field = func_node.child_by_field_name("field")
|
|
if field:
|
|
callee_name = _read_text(field, source)
|
|
elif func_node.type == "scoped_identifier":
|
|
name = func_node.child_by_field_name("name")
|
|
if name:
|
|
callee_name = _read_text(name, source)
|
|
if callee_name:
|
|
tgt_nid = label_to_nid.get(callee_name.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
else:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee_name,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
valid_ids = seen_ids
|
|
clean_edges = []
|
|
for edge in edges:
|
|
src, tgt = edge["source"], edge["target"]
|
|
if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):
|
|
clean_edges.append(edge)
|
|
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── Zig ───────────────────────────────────────────────────────────────────────
|
|
|
|
def extract_zig(path: Path) -> dict:
|
|
"""Extract functions, structs, enums, unions, and imports from a .zig file."""
|
|
try:
|
|
import tree_sitter_zig as tszig
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_zig not installed"}
|
|
|
|
try:
|
|
language = Language(tszig.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, Any]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _extract_import(node) -> None:
|
|
for child in node.children:
|
|
if child.type == "builtin_function":
|
|
bi = None
|
|
args = None
|
|
for c in child.children:
|
|
if c.type == "builtin_identifier":
|
|
bi = _read_text(c, source)
|
|
elif c.type == "arguments":
|
|
args = c
|
|
if bi in ("@import", "@cImport") and args:
|
|
for arg in args.children:
|
|
if arg.type in ("string_literal", "string"):
|
|
raw = _read_text(arg, source).strip('"')
|
|
module_name = raw.split("/")[-1].split(".")[0]
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
add_edge(file_nid, tgt_nid, "imports_from",
|
|
node.start_point[0] + 1)
|
|
return
|
|
elif child.type == "field_expression":
|
|
_extract_import(child)
|
|
return
|
|
|
|
def walk(node, parent_struct_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
if t == "function_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
if parent_struct_nid:
|
|
func_nid = _make_id(parent_struct_nid, func_name)
|
|
add_node(func_nid, f".{func_name}()", line)
|
|
add_edge(parent_struct_nid, func_nid, "method", line)
|
|
else:
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
if t == "variable_declaration":
|
|
name_node = None
|
|
value_node = None
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
name_node = child
|
|
elif child.type in ("struct_declaration", "enum_declaration",
|
|
"union_declaration", "builtin_function",
|
|
"field_expression"):
|
|
value_node = child
|
|
|
|
if value_node and value_node.type == "struct_declaration":
|
|
if name_node:
|
|
struct_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
struct_nid = _make_id(stem, struct_name)
|
|
add_node(struct_nid, struct_name, line)
|
|
add_edge(file_nid, struct_nid, "contains", line)
|
|
for child in value_node.children:
|
|
walk(child, parent_struct_nid=struct_nid)
|
|
return
|
|
|
|
if value_node and value_node.type in ("enum_declaration", "union_declaration"):
|
|
if name_node:
|
|
type_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
type_nid = _make_id(stem, type_name)
|
|
add_node(type_nid, type_name, line)
|
|
add_edge(file_nid, type_nid, "contains", line)
|
|
return
|
|
|
|
if value_node and value_node.type in ("builtin_function", "field_expression"):
|
|
_extract_import(node)
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_struct_nid)
|
|
|
|
walk(root)
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type == "function_declaration":
|
|
return
|
|
if node.type == "call_expression":
|
|
fn = node.child_by_field_name("function")
|
|
if fn:
|
|
fn_text = _read_text(fn, source)
|
|
callee = fn_text.split(".")[-1]
|
|
is_member_call = "." in fn_text
|
|
tgt_nid = next((n["id"] for n in nodes if n["label"] in
|
|
(f"{callee}()", f".{callee}()")), None)
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
add_edge(caller_nid, tgt_nid, "calls",
|
|
node.start_point[0] + 1,
|
|
confidence="EXTRACTED", weight=1.0)
|
|
elif callee:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
|
(e["target"] in seen_ids or e["relation"] == "imports_from")]
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── PowerShell ────────────────────────────────────────────────────────────────
|
|
|
|
def extract_powershell(path: Path) -> dict:
|
|
"""Extract functions, classes, methods, and using statements from a .ps1 file."""
|
|
try:
|
|
import tree_sitter_powershell as tsps
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"}
|
|
|
|
try:
|
|
language = Language(tsps.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, Any]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
_PS_SKIP = frozenset({
|
|
"using", "return", "if", "else", "elseif", "foreach", "for",
|
|
"while", "do", "switch", "try", "catch", "finally", "throw",
|
|
"break", "continue", "exit", "param", "begin", "process", "end",
|
|
})
|
|
|
|
def _find_script_block_body(node):
|
|
for child in node.children:
|
|
if child.type == "script_block":
|
|
for sc in child.children:
|
|
if sc.type == "script_block_body":
|
|
return sc
|
|
return child
|
|
return None
|
|
|
|
def walk(node, parent_class_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
if t == "function_statement":
|
|
name_node = next((c for c in node.children if c.type == "function_name"), None)
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
body = _find_script_block_body(node)
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
if t == "class_statement":
|
|
name_node = next((c for c in node.children if c.type == "simple_name"), None)
|
|
if name_node:
|
|
class_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
class_nid = _make_id(stem, class_name)
|
|
add_node(class_nid, class_name, line)
|
|
add_edge(file_nid, class_nid, "contains", line)
|
|
for child in node.children:
|
|
walk(child, parent_class_nid=class_nid)
|
|
return
|
|
|
|
if t == "class_method_definition":
|
|
name_node = next((c for c in node.children if c.type == "simple_name"), None)
|
|
if name_node:
|
|
method_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
if parent_class_nid:
|
|
method_nid = _make_id(parent_class_nid, method_name)
|
|
add_node(method_nid, f".{method_name}()", line)
|
|
add_edge(parent_class_nid, method_nid, "method", line)
|
|
else:
|
|
method_nid = _make_id(stem, method_name)
|
|
add_node(method_nid, f"{method_name}()", line)
|
|
add_edge(file_nid, method_nid, "contains", line)
|
|
body = _find_script_block_body(node)
|
|
if body:
|
|
function_bodies.append((method_nid, body))
|
|
return
|
|
|
|
if t == "command":
|
|
cmd_name_node = next((c for c in node.children if c.type == "command_name"), None)
|
|
if cmd_name_node:
|
|
cmd_text = _read_text(cmd_name_node, source).lower()
|
|
if cmd_text == "using":
|
|
tokens = []
|
|
for child in node.children:
|
|
if child.type == "command_elements":
|
|
for el in child.children:
|
|
if el.type == "generic_token":
|
|
tokens.append(_read_text(el, source))
|
|
module_tokens = [t for t in tokens
|
|
if t.lower() not in ("namespace", "module", "assembly")]
|
|
if module_tokens:
|
|
module_name = module_tokens[-1].split(".")[-1]
|
|
add_edge(file_nid, _make_id(module_name), "imports_from",
|
|
node.start_point[0] + 1)
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_class_nid)
|
|
|
|
walk(root)
|
|
|
|
label_to_nid = {n["label"].strip("()").lstrip(".").lower(): n["id"] for n in nodes}
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type in ("function_statement", "class_statement"):
|
|
return
|
|
if node.type == "command":
|
|
cmd_name_node = next((c for c in node.children if c.type == "command_name"), None)
|
|
if cmd_name_node:
|
|
cmd_text = _read_text(cmd_name_node, source)
|
|
if cmd_text.lower() not in _PS_SKIP:
|
|
tgt_nid = label_to_nid.get(cmd_text.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
add_edge(caller_nid, tgt_nid, "calls",
|
|
node.start_point[0] + 1,
|
|
confidence="EXTRACTED", weight=1.0)
|
|
elif cmd_text:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": cmd_text,
|
|
"is_member_call": False,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
|
(e["target"] in seen_ids or e["relation"] == "imports_from")]
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── Cross-file import resolution ──────────────────────────────────────────────
|
|
|
|
def _resolve_cross_file_imports(
|
|
per_file: list[dict],
|
|
paths: list[Path],
|
|
) -> list[dict]:
|
|
"""
|
|
Two-pass import resolution: turn file-level imports into class-level edges.
|
|
|
|
Pass 1 - build a global map: class/function name → node_id, per stem.
|
|
Pass 2 - for each `from .module import Name`, look up Name in the global
|
|
map and add a direct INFERRED edge from each class in the
|
|
importing file to the imported entity.
|
|
|
|
This turns:
|
|
auth.py --imports_from--> models.py (obvious, filtered out)
|
|
Into:
|
|
DigestAuth --uses--> Response [INFERRED] (cross-file, interesting!)
|
|
BasicAuth --uses--> Request [INFERRED]
|
|
"""
|
|
try:
|
|
import tree_sitter_python as tspython
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return []
|
|
|
|
language = Language(tspython.language())
|
|
parser = Parser(language)
|
|
|
|
# Pass 1: name → node_id across all files
|
|
# Map: stem → {ClassName: node_id}
|
|
stem_to_entities: dict[str, dict[str, str]] = {}
|
|
for file_result in per_file:
|
|
for node in file_result.get("nodes", []):
|
|
src = node.get("source_file", "")
|
|
if not src:
|
|
continue
|
|
stem = Path(src).stem
|
|
label = node.get("label", "")
|
|
nid = node.get("id", "")
|
|
# Index class-level entities only. Function/method labels end in "()"
|
|
# so are excluded by the `endswith(")")` filter; file nodes end in ".py";
|
|
# private/internal labels start with "_"; rationale nodes carry
|
|
# file_type=="rationale" and must never participate in cross-file
|
|
# import resolution (#563).
|
|
if (
|
|
label
|
|
and not label.endswith((")", ".py"))
|
|
and "_" not in label[:1]
|
|
and node.get("file_type") != "rationale"
|
|
):
|
|
stem_to_entities.setdefault(stem, {})[label] = nid
|
|
|
|
# Pass 2: for each file, find `from .X import A, B, C` and resolve
|
|
new_edges: list[dict] = []
|
|
stem_to_path: dict[str, Path] = {p.stem: p for p in paths}
|
|
|
|
for file_result, path in zip(per_file, paths):
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
|
|
# Find all classes defined in this file (the importers).
|
|
# Excludes rationale nodes whose labels happen not to end in ")" or ".py"
|
|
# but which must never be treated as importing entities (#563).
|
|
local_classes = [
|
|
n["id"] for n in file_result.get("nodes", [])
|
|
if n.get("source_file") == str_path
|
|
and not n["label"].endswith((")", ".py"))
|
|
and n["id"] != _make_id(stem) # exclude file-level node
|
|
and n.get("file_type") != "rationale"
|
|
]
|
|
if not local_classes:
|
|
continue
|
|
|
|
# Parse imports from this file
|
|
try:
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
except Exception:
|
|
continue
|
|
|
|
def walk_imports(node) -> None:
|
|
if node.type == "import_from_statement":
|
|
# Find the module name - handles both absolute and relative imports.
|
|
# Relative: `from .models import X` → relative_import → dotted_name
|
|
# Absolute: `from models import X` → module_name field
|
|
target_stem: str | None = None
|
|
for child in node.children:
|
|
if child.type == "relative_import":
|
|
# Dig into relative_import → dotted_name → identifier
|
|
for sub in child.children:
|
|
if sub.type == "dotted_name":
|
|
raw = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
|
|
target_stem = raw.split(".")[-1]
|
|
break
|
|
break
|
|
if child.type == "dotted_name" and target_stem is None:
|
|
raw = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
target_stem = raw.split(".")[-1]
|
|
|
|
if not target_stem or target_stem not in stem_to_entities:
|
|
return
|
|
|
|
# Collect imported names: dotted_name children of import_from_statement
|
|
# that come AFTER the 'import' keyword token.
|
|
imported_names: list[str] = []
|
|
past_import_kw = False
|
|
for child in node.children:
|
|
if child.type == "import":
|
|
past_import_kw = True
|
|
continue
|
|
if not past_import_kw:
|
|
continue
|
|
if child.type == "dotted_name":
|
|
imported_names.append(
|
|
source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
)
|
|
elif child.type == "aliased_import":
|
|
# `import X as Y` - take the original name
|
|
name_node = child.child_by_field_name("name")
|
|
if name_node:
|
|
imported_names.append(
|
|
source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
)
|
|
|
|
line = node.start_point[0] + 1
|
|
for name in imported_names:
|
|
tgt_nid = stem_to_entities[target_stem].get(name)
|
|
if tgt_nid:
|
|
for src_class_nid in local_classes:
|
|
new_edges.append({
|
|
"source": src_class_nid,
|
|
"target": tgt_nid,
|
|
"relation": "uses",
|
|
"confidence": "INFERRED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 0.8,
|
|
})
|
|
for child in node.children:
|
|
walk_imports(child)
|
|
|
|
walk_imports(tree.root_node)
|
|
|
|
return new_edges
|
|
|
|
|
|
def _resolve_cross_file_java_imports(
|
|
per_file: list[dict],
|
|
paths: list[Path],
|
|
) -> list[dict]:
|
|
"""Two-pass Java import resolution.
|
|
|
|
Pass 1: build a global index {ClassName: [node_id, ...]} across all Java nodes.
|
|
Pass 2: re-parse each Java file; for every `import a.b.C;`, resolve C against
|
|
the index. Wildcard and stdlib imports produce no edge.
|
|
"""
|
|
try:
|
|
import tree_sitter_java as tsjava
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return []
|
|
|
|
language = Language(tsjava.language())
|
|
parser = Parser(language)
|
|
|
|
# Pass 1: class-name → node_id index (only internal, uppercase-starting names)
|
|
name_to_ids: dict[str, list[str]] = {}
|
|
for file_result in per_file:
|
|
for node in file_result.get("nodes", []):
|
|
label = node.get("label", "")
|
|
nid = node.get("id", "")
|
|
src = node.get("source_file", "")
|
|
if not label or not nid or not src:
|
|
continue
|
|
if label.endswith(")") or label.endswith(".java"):
|
|
continue
|
|
if not label[0].isalpha() or not label[0].isupper():
|
|
continue
|
|
name_to_ids.setdefault(label, []).append(nid)
|
|
|
|
# Pass 2: resolve imports to real node IDs
|
|
new_edges: list[dict] = []
|
|
seen_pairs: set[tuple[str, str]] = set()
|
|
for path in paths:
|
|
file_nid = _make_id(str(path))
|
|
try:
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
except Exception:
|
|
continue
|
|
|
|
def walk(n) -> None:
|
|
if n.type == "import_declaration":
|
|
raw = _read_text(n, source).strip()
|
|
body = raw[len("import"):].strip().rstrip(";").strip()
|
|
if body.startswith("static "):
|
|
body = body[len("static "):].strip()
|
|
if body.endswith(".*"):
|
|
return
|
|
parts = body.split(".")
|
|
if not parts:
|
|
return
|
|
last = parts[-1]
|
|
if last and last[0].islower() and len(parts) >= 2:
|
|
last = parts[-2]
|
|
at_line = n.start_point[0] + 1
|
|
for tgt_nid in name_to_ids.get(last, []):
|
|
if tgt_nid == file_nid:
|
|
continue
|
|
key = (file_nid, tgt_nid)
|
|
if key in seen_pairs:
|
|
continue
|
|
seen_pairs.add(key)
|
|
new_edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str(path),
|
|
"source_location": f"L{at_line}",
|
|
"weight": 1.0,
|
|
})
|
|
for child in n.children:
|
|
walk(child)
|
|
|
|
walk(tree.root_node)
|
|
|
|
return new_edges
|
|
|
|
|
|
def extract_objc(path: Path) -> dict:
|
|
"""Extract interfaces, implementations, protocols, methods, and imports from .m/.mm/.h files."""
|
|
try:
|
|
import tree_sitter_objc as tsobjc
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_objc not installed"}
|
|
|
|
try:
|
|
language = Language(tsobjc.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
method_bodies: list[tuple[str, Any]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _read(node) -> str:
|
|
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
|
|
def _get_name(node, field: str) -> str | None:
|
|
n = node.child_by_field_name(field)
|
|
return _read(n) if n else None
|
|
|
|
def walk(node, parent_nid: str | None = None) -> None:
|
|
t = node.type
|
|
line = node.start_point[0] + 1
|
|
|
|
if t == "preproc_include":
|
|
# #import <Foundation/Foundation.h> or #import "MyClass.h"
|
|
for child in node.children:
|
|
if child.type == "system_lib_string":
|
|
raw = _read(child).strip("<>")
|
|
module = raw.split("/")[-1].replace(".h", "")
|
|
if module:
|
|
tgt_nid = _make_id(module)
|
|
add_edge(file_nid, tgt_nid, "imports", line, context="import")
|
|
elif child.type == "string_literal":
|
|
# recurse into string_literal to find string_content
|
|
for sub in child.children:
|
|
if sub.type == "string_content":
|
|
raw = _read(sub)
|
|
module = raw.split("/")[-1].replace(".h", "")
|
|
if module:
|
|
tgt_nid = _make_id(module)
|
|
add_edge(file_nid, tgt_nid, "imports", line, context="import")
|
|
return
|
|
|
|
if t == "class_interface":
|
|
# @interface ClassName : SuperClass <Protocols>
|
|
# children: @interface, identifier(name), ':', identifier(super), parameterized_arguments, ...
|
|
identifiers = [c for c in node.children if c.type == "identifier"]
|
|
if not identifiers:
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
return
|
|
name = _read(identifiers[0])
|
|
cls_nid = _make_id(stem, name)
|
|
add_node(cls_nid, name, line)
|
|
add_edge(file_nid, cls_nid, "contains", line)
|
|
# superclass is second identifier after ':'
|
|
colon_seen = False
|
|
for child in node.children:
|
|
if child.type == ":":
|
|
colon_seen = True
|
|
elif colon_seen and child.type == "identifier":
|
|
super_nid = _make_id(_read(child))
|
|
add_edge(cls_nid, super_nid, "inherits", line)
|
|
colon_seen = False
|
|
elif child.type == "parameterized_arguments":
|
|
# protocols adopted
|
|
for sub in child.children:
|
|
if sub.type == "type_name":
|
|
for s in sub.children:
|
|
if s.type == "type_identifier":
|
|
proto_nid = _make_id(_read(s))
|
|
add_edge(cls_nid, proto_nid, "imports", line, context="import")
|
|
elif child.type == "method_declaration":
|
|
walk(child, cls_nid)
|
|
return
|
|
|
|
if t == "class_implementation":
|
|
# @implementation ClassName
|
|
name = None
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
name = _read(child)
|
|
break
|
|
if not name:
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
return
|
|
impl_nid = _make_id(stem, name)
|
|
if impl_nid not in seen_ids:
|
|
add_node(impl_nid, name, line)
|
|
add_edge(file_nid, impl_nid, "contains", line)
|
|
for child in node.children:
|
|
if child.type == "implementation_definition":
|
|
for sub in child.children:
|
|
walk(sub, impl_nid)
|
|
return
|
|
|
|
if t == "protocol_declaration":
|
|
name = None
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
name = _read(child)
|
|
break
|
|
if name:
|
|
proto_nid = _make_id(stem, name)
|
|
add_node(proto_nid, f"<{name}>", line)
|
|
add_edge(file_nid, proto_nid, "contains", line)
|
|
for child in node.children:
|
|
walk(child, proto_nid)
|
|
return
|
|
|
|
if t in ("method_declaration", "method_definition"):
|
|
container = parent_nid or file_nid
|
|
# method name is the first identifier child (simple selector)
|
|
# for compound selectors: identifier + method_parameter pairs
|
|
parts = []
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
parts.append(_read(child))
|
|
elif child.type == "method_parameter":
|
|
for sub in child.children:
|
|
if sub.type == "identifier":
|
|
# selector keyword before ':'
|
|
pass
|
|
method_name = "".join(parts) if parts else None
|
|
if method_name:
|
|
method_nid = _make_id(container, method_name)
|
|
add_node(method_nid, f"-{method_name}", line)
|
|
add_edge(container, method_nid, "method", line)
|
|
if t == "method_definition":
|
|
method_bodies.append((method_nid, node))
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
|
|
walk(root)
|
|
|
|
# Second pass: resolve calls inside method bodies
|
|
all_method_nids = {n["id"] for n in nodes if n["id"] != file_nid}
|
|
seen_calls: set[tuple[str, str]] = set()
|
|
for caller_nid, body_node in method_bodies:
|
|
def walk_calls(n) -> None:
|
|
if n.type == "message_expression":
|
|
# [receiver selector]
|
|
for child in n.children:
|
|
if child.type in ("selector", "keyword_argument_list"):
|
|
sel = []
|
|
if child.type == "selector":
|
|
sel.append(_read(child))
|
|
else:
|
|
for sub in child.children:
|
|
if sub.type == "keyword_argument":
|
|
for s in sub.children:
|
|
if s.type == "selector":
|
|
sel.append(_read(s))
|
|
method_name = "".join(sel)
|
|
for candidate in all_method_nids:
|
|
if candidate.endswith(_make_id("", method_name).lstrip("_")):
|
|
pair = (caller_nid, candidate)
|
|
if pair not in seen_calls and caller_nid != candidate:
|
|
seen_calls.add(pair)
|
|
add_edge(caller_nid, candidate, "calls", body_node.start_point[0] + 1,
|
|
confidence="EXTRACTED", weight=1.0, context="call")
|
|
for child in n.children:
|
|
walk_calls(child)
|
|
walk_calls(body_node)
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
def extract_elixir(path: Path) -> dict:
|
|
"""Extract modules, functions, imports, and calls from a .ex/.exs file."""
|
|
try:
|
|
import tree_sitter_elixir as tselixir
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_elixir not installed"}
|
|
|
|
try:
|
|
language = Language(tselixir.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, Any]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
_IMPORT_KEYWORDS = frozenset({"alias", "import", "require", "use"})
|
|
|
|
def _get_alias_text(node) -> str | None:
|
|
for child in node.children:
|
|
if child.type == "alias":
|
|
return source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
return None
|
|
|
|
def walk(node, parent_module_nid: str | None = None) -> None:
|
|
if node.type != "call":
|
|
for child in node.children:
|
|
walk(child, parent_module_nid)
|
|
return
|
|
|
|
identifier_node = None
|
|
arguments_node = None
|
|
do_block_node = None
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
identifier_node = child
|
|
elif child.type == "arguments":
|
|
arguments_node = child
|
|
elif child.type == "do_block":
|
|
do_block_node = child
|
|
|
|
if identifier_node is None:
|
|
for child in node.children:
|
|
walk(child, parent_module_nid)
|
|
return
|
|
|
|
keyword = source[identifier_node.start_byte:identifier_node.end_byte].decode("utf-8", errors="replace")
|
|
line = node.start_point[0] + 1
|
|
|
|
if keyword == "defmodule":
|
|
module_name = _get_alias_text(arguments_node) if arguments_node else None
|
|
if not module_name:
|
|
return
|
|
module_nid = _make_id(stem, module_name)
|
|
add_node(module_nid, module_name, line)
|
|
add_edge(file_nid, module_nid, "contains", line)
|
|
if do_block_node:
|
|
for child in do_block_node.children:
|
|
walk(child, parent_module_nid=module_nid)
|
|
return
|
|
|
|
if keyword in ("def", "defp"):
|
|
func_name = None
|
|
if arguments_node:
|
|
for child in arguments_node.children:
|
|
if child.type == "call":
|
|
for sub in child.children:
|
|
if sub.type == "identifier":
|
|
func_name = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
|
|
break
|
|
elif child.type == "identifier":
|
|
func_name = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
break
|
|
if not func_name:
|
|
return
|
|
container = parent_module_nid or file_nid
|
|
func_nid = _make_id(container, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
if parent_module_nid:
|
|
add_edge(parent_module_nid, func_nid, "method", line)
|
|
else:
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
if do_block_node:
|
|
function_bodies.append((func_nid, do_block_node))
|
|
return
|
|
|
|
if keyword in _IMPORT_KEYWORDS and arguments_node:
|
|
module_name = _get_alias_text(arguments_node)
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
add_edge(file_nid, tgt_nid, "imports", line, context="import")
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_module_nid)
|
|
|
|
walk(root)
|
|
|
|
label_to_nid: dict[str, str] = {}
|
|
for n in nodes:
|
|
normalised = n["label"].strip("()").lstrip(".")
|
|
label_to_nid[normalised.lower()] = n["id"]
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
_SKIP_KEYWORDS = frozenset({
|
|
"def", "defp", "defmodule", "defmacro", "defmacrop",
|
|
"defstruct", "defprotocol", "defimpl", "defguard",
|
|
"alias", "import", "require", "use",
|
|
"if", "unless", "case", "cond", "with", "for",
|
|
})
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type != "call":
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
return
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
kw = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
if kw in _SKIP_KEYWORDS:
|
|
for c in node.children:
|
|
walk_calls(c, caller_nid)
|
|
return
|
|
break
|
|
callee_name: str | None = None
|
|
is_member_call: bool = False
|
|
for child in node.children:
|
|
if child.type == "dot":
|
|
is_member_call = True
|
|
dot_text = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
parts = dot_text.rstrip(".").split(".")
|
|
if parts:
|
|
callee_name = parts[-1]
|
|
break
|
|
if child.type == "identifier":
|
|
callee_name = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
break
|
|
if callee_name:
|
|
tgt_nid = label_to_nid.get(callee_name.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
add_edge(caller_nid, tgt_nid, "calls",
|
|
node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0,
|
|
context="call")
|
|
else:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee_name,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body in function_bodies:
|
|
walk_calls(body, caller_nid)
|
|
|
|
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
|
(e["target"] in seen_ids or e["relation"] == "imports")]
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
# ── Main extract and collect_files ────────────────────────────────────────────
|
|
|
|
|
|
def _check_tree_sitter_version() -> None:
|
|
"""Raise a clear error if tree-sitter is too old for the new Language API."""
|
|
try:
|
|
from tree_sitter import LANGUAGE_VERSION
|
|
except ImportError:
|
|
raise ImportError(
|
|
"tree-sitter is not installed. Run: pip install 'tree-sitter>=0.23.0'"
|
|
)
|
|
# Language API v2 starts at LANGUAGE_VERSION 14
|
|
if LANGUAGE_VERSION < 14:
|
|
import tree_sitter as _ts
|
|
raise RuntimeError(
|
|
f"tree-sitter {getattr(_ts, '__version__', 'unknown')} is too old. "
|
|
f"graphify requires tree-sitter >= 0.23.0 (Language API v2). "
|
|
f"Run: pip install --upgrade tree-sitter"
|
|
)
|
|
|
|
|
|
_DISPATCH: dict[str, Any] = {
|
|
".py": extract_python,
|
|
".js": extract_js,
|
|
".jsx": extract_js,
|
|
".mjs": extract_js,
|
|
".ts": extract_js,
|
|
".tsx": extract_js,
|
|
".go": extract_go,
|
|
".rs": extract_rust,
|
|
".java": extract_java,
|
|
".c": extract_c,
|
|
".h": extract_c,
|
|
".cpp": extract_cpp,
|
|
".cc": extract_cpp,
|
|
".cxx": extract_cpp,
|
|
".hpp": extract_cpp,
|
|
".rb": extract_ruby,
|
|
".cs": extract_csharp,
|
|
".kt": extract_kotlin,
|
|
".kts": extract_kotlin,
|
|
".scala": extract_scala,
|
|
".php": extract_php,
|
|
".swift": extract_swift,
|
|
".lua": extract_lua,
|
|
".toc": extract_lua,
|
|
".zig": extract_zig,
|
|
".ps1": extract_powershell,
|
|
".ex": extract_elixir,
|
|
".exs": extract_elixir,
|
|
".m": extract_objc,
|
|
".mm": extract_objc,
|
|
".jl": extract_julia,
|
|
".f": extract_fortran,
|
|
".F": extract_fortran,
|
|
".f90": extract_fortran,
|
|
".F90": extract_fortran,
|
|
".f95": extract_fortran,
|
|
".F95": extract_fortran,
|
|
".f03": extract_fortran,
|
|
".F03": extract_fortran,
|
|
".f08": extract_fortran,
|
|
".F08": extract_fortran,
|
|
".vue": extract_js,
|
|
".svelte": extract_svelte,
|
|
".dart": extract_dart,
|
|
".v": extract_verilog,
|
|
".sv": extract_verilog,
|
|
".sql": extract_sql,
|
|
}
|
|
|
|
|
|
def _get_extractor(path: Path) -> Any | None:
|
|
"""Return the correct extractor function for a file, or None if unsupported."""
|
|
if path.name.endswith(".blade.php"):
|
|
return extract_blade
|
|
return _DISPATCH.get(path.suffix)
|
|
|
|
|
|
def _extract_single_file(args: tuple) -> tuple[int, dict]:
|
|
"""Worker function for parallel extraction. Runs in a subprocess.
|
|
|
|
Must be at module level (not a closure) so it can be pickled by
|
|
ProcessPoolExecutor.
|
|
|
|
Args:
|
|
args: (index, path_str, cache_root_str) tuple
|
|
|
|
Returns:
|
|
(index, result_dict) so results can be placed back in order.
|
|
"""
|
|
idx, path_str, cache_root_str = args
|
|
path = Path(path_str)
|
|
cache_root = Path(cache_root_str)
|
|
_raise_recursion_limit()
|
|
|
|
# Check cache first (avoid re-extraction)
|
|
cached = load_cached(path, cache_root)
|
|
if cached is not None:
|
|
return idx, cached
|
|
|
|
extractor = _get_extractor(path)
|
|
if extractor is None:
|
|
return idx, {"nodes": [], "edges": []}
|
|
|
|
result = _safe_extract(extractor, path)
|
|
if "error" not in result:
|
|
save_cached(path, result, cache_root)
|
|
return idx, result
|
|
|
|
|
|
def _extract_parallel(
|
|
uncached_work: list[tuple[int, Path]],
|
|
per_file: list[dict | None],
|
|
effective_root: Path,
|
|
max_workers: int | None,
|
|
total_files: int,
|
|
) -> None:
|
|
"""Extract uncached files in parallel using ProcessPoolExecutor."""
|
|
import concurrent.futures
|
|
|
|
if max_workers is None:
|
|
max_workers = min(os.cpu_count() or 4, len(uncached_work), 8)
|
|
|
|
root_str = str(effective_root)
|
|
work_items = [(idx, str(path), root_str) for idx, path in uncached_work]
|
|
|
|
done_count = 0
|
|
_PROGRESS_INTERVAL = 100
|
|
with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as pool:
|
|
futures = {
|
|
pool.submit(_extract_single_file, item): item[0] for item in work_items
|
|
}
|
|
for future in concurrent.futures.as_completed(futures):
|
|
idx, result = future.result()
|
|
per_file[idx] = result
|
|
done_count += 1
|
|
if (
|
|
total_files >= _PROGRESS_INTERVAL
|
|
and done_count % _PROGRESS_INTERVAL == 0
|
|
):
|
|
print(
|
|
f" AST extraction: {done_count}/{len(uncached_work)} uncached files "
|
|
f"({done_count * 100 // len(uncached_work)}%) [{max_workers} workers]",
|
|
flush=True,
|
|
)
|
|
if total_files >= _PROGRESS_INTERVAL:
|
|
print(
|
|
f" AST extraction: {total_files}/{total_files} files (100%) [{max_workers} workers]",
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def _extract_sequential(
|
|
uncached_work: list[tuple[int, Path]],
|
|
per_file: list[dict | None],
|
|
effective_root: Path,
|
|
total_files: int,
|
|
) -> None:
|
|
"""Extract uncached files sequentially (fallback for small batches)."""
|
|
_PROGRESS_INTERVAL = 100
|
|
for work_idx, (idx, path) in enumerate(uncached_work):
|
|
if (
|
|
total_files >= _PROGRESS_INTERVAL
|
|
and work_idx % _PROGRESS_INTERVAL == 0
|
|
and work_idx > 0
|
|
):
|
|
print(
|
|
f" AST extraction: {work_idx}/{len(uncached_work)} uncached files ({work_idx * 100 // len(uncached_work)}%)",
|
|
flush=True,
|
|
)
|
|
extractor = _get_extractor(path)
|
|
if extractor is None:
|
|
per_file[idx] = {"nodes": [], "edges": []}
|
|
continue
|
|
result = _safe_extract(extractor, path)
|
|
if "error" not in result:
|
|
save_cached(path, result, effective_root)
|
|
per_file[idx] = result
|
|
if total_files >= _PROGRESS_INTERVAL:
|
|
print(f" AST extraction: {total_files}/{total_files} files (100%)", flush=True)
|
|
|
|
|
|
_PARALLEL_THRESHOLD = 20
|
|
|
|
|
|
def extract(
|
|
paths: list[Path],
|
|
cache_root: Path | None = None,
|
|
*,
|
|
parallel: bool = True,
|
|
max_workers: int | None = None,
|
|
) -> dict:
|
|
"""Extract AST nodes and edges from a list of code files.
|
|
|
|
Two-pass process:
|
|
1. Per-file structural extraction (classes, functions, imports)
|
|
2. Cross-file import resolution: turns file-level imports into
|
|
class-level INFERRED edges (DigestAuth --uses--> Response)
|
|
|
|
Args:
|
|
paths: files to extract from
|
|
cache_root: explicit root for graphify-out/cache/ (overrides the
|
|
inferred common path prefix). Pass Path('.') when running on a
|
|
subdirectory so the cache stays at ./graphify-out/cache/.
|
|
parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files,
|
|
use ProcessPoolExecutor for multi-core extraction.
|
|
max_workers: max subprocess count. Defaults to min(cpu_count, 8).
|
|
"""
|
|
_check_tree_sitter_version()
|
|
_raise_recursion_limit()
|
|
|
|
# Infer a common root for cache keys (use first diverging segment, not sum of all matches)
|
|
try:
|
|
if not paths:
|
|
root = Path(".")
|
|
elif len(paths) == 1:
|
|
root = paths[0].parent
|
|
else:
|
|
min_parts = min(len(p.parts) for p in paths)
|
|
common_len = 0
|
|
for i in range(min_parts):
|
|
if len({p.parts[i] for p in paths}) == 1:
|
|
common_len += 1
|
|
else:
|
|
break
|
|
root = Path(*paths[0].parts[:common_len]) if common_len else Path(".")
|
|
except Exception:
|
|
root = Path(".")
|
|
root = root.resolve()
|
|
|
|
effective_root = cache_root or root
|
|
total = len(paths)
|
|
|
|
# Phase 1: separate cached hits from uncached work
|
|
per_file: list[dict | None] = [None] * total
|
|
uncached_work: list[tuple[int, Path]] = []
|
|
|
|
for i, path in enumerate(paths):
|
|
if _get_extractor(path) is None:
|
|
per_file[i] = {"nodes": [], "edges": []}
|
|
continue
|
|
cached = load_cached(path, effective_root)
|
|
if cached is not None:
|
|
per_file[i] = cached
|
|
continue
|
|
uncached_work.append((i, path))
|
|
|
|
# Phase 2: extract uncached files (parallel or sequential)
|
|
if uncached_work:
|
|
if parallel and len(uncached_work) >= _PARALLEL_THRESHOLD:
|
|
_extract_parallel(
|
|
uncached_work, per_file, effective_root, max_workers, total
|
|
)
|
|
else:
|
|
_extract_sequential(uncached_work, per_file, effective_root, total)
|
|
|
|
# Fill any remaining None slots (shouldn't happen, but defensive)
|
|
for i in range(total):
|
|
if per_file[i] is None:
|
|
per_file[i] = {"nodes": [], "edges": []}
|
|
|
|
all_nodes: list[dict] = []
|
|
all_edges: list[dict] = []
|
|
for result in per_file:
|
|
all_nodes.extend(result.get("nodes", []))
|
|
all_edges.extend(result.get("edges", []))
|
|
|
|
# Remap file node IDs from absolute-path-derived to project-relative so
|
|
# graph.json edge endpoints are stable across machines (#502)
|
|
id_remap: dict[str, str] = {}
|
|
for path in paths:
|
|
old_id = _make_id(str(path))
|
|
try:
|
|
new_id = _make_id(str(path.relative_to(root)))
|
|
except ValueError:
|
|
continue
|
|
if old_id != new_id:
|
|
id_remap[old_id] = new_id
|
|
if id_remap:
|
|
for n in all_nodes:
|
|
if n.get("id") in id_remap:
|
|
n["id"] = id_remap[n["id"]]
|
|
for e in all_edges:
|
|
if e.get("source") in id_remap:
|
|
e["source"] = id_remap[e["source"]]
|
|
if e.get("target") in id_remap:
|
|
e["target"] = id_remap[e["target"]]
|
|
|
|
# Add cross-file class-level edges (Python only - uses Python parser internally)
|
|
py_paths = [p for p in paths if p.suffix == ".py"]
|
|
if py_paths:
|
|
py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"]
|
|
try:
|
|
cross_file_edges = _resolve_cross_file_imports(py_results, py_paths)
|
|
all_edges.extend(cross_file_edges)
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("Cross-file import resolution failed, skipping: %s", exc)
|
|
|
|
# Cross-file Java import resolution
|
|
java_paths = [p for p in paths if p.suffix == ".java"]
|
|
if java_paths:
|
|
java_results = [r for r, p in zip(per_file, paths) if p.suffix == ".java"]
|
|
try:
|
|
all_edges.extend(_resolve_cross_file_java_imports(java_results, java_paths))
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("Java cross-file import resolution failed, skipping: %s", exc)
|
|
|
|
# Cross-file call resolution for all languages
|
|
# Each extractor saved unresolved calls in raw_calls. Now that we have all
|
|
# nodes from all files, resolve any callee that exists in another file.
|
|
# Build name → ALL matching node IDs so we can skip ambiguous common names
|
|
# (e.g. "log", "execute", "find") that appear in multiple files — resolving
|
|
# those inflates god_nodes ranking with spurious cross-file edges.
|
|
# Build label -> node_id index for cross-file call resolution.
|
|
# Skip rationale nodes (their labels are docstring text, not callable
|
|
# identifiers, and they were polluting matches for short names — #563).
|
|
global_label_to_nids: dict[str, list[str]] = {}
|
|
for n in all_nodes:
|
|
if n.get("file_type") == "rationale":
|
|
continue
|
|
raw = n.get("label", "")
|
|
normalised = raw.strip("()").lstrip(".")
|
|
if normalised:
|
|
key = normalised.lower()
|
|
global_label_to_nids.setdefault(key, []).append(n["id"])
|
|
|
|
existing_pairs = {(e["source"], e["target"]) for e in all_edges}
|
|
for result in per_file:
|
|
for rc in result.get("raw_calls", []):
|
|
callee = rc.get("callee", "")
|
|
if not callee:
|
|
continue
|
|
# Skip member-call callees: obj.log() → "log" has no import evidence
|
|
# and collides with any top-level function named "log" in the corpus.
|
|
if rc.get("is_member_call"):
|
|
continue
|
|
candidates = global_label_to_nids.get(callee.lower(), [])
|
|
# Skip ambiguous names that resolve to multiple nodes — these are
|
|
# common short names (log, execute, find) with no import evidence
|
|
# to pick the right target; emitting all edges inflates god_nodes.
|
|
if len(candidates) != 1:
|
|
continue
|
|
tgt = candidates[0]
|
|
caller = rc["caller_nid"]
|
|
if tgt != caller and (caller, tgt) not in existing_pairs:
|
|
existing_pairs.add((caller, tgt))
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": tgt,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "INFERRED",
|
|
"confidence_score": 0.8,
|
|
"source_file": rc.get("source_file", ""),
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Relativize source_file fields so paths are portable across machines (#555)
|
|
for item in all_nodes + all_edges:
|
|
sf = item.get("source_file")
|
|
if not sf:
|
|
continue
|
|
sf_path = Path(sf)
|
|
if not sf_path.is_absolute():
|
|
continue
|
|
try:
|
|
item["source_file"] = str(sf_path.relative_to(root))
|
|
except ValueError:
|
|
pass
|
|
|
|
return {
|
|
"nodes": all_nodes,
|
|
"edges": all_edges,
|
|
"input_tokens": 0,
|
|
"output_tokens": 0,
|
|
}
|
|
|
|
|
|
def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | None = None) -> list[Path]:
|
|
if target.is_file():
|
|
return [target]
|
|
_EXTENSIONS = {
|
|
".py", ".js", ".ts", ".tsx", ".go", ".rs",
|
|
".java", ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp",
|
|
".rb", ".cs", ".kt", ".kts", ".scala", ".php", ".swift",
|
|
".lua", ".toc", ".zig", ".ps1",
|
|
".m", ".mm",
|
|
}
|
|
from graphify.detect import _load_graphifyignore, _is_ignored
|
|
ignore_root = root if root is not None else target
|
|
patterns = _load_graphifyignore(ignore_root)
|
|
|
|
def _ignored(p: Path) -> bool:
|
|
return bool(patterns and _is_ignored(p, ignore_root, patterns))
|
|
|
|
if not follow_symlinks:
|
|
results: list[Path] = []
|
|
for ext in sorted(_EXTENSIONS):
|
|
results.extend(
|
|
p for p in target.rglob(f"*{ext}")
|
|
if not any(part.startswith(".") for part in p.parts)
|
|
and not _ignored(p)
|
|
)
|
|
return sorted(results)
|
|
# Walk with symlink following + cycle detection
|
|
results = []
|
|
for dirpath, dirnames, filenames in os.walk(target, followlinks=True):
|
|
if os.path.islink(dirpath):
|
|
real = os.path.realpath(dirpath)
|
|
parent_real = os.path.realpath(os.path.dirname(dirpath))
|
|
if parent_real == real or parent_real.startswith(real + os.sep):
|
|
dirnames.clear()
|
|
continue
|
|
dp = Path(dirpath)
|
|
if any(part.startswith(".") for part in dp.parts):
|
|
dirnames.clear()
|
|
continue
|
|
for fname in filenames:
|
|
p = dp / fname
|
|
if p.suffix in _EXTENSIONS and not fname.startswith(".") and not _ignored(p):
|
|
results.append(p)
|
|
return sorted(results)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
print("Usage: python -m graphify.extract <file_or_dir> ...", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
paths: list[Path] = []
|
|
for arg in sys.argv[1:]:
|
|
paths.extend(collect_files(Path(arg)))
|
|
|
|
result = extract(paths)
|
|
print(json.dumps(result, indent=2))
|