Index .psm1, anchor Swift import targets, dedupe no-cluster edges
- #1315: add .psm1 to CODE_EXTENSIONS + _DISPATCH so PowerShell modules are indexed - #1327: synthesize a module node for Swift import targets (new LanguageConfig flag synthesize_import_module_nodes) so imports edges survive build.py pruning; strengthen the Swift dangling-edge test to also assert edge targets - #1317: dedupe parallel edges by (source,target,relation) in the --no-cluster and incremental update write paths so edge counts are deterministic and `update` is idempotent Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -4425,7 +4425,11 @@ def main() -> None:
|
||||
if no_cluster:
|
||||
# --no-cluster: dump the raw merged extraction as graph.json.
|
||||
# No NetworkX, no community detection, no analysis sidecar.
|
||||
# Dedupe parallel edges so counts match the clustered path (whose
|
||||
# DiGraph collapses them) and stay deterministic across modes (#1317).
|
||||
from graphify.build import dedupe_edges as _dedupe_edges
|
||||
from graphify.export import backup_if_protected as _backup
|
||||
merged["edges"] = _dedupe_edges(merged["edges"])
|
||||
_backup(graphify_out)
|
||||
graph_json_path.write_text(
|
||||
json.dumps(merged, indent=2), encoding="utf-8"
|
||||
|
||||
@@ -104,6 +104,29 @@ def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]:
|
||||
return [raw]
|
||||
|
||||
|
||||
def dedupe_edges(edges: list[dict]) -> list[dict]:
|
||||
"""Collapse exact parallel edges by ``(source, target, relation)``, keeping the
|
||||
first occurrence.
|
||||
|
||||
The clustered build path runs edges through a NetworkX ``DiGraph``, which
|
||||
collapses parallel edges automatically. The ``--no-cluster`` and incremental
|
||||
``update`` write paths bypass NetworkX and concatenate edge lists raw, so
|
||||
duplicates accumulate and edge counts become non-deterministic across build
|
||||
modes / repeated updates (#1317). Deduping on the connectivity identity is
|
||||
zero-signal-loss and restores idempotency. Callers that intentionally keep
|
||||
parallel edges (multigraph output) must not use this.
|
||||
"""
|
||||
seen: set[tuple] = set()
|
||||
out: list[dict] = []
|
||||
for e in edges:
|
||||
key = (e.get("source"), e.get("target"), e.get("relation"))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
|
||||
def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph:
|
||||
"""Build a NetworkX graph from an extraction dict.
|
||||
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ class FileType(str, Enum):
|
||||
|
||||
_MANIFEST_PATH = "graphify-out/manifest.json"
|
||||
|
||||
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'}
|
||||
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'}
|
||||
DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'}
|
||||
PAPER_EXTENSIONS = {'.pdf'}
|
||||
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
|
||||
|
||||
@@ -466,6 +466,12 @@ class LanguageConfig:
|
||||
# Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.)
|
||||
extra_walk_fn: Callable | None = None
|
||||
|
||||
# When True, synthesize a node for each `imports` edge target an import_handler
|
||||
# emits (carrying an `_import_label` key). Languages whose imports name modules
|
||||
# rather than resolvable files (e.g. Swift `import CoreKit`) need this, else the
|
||||
# edge is pruned in build.py for pointing at a non-existent target node.
|
||||
synthesize_import_module_nodes: bool = False
|
||||
|
||||
|
||||
# ── Generic helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
@@ -2229,6 +2235,9 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{node.start_point[0] + 1}",
|
||||
"weight": 1.0,
|
||||
# Consumed by import-node synthesis at the walk() call site
|
||||
# (LanguageConfig.synthesize_import_module_nodes); see #1327.
|
||||
"_import_label": raw,
|
||||
})
|
||||
break
|
||||
|
||||
@@ -2267,6 +2276,7 @@ _SWIFT_CONFIG = LanguageConfig(
|
||||
body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"),
|
||||
function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}),
|
||||
import_handler=_import_swift,
|
||||
synthesize_import_module_nodes=True,
|
||||
)
|
||||
|
||||
# ── Generic extractor ─────────────────────────────────────────────────────────
|
||||
@@ -2372,7 +2382,25 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict:
|
||||
# Import types
|
||||
if t in config.import_types:
|
||||
if config.import_handler:
|
||||
_imp_before = len(edges)
|
||||
config.import_handler(node, source, file_nid, stem, edges, str_path)
|
||||
if config.synthesize_import_module_nodes:
|
||||
# Imports that name a module (not a resolvable file) point at a
|
||||
# synthetic target node; create it so build.py keeps the edge (#1327).
|
||||
for _e in edges[_imp_before:]:
|
||||
_lbl = _e.pop("_import_label", None)
|
||||
if _lbl is None or _e.get("relation") != "imports":
|
||||
continue
|
||||
_tgt = _e["target"]
|
||||
if _tgt not in seen_ids:
|
||||
seen_ids.add(_tgt)
|
||||
nodes.append({
|
||||
"id": _tgt,
|
||||
"label": _lbl,
|
||||
"file_type": "code",
|
||||
"source_file": str_path,
|
||||
"source_location": _e.get("source_location", "L1"),
|
||||
})
|
||||
# For export_statement: only return (skip children) if it's a re-export
|
||||
# (has a `from` source). Otherwise fall through to walk children which may
|
||||
# contain function_declaration, class_declaration, etc.
|
||||
@@ -11568,6 +11596,7 @@ _DISPATCH: dict[str, Any] = {
|
||||
".toc": extract_lua,
|
||||
".zig": extract_zig,
|
||||
".ps1": extract_powershell,
|
||||
".psm1": extract_powershell,
|
||||
".ex": extract_elixir,
|
||||
".exs": extract_elixir,
|
||||
".m": extract_objc,
|
||||
|
||||
+7
-2
@@ -584,9 +584,13 @@ def _rebuild_code(
|
||||
|
||||
if no_cluster:
|
||||
# Normalise to "links" key so schema is consistent with the full clustered path.
|
||||
# Dedupe parallel edges (the clustered path's DiGraph collapses them implicitly);
|
||||
# without it, --no-cluster + repeated `update` accumulate duplicates and edge
|
||||
# counts diverge across build modes (#1317).
|
||||
from graphify.build import dedupe_edges as _dedupe_edges
|
||||
candidate_graph_data = {
|
||||
**{k: v for k, v in result.items() if k != "edges"},
|
||||
"links": result.get("edges", []),
|
||||
"links": _dedupe_edges(result.get("edges", [])),
|
||||
}
|
||||
candidate_graph_text = _json_text(candidate_graph_data)
|
||||
same_graph = False
|
||||
@@ -624,7 +628,8 @@ def _rebuild_code(
|
||||
else:
|
||||
print(
|
||||
"[graphify watch] Rebuilt (no clustering): "
|
||||
f"{len(result.get('nodes', []))} nodes, {len(result.get('edges', []))} edges"
|
||||
f"{len(candidate_graph_data.get('nodes', []))} nodes, "
|
||||
f"{len(candidate_graph_data.get('links', []))} edges"
|
||||
)
|
||||
print(f"[graphify watch] graph.json updated in {out}")
|
||||
return True
|
||||
|
||||
+27
-1
@@ -2,10 +2,36 @@ import json
|
||||
from pathlib import Path
|
||||
import networkx as nx
|
||||
from networkx.readwrite import json_graph
|
||||
from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas
|
||||
from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas, dedupe_edges
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_dedupe_edges_collapses_exact_parallels():
|
||||
# #1317: --no-cluster / incremental update concatenate edge lists raw.
|
||||
edges = [
|
||||
{"source": "a", "target": "b", "relation": "calls", "source_location": "L1"},
|
||||
{"source": "a", "target": "b", "relation": "calls", "source_location": "L9"}, # dup
|
||||
{"source": "a", "target": "b", "relation": "imports"}, # different relation: kept
|
||||
{"source": "b", "target": "c", "relation": "calls"},
|
||||
]
|
||||
out = dedupe_edges(edges)
|
||||
keys = [(e["source"], e["target"], e["relation"]) for e in out]
|
||||
assert keys == [("a", "b", "calls"), ("a", "b", "imports"), ("b", "c", "calls")]
|
||||
# first occurrence wins (keeps L1, not L9)
|
||||
assert out[0]["source_location"] == "L1"
|
||||
|
||||
|
||||
def test_dedupe_edges_is_idempotent():
|
||||
edges = [
|
||||
{"source": "a", "target": "b", "relation": "calls"},
|
||||
{"source": "a", "target": "b", "relation": "calls"},
|
||||
]
|
||||
once = dedupe_edges(edges)
|
||||
twice = dedupe_edges(once + edges) # simulate a second `update` re-concatenating
|
||||
assert len(once) == 1
|
||||
assert len(twice) == 1
|
||||
|
||||
def load_extraction():
|
||||
return json.loads((FIXTURES / "extraction.json").read_text())
|
||||
|
||||
|
||||
@@ -11,6 +11,10 @@ def test_classify_python():
|
||||
def test_classify_typescript():
|
||||
assert classify_file(Path("bar.ts")) == FileType.CODE
|
||||
|
||||
def test_classify_powershell_module():
|
||||
# #1315: .psm1 modules were never indexed (CODE_EXTENSIONS gap).
|
||||
assert classify_file(Path("Utils.psm1")) == FileType.CODE
|
||||
|
||||
def test_classify_markdown():
|
||||
assert classify_file(Path("README.md")) == FileType.DOCUMENT
|
||||
|
||||
|
||||
@@ -596,6 +596,28 @@ def test_swift_no_dangling_edges():
|
||||
node_ids = {n["id"] for n in r["nodes"]}
|
||||
for e in r["edges"]:
|
||||
assert e["source"] in node_ids
|
||||
# #1327: targets must resolve to a node too, else build.py prunes the edge.
|
||||
assert e["target"] in node_ids, f"dangling target {e['target']} ({e['relation']})"
|
||||
|
||||
|
||||
def test_swift_imports_survive_build():
|
||||
# #1327: `import Foundation` / `import UIKit` previously emitted edges to bare
|
||||
# module ids with no backing node, so build.py dropped 100% of Swift imports.
|
||||
from graphify.build import build_from_json
|
||||
r = extract_swift(FIXTURES / "sample.swift")
|
||||
import_edges = [e for e in r["edges"] if e["relation"] == "imports"]
|
||||
assert import_edges, "extractor should emit Swift import edges"
|
||||
node_ids = {n["id"] for n in r["nodes"]}
|
||||
for e in import_edges:
|
||||
assert e["target"] in node_ids # synthesized module node exists
|
||||
# No private bookkeeping key should leak into output edges.
|
||||
assert all("_import_label" not in e for e in r["edges"])
|
||||
# Edges must survive the build (which prunes edges with unknown endpoints).
|
||||
G = build_from_json(r)
|
||||
surviving = [
|
||||
(u, v) for u, v, d in G.edges(data=True) if d.get("relation") == "imports"
|
||||
]
|
||||
assert surviving, "Swift import edges must survive build_from_json (#1327)"
|
||||
|
||||
def test_swift_finds_actor():
|
||||
r = extract_swift(FIXTURES / "sample.swift")
|
||||
@@ -1005,6 +1027,20 @@ def test_powershell_no_error():
|
||||
assert "error" not in r
|
||||
|
||||
|
||||
def test_powershell_psm1_dispatched_and_extracted(tmp_path):
|
||||
# #1315: .psm1 modules were never indexed — no dispatch entry, no CODE_EXTENSIONS.
|
||||
from graphify.extract import _get_extractor
|
||||
mod = tmp_path / "Utils.psm1"
|
||||
mod.write_text(
|
||||
"function Get-Greeting { param([string]$Name) return \"Hi $Name\" }\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert _get_extractor(mod) is extract_powershell
|
||||
r = extract_powershell(mod)
|
||||
assert "error" not in r
|
||||
assert any("Get-Greeting" in n["label"] for n in r["nodes"])
|
||||
|
||||
|
||||
def test_powershell_finds_class_and_method():
|
||||
r = extract_powershell(FIXTURES / "sample.ps1")
|
||||
labels = [n["label"] for n in r["nodes"]]
|
||||
|
||||
Reference in New Issue
Block a user