Index .psm1, anchor Swift import targets, dedupe no-cluster edges

- #1315: add .psm1 to CODE_EXTENSIONS + _DISPATCH so PowerShell modules are indexed
- #1327: synthesize a module node for Swift import targets (new LanguageConfig
  flag synthesize_import_module_nodes) so imports edges survive build.py pruning;
  strengthen the Swift dangling-edge test to also assert edge targets
- #1317: dedupe parallel edges by (source,target,relation) in the --no-cluster
  and incremental update write paths so edge counts are deterministic and
  `update` is idempotent

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Safi
2026-06-16 02:28:16 +01:00
co-authored by Claude Opus 4.8
parent 09da5294e4
commit adb52a1ea0
8 changed files with 131 additions and 4 deletions
+4
View File
@@ -4425,7 +4425,11 @@ def main() -> None:
if no_cluster:
# --no-cluster: dump the raw merged extraction as graph.json.
# No NetworkX, no community detection, no analysis sidecar.
# Dedupe parallel edges so counts match the clustered path (whose
# DiGraph collapses them) and stay deterministic across modes (#1317).
from graphify.build import dedupe_edges as _dedupe_edges
from graphify.export import backup_if_protected as _backup
merged["edges"] = _dedupe_edges(merged["edges"])
_backup(graphify_out)
graph_json_path.write_text(
json.dumps(merged, indent=2), encoding="utf-8"
+23
View File
@@ -104,6 +104,29 @@ def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]:
return [raw]
def dedupe_edges(edges: list[dict]) -> list[dict]:
"""Collapse exact parallel edges by ``(source, target, relation)``, keeping the
first occurrence.
The clustered build path runs edges through a NetworkX ``DiGraph``, which
collapses parallel edges automatically. The ``--no-cluster`` and incremental
``update`` write paths bypass NetworkX and concatenate edge lists raw, so
duplicates accumulate and edge counts become non-deterministic across build
modes / repeated updates (#1317). Deduping on the connectivity identity is
zero-signal-loss and restores idempotency. Callers that intentionally keep
parallel edges (multigraph output) must not use this.
"""
seen: set[tuple] = set()
out: list[dict] = []
for e in edges:
key = (e.get("source"), e.get("target"), e.get("relation"))
if key in seen:
continue
seen.add(key)
out.append(e)
return out
def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph:
"""Build a NetworkX graph from an extraction dict.
+1 -1
View File
@@ -26,7 +26,7 @@ class FileType(str, Enum):
_MANIFEST_PATH = "graphify-out/manifest.json"
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'}
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'}
DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'}
PAPER_EXTENSIONS = {'.pdf'}
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
+29
View File
@@ -466,6 +466,12 @@ class LanguageConfig:
# Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.)
extra_walk_fn: Callable | None = None
# When True, synthesize a node for each `imports` edge target an import_handler
# emits (carrying an `_import_label` key). Languages whose imports name modules
# rather than resolvable files (e.g. Swift `import CoreKit`) need this, else the
# edge is pruned in build.py for pointing at a non-existent target node.
synthesize_import_module_nodes: bool = False
# ── Generic helpers ───────────────────────────────────────────────────────────
@@ -2229,6 +2235,9 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st
"source_file": str_path,
"source_location": f"L{node.start_point[0] + 1}",
"weight": 1.0,
# Consumed by import-node synthesis at the walk() call site
# (LanguageConfig.synthesize_import_module_nodes); see #1327.
"_import_label": raw,
})
break
@@ -2267,6 +2276,7 @@ _SWIFT_CONFIG = LanguageConfig(
body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"),
function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}),
import_handler=_import_swift,
synthesize_import_module_nodes=True,
)
# ── Generic extractor ─────────────────────────────────────────────────────────
@@ -2372,7 +2382,25 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict:
# Import types
if t in config.import_types:
if config.import_handler:
_imp_before = len(edges)
config.import_handler(node, source, file_nid, stem, edges, str_path)
if config.synthesize_import_module_nodes:
# Imports that name a module (not a resolvable file) point at a
# synthetic target node; create it so build.py keeps the edge (#1327).
for _e in edges[_imp_before:]:
_lbl = _e.pop("_import_label", None)
if _lbl is None or _e.get("relation") != "imports":
continue
_tgt = _e["target"]
if _tgt not in seen_ids:
seen_ids.add(_tgt)
nodes.append({
"id": _tgt,
"label": _lbl,
"file_type": "code",
"source_file": str_path,
"source_location": _e.get("source_location", "L1"),
})
# For export_statement: only return (skip children) if it's a re-export
# (has a `from` source). Otherwise fall through to walk children which may
# contain function_declaration, class_declaration, etc.
@@ -11568,6 +11596,7 @@ _DISPATCH: dict[str, Any] = {
".toc": extract_lua,
".zig": extract_zig,
".ps1": extract_powershell,
".psm1": extract_powershell,
".ex": extract_elixir,
".exs": extract_elixir,
".m": extract_objc,
+7 -2
View File
@@ -584,9 +584,13 @@ def _rebuild_code(
if no_cluster:
# Normalise to "links" key so schema is consistent with the full clustered path.
# Dedupe parallel edges (the clustered path's DiGraph collapses them implicitly);
# without it, --no-cluster + repeated `update` accumulate duplicates and edge
# counts diverge across build modes (#1317).
from graphify.build import dedupe_edges as _dedupe_edges
candidate_graph_data = {
**{k: v for k, v in result.items() if k != "edges"},
"links": result.get("edges", []),
"links": _dedupe_edges(result.get("edges", [])),
}
candidate_graph_text = _json_text(candidate_graph_data)
same_graph = False
@@ -624,7 +628,8 @@ def _rebuild_code(
else:
print(
"[graphify watch] Rebuilt (no clustering): "
f"{len(result.get('nodes', []))} nodes, {len(result.get('edges', []))} edges"
f"{len(candidate_graph_data.get('nodes', []))} nodes, "
f"{len(candidate_graph_data.get('links', []))} edges"
)
print(f"[graphify watch] graph.json updated in {out}")
return True
+27 -1
View File
@@ -2,10 +2,36 @@ import json
from pathlib import Path
import networkx as nx
from networkx.readwrite import json_graph
from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas
from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas, dedupe_edges
FIXTURES = Path(__file__).parent / "fixtures"
def test_dedupe_edges_collapses_exact_parallels():
# #1317: --no-cluster / incremental update concatenate edge lists raw.
edges = [
{"source": "a", "target": "b", "relation": "calls", "source_location": "L1"},
{"source": "a", "target": "b", "relation": "calls", "source_location": "L9"}, # dup
{"source": "a", "target": "b", "relation": "imports"}, # different relation: kept
{"source": "b", "target": "c", "relation": "calls"},
]
out = dedupe_edges(edges)
keys = [(e["source"], e["target"], e["relation"]) for e in out]
assert keys == [("a", "b", "calls"), ("a", "b", "imports"), ("b", "c", "calls")]
# first occurrence wins (keeps L1, not L9)
assert out[0]["source_location"] == "L1"
def test_dedupe_edges_is_idempotent():
edges = [
{"source": "a", "target": "b", "relation": "calls"},
{"source": "a", "target": "b", "relation": "calls"},
]
once = dedupe_edges(edges)
twice = dedupe_edges(once + edges) # simulate a second `update` re-concatenating
assert len(once) == 1
assert len(twice) == 1
def load_extraction():
return json.loads((FIXTURES / "extraction.json").read_text())
+4
View File
@@ -11,6 +11,10 @@ def test_classify_python():
def test_classify_typescript():
assert classify_file(Path("bar.ts")) == FileType.CODE
def test_classify_powershell_module():
# #1315: .psm1 modules were never indexed (CODE_EXTENSIONS gap).
assert classify_file(Path("Utils.psm1")) == FileType.CODE
def test_classify_markdown():
assert classify_file(Path("README.md")) == FileType.DOCUMENT
+36
View File
@@ -596,6 +596,28 @@ def test_swift_no_dangling_edges():
node_ids = {n["id"] for n in r["nodes"]}
for e in r["edges"]:
assert e["source"] in node_ids
# #1327: targets must resolve to a node too, else build.py prunes the edge.
assert e["target"] in node_ids, f"dangling target {e['target']} ({e['relation']})"
def test_swift_imports_survive_build():
# #1327: `import Foundation` / `import UIKit` previously emitted edges to bare
# module ids with no backing node, so build.py dropped 100% of Swift imports.
from graphify.build import build_from_json
r = extract_swift(FIXTURES / "sample.swift")
import_edges = [e for e in r["edges"] if e["relation"] == "imports"]
assert import_edges, "extractor should emit Swift import edges"
node_ids = {n["id"] for n in r["nodes"]}
for e in import_edges:
assert e["target"] in node_ids # synthesized module node exists
# No private bookkeeping key should leak into output edges.
assert all("_import_label" not in e for e in r["edges"])
# Edges must survive the build (which prunes edges with unknown endpoints).
G = build_from_json(r)
surviving = [
(u, v) for u, v, d in G.edges(data=True) if d.get("relation") == "imports"
]
assert surviving, "Swift import edges must survive build_from_json (#1327)"
def test_swift_finds_actor():
r = extract_swift(FIXTURES / "sample.swift")
@@ -1005,6 +1027,20 @@ def test_powershell_no_error():
assert "error" not in r
def test_powershell_psm1_dispatched_and_extracted(tmp_path):
# #1315: .psm1 modules were never indexed — no dispatch entry, no CODE_EXTENSIONS.
from graphify.extract import _get_extractor
mod = tmp_path / "Utils.psm1"
mod.write_text(
"function Get-Greeting { param([string]$Name) return \"Hi $Name\" }\n",
encoding="utf-8",
)
assert _get_extractor(mod) is extract_powershell
r = extract_powershell(mod)
assert "error" not in r
assert any("Get-Greeting" in n["label"] for n in r["nodes"])
def test_powershell_finds_class_and_method():
r = extract_powershell(FIXTURES / "sample.ps1")
labels = [n["label"] for n in r["nodes"]]