Merge pull request #1271 from giovanecesar/feat/cargo-introspect

feat(extract): add opt-in --cargo crate dependency extractor

README.md

@@ -574,6 +574,7 @@ graphify extract ./docs --backend claude-cli   # route through Claude Code CLI -
 graphify extract ./docs --backend azure        # Azure OpenAI (set AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT)
 graphify extract ./docs --max-workers 16       # AST parallelism (also GRAPHIFY_MAX_WORKERS)
 graphify extract --postgres "postgresql://user:pass@host/db"   # introspect live PostgreSQL schema directly
+graphify extract ./my-workspace --cargo        # introspect Rust Cargo workspace dependencies directly
 graphify extract ./docs --token-budget 30000   # smaller semantic chunks for local/small models
 graphify extract ./docs --max-concurrency 2    # fewer parallel LLM calls (useful for local inference)
 graphify extract ./docs --api-timeout 900      # longer HTTP timeout for slow local models (default 600s)

graphify/__main__.py

@@ -2182,6 +2182,7 @@ def main() -> None:
         print("    --postgres DSN          extract schema from a live PostgreSQL database")
         print("                            maps tables, views, functions + FK relationships;")
         print("                            column-level detail is not represented in the graph")
+        print("    --cargo                 extract crate→crate deps from Cargo.toml")
         print("    --global                also merge the resulting graph into the global graph")
         print("    --as <tag>              repo tag for --global (default: target directory name)")
         print("  global add <graph.json>  add/update a project graph in the global graph (~/.graphify/global-graph.json)")
@@ -3904,7 +3905,7 @@ def main() -> None:
                 "Usage: graphify extract <path> [--backend gemini|kimi|claude|openai|deepseek|ollama] "
                 "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] "
                 "[--max-workers N] [--token-budget N] [--max-concurrency N] "
-                "[--api-timeout S] [--postgres DSN]",
+                "[--api-timeout S] [--postgres DSN] [--cargo]",
                 file=sys.stderr,
             )
             sys.exit(1)
@@ -3924,6 +3925,7 @@ def main() -> None:
         extract_mode: str | None = None
         out_dir: Path | None = None
         cli_postgres_dsn: str | None = None
+        cli_cargo: bool = False
         no_cluster = False
         dedup_llm = False
         google_workspace = False
@@ -4023,6 +4025,9 @@ def main() -> None:
                 cli_postgres_dsn = args[i + 1]; i += 2
             elif a.startswith("--postgres="):
                 cli_postgres_dsn = a.split("=", 1)[1]; i += 1
+            elif a == "--cargo":
+                cli_cargo = True
+                i += 1
             else:
                 i += 1
 
@@ -4325,13 +4330,25 @@ def main() -> None:
             print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, "
                   f"{len(pg_result['edges'])} edges")
 
-        # Merge AST + semantic + pg_result. Order matters for deduplication: passing AST
+        cargo_result: dict = {"nodes": [], "edges": []}
+        if cli_cargo:
+            from graphify.cargo_introspect import introspect_cargo
+            print("[graphify extract] introspecting Cargo workspace...")
+            try:
+                cargo_result = introspect_cargo(target)
+            except (ConnectionError, ImportError) as exc:
+                print(f"error: {exc}", file=sys.stderr)
+                sys.exit(1)
+            print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, "
+                  f"{len(cargo_result['edges'])} edges")
+
+        # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST
         # first means semantic node attributes win on collision (richer labels
         # for symbols also referenced in docs). Hyperedges only come from the
         # semantic side.
         merged: dict = {
-            "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])),
+            "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])),
-            "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])),
+            "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])),
             "hyperedges": list(sem_result.get("hyperedges", [])),
             "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0),
             "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0),

graphify/cargo_introspect.py

@@ -0,0 +1,98 @@
+"""Cargo manifest introspection for workspace-internal crate dependencies."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+_CONFIDENCE_EXTRACTED = "EXTRACTED"
+
+
+def _load_toml(path: Path) -> dict[str, Any]:
+    try:
+        import tomllib  # type: ignore[import-not-found]
+    except ModuleNotFoundError:
+        try:
+            import tomli as tomllib  # type: ignore[import-not-found,no-redef]
+        except ModuleNotFoundError:
+            raise ImportError(
+                "--cargo on Python 3.10 needs tomli. Install with: pip install tomli"
+            ) from None
+
+    with path.open("rb") as manifest:
+        return tomllib.load(manifest)
+
+
+def _member_manifest_paths(root: Path, root_data: dict[str, Any]) -> list[Path]:
+    paths: list[Path] = []
+    if isinstance(root_data.get("package"), dict):
+        paths.append(root / "Cargo.toml")
+
+    workspace = root_data.get("workspace")
+    members = workspace.get("members", []) if isinstance(workspace, dict) else []
+    if not isinstance(members, list):
+        return paths
+
+    for pattern in members:
+        if not isinstance(pattern, str):
+            continue
+        for member in sorted(root.glob(pattern)):
+            manifest = member / "Cargo.toml"
+            if manifest.is_file() and manifest not in paths:
+                paths.append(manifest)
+    return paths
+
+
+def introspect_cargo(root: str | Path) -> dict[str, Any]:
+    """Return crate nodes and internal dependency edges from Cargo manifests."""
+    root_path = Path(root).resolve()
+    root_manifest = root_path / "Cargo.toml"
+    root_data = _load_toml(root_manifest)
+
+    manifests = _member_manifest_paths(root_path, root_data)
+    crates: dict[str, tuple[str, Path, dict[str, Any]]] = {}
+
+    for manifest in manifests:
+        data = root_data if manifest == root_manifest else _load_toml(manifest)
+        package = data.get("package")
+        if not isinstance(package, dict):
+            continue
+        name = package.get("name")
+        if isinstance(name, str):
+            crates[name] = (f"crate:{name}", manifest, data)
+
+    nodes = [
+        {
+            "id": crate_id,
+            "label": name,
+            "source_file": manifest.relative_to(root_path).as_posix(),
+            "source_location": "L1",
+        }
+        for name, (crate_id, manifest, _data) in sorted(crates.items())
+    ]
+
+    edges: list[dict[str, Any]] = []
+    for source_name, (source_id, manifest, data) in sorted(crates.items()):
+        dependencies = data.get("dependencies", {})
+        if not isinstance(dependencies, dict):
+            continue
+        source_file = manifest.relative_to(root_path).as_posix()
+        for dependency_name in sorted(dependencies):
+            target = crates.get(dependency_name)
+            if target is None:
+                continue
+            edges.append(
+                {
+                    "source": source_id,
+                    "target": target[0],
+                    "relation": "crate_depends_on",
+                    "context": "cargo_dependency",
+                    "weight": 1.0,
+                    "confidence": _CONFIDENCE_EXTRACTED,
+                    "source_file": source_file,
+                    "source_location": "L1",
+                }
+            )
+
+    return {"nodes": nodes, "edges": edges}

graphify/watch.py

@@ -838,7 +838,7 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None:
             nonlocal last_trigger, pending
             if event.is_directory:
                 return
-            path = Path(event.src_path)
+            path = Path(os.fsdecode(event.src_path))
             # Check .graphifyignore BEFORE the extension/dotfile/out filters so
             # the cheapest short-circuit for users with broad ignore patterns
             # (node_modules/, .venv/, build/, …) fires first. _is_ignored
@@ -848,9 +848,13 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None:
                 return
             if path.suffix.lower() not in _WATCHED_EXTENSIONS:
                 return
-            if any(part.startswith(".") for part in path.parts):
+            try:
+                filter_parts = path.relative_to(watch_root_for_ignore).parts
+            except ValueError:
+                filter_parts = path.parts
+            if any(part.startswith(".") for part in filter_parts):
                 return
-            if _GRAPHIFY_OUT in path.parts:
+            if _GRAPHIFY_OUT in filter_parts:
                 return
             last_trigger = time.monotonic()
             pending = True

tests/test_cargo_introspect.py

@@ -0,0 +1,365 @@
+import pytest
+
+from graphify.cargo_introspect import introspect_cargo
+
+
+def _write_manifest(path, content):
+    path.write_text(content.lstrip(), encoding="utf-8")
+
+
+def test_cargo_introspect_workspace_internal_dependency_only(tmp_path):
+    """Real workspace: pin raw graph fields while excluding registry-only deps."""
+    # This exercises actual Cargo.toml discovery from disk, proving internal path
+    # dependencies become edges while external registry packages stay out of the graph.
+    _write_manifest(
+        tmp_path / "Cargo.toml",
+        """
+[workspace]
+members = ["app", "core"]
+""",
+    )
+    app = tmp_path / "app"
+    core = tmp_path / "core"
+    app.mkdir()
+    core.mkdir()
+    _write_manifest(
+        app / "Cargo.toml",
+        """
+[package]
+name = "app"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+core = { path = "../core" }
+serde = "1"
+""",
+    )
+    _write_manifest(
+        core / "Cargo.toml",
+        """
+[package]
+name = "core"
+version = "0.1.0"
+edition = "2021"
+""",
+    )
+
+    result = introspect_cargo(tmp_path)
+
+    node_ids = {node["id"] for node in result["nodes"]}
+    assert node_ids == {"crate:app", "crate:core"}
+    assert "crate:serde" not in node_ids
+    assert {
+        "id": "crate:app",
+        "label": "app",
+        "source_file": "app/Cargo.toml",
+        "source_location": "L1",
+    } in result["nodes"]
+
+    assert {
+        "source": "crate:app",
+        "target": "crate:core",
+        "relation": "crate_depends_on",
+        "context": "cargo_dependency",
+        "weight": 1.0,
+        "confidence": "EXTRACTED",
+        "source_file": "app/Cargo.toml",
+        "source_location": "L1",
+    } in result["edges"]
+    assert not any(
+        edge["source"] == "crate:app" and edge["target"] == "crate:serde"
+        for edge in result["edges"]
+    )
+
+
+def test_cargo_introspect_malformed_toml_reports_parser_error(tmp_path):
+    """Malformed manifests surface the TOML parser failure, not an arbitrary crash."""
+    # Pin the class name so this works with stdlib tomllib and Python 3.10 tomli.
+    _write_manifest(
+        tmp_path / "Cargo.toml",
+        """
+[package
+name = "broken"
+""",
+    )
+
+    with pytest.raises(Exception) as exc_info:
+        introspect_cargo(tmp_path)
+
+    assert exc_info.type.__name__ == "TOMLDecodeError"
+
+
+def test_cargo_introspect_degenerate_manifests_return_empty_or_skip_bad_deps(tmp_path):
+    """Degenerate but parseable manifests should not invent graph data or crash."""
+    # Empty and nameless packages prove crate nodes require package identity; the
+    # scalar dependencies case proves malformed dependency sections are ignored safely.
+    empty_manifest = tmp_path / "empty"
+    empty_manifest.mkdir()
+    _write_manifest(empty_manifest / "Cargo.toml", "")
+
+    empty_result = introspect_cargo(empty_manifest)
+
+    assert empty_result["nodes"] == []
+    assert empty_result["edges"] == []
+
+    nameless_package = tmp_path / "nameless"
+    nameless_package.mkdir()
+    _write_manifest(
+        nameless_package / "Cargo.toml",
+        """
+[package]
+version = "0.1.0"
+""",
+    )
+
+    nameless_result = introspect_cargo(nameless_package)
+
+    assert nameless_result["nodes"] == []
+    assert nameless_result["edges"] == []
+
+    scalar_dependencies = tmp_path / "scalar-dependencies"
+    scalar_dependencies.mkdir()
+    _write_manifest(
+        scalar_dependencies / "Cargo.toml",
+        """
+[package]
+name = "app"
+version = "0.1.0"
+
+dependencies = "not-a-table"
+""",
+    )
+
+    scalar_result = introspect_cargo(scalar_dependencies)
+
+    assert scalar_result["nodes"] == [
+        {
+            "id": "crate:app",
+            "label": "app",
+            "source_file": "Cargo.toml",
+            "source_location": "L1",
+        }
+    ]
+    assert scalar_result["edges"] == []
+
+
+def test_cargo_introspect_old_manifest_keeps_internal_path_dep_and_skips_external(tmp_path):
+    """Legacy manifests still resolve path deps and ignore bare-string externals."""
+    # Older Cargo files may omit modern metadata and use bare version strings; the
+    # graph should keep only workspace-internal relationships.
+    _write_manifest(
+        tmp_path / "Cargo.toml",
+        """
+[workspace]
+members = ["legacy", "internal"]
+""",
+    )
+    legacy = tmp_path / "legacy"
+    internal = tmp_path / "internal"
+    legacy.mkdir()
+    internal.mkdir()
+    _write_manifest(
+        legacy / "Cargo.toml",
+        """
+[package]
+name = "legacy"
+version = "0.1.0"
+
+[dependencies]
+rand = "0.8"
+internal = { path = "../internal" }
+""",
+    )
+    _write_manifest(
+        internal / "Cargo.toml",
+        """
+[package]
+name = "internal"
+version = "0.1.0"
+""",
+    )
+
+    result = introspect_cargo(tmp_path)
+
+    node_ids = {node["id"] for node in result["nodes"]}
+    edge_pairs = {(edge["source"], edge["target"]) for edge in result["edges"]}
+    assert node_ids == {"crate:legacy", "crate:internal"}
+    assert "crate:rand" not in node_ids
+    assert len(result["edges"]) == 1
+    assert ("crate:legacy", "crate:internal") in edge_pairs
+    assert ("crate:legacy", "crate:rand") not in edge_pairs
+
+
+def test_cargo_introspect_modern_virtual_and_root_package_workspaces(tmp_path):
+    """Modern workspace forms cover virtual roots, workspace deps, and root packages."""
+    # Virtual manifests and root-package workspaces discover members differently;
+    # both must produce exact internal graph shapes without registry-only edges.
+    virtual_root = tmp_path / "virtual"
+    virtual_root.mkdir()
+    _write_manifest(
+        virtual_root / "Cargo.toml",
+        """
+[workspace]
+members = ["crates/*"]
+
+[workspace.dependencies]
+beta = { path = "crates/beta" }
+serde = "1"
+""",
+    )
+    alpha = virtual_root / "crates" / "alpha"
+    beta = virtual_root / "crates" / "beta"
+    alpha.mkdir(parents=True)
+    beta.mkdir(parents=True)
+    _write_manifest(
+        alpha / "Cargo.toml",
+        """
+[package]
+name = "alpha"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+beta = { workspace = true }
+serde = { workspace = true }
+""",
+    )
+    _write_manifest(
+        beta / "Cargo.toml",
+        """
+[package]
+name = "beta"
+version = "0.1.0"
+edition = "2021"
+""",
+    )
+
+    virtual_result = introspect_cargo(virtual_root)
+
+    assert {node["id"] for node in virtual_result["nodes"]} == {
+        "crate:alpha",
+        "crate:beta",
+    }
+    assert len(virtual_result["nodes"]) == 2
+    assert len(virtual_result["edges"]) == 1
+    assert {
+        "source": "crate:alpha",
+        "target": "crate:beta",
+        "relation": "crate_depends_on",
+        "context": "cargo_dependency",
+        "weight": 1.0,
+        "confidence": "EXTRACTED",
+        "source_file": "crates/alpha/Cargo.toml",
+        "source_location": "L1",
+    } in virtual_result["edges"]
+
+    package_root = tmp_path / "package-root"
+    package_root.mkdir()
+    _write_manifest(
+        package_root / "Cargo.toml",
+        """
+[package]
+name = "root_pkg"
+version = "0.1.0"
+edition = "2021"
+
+[workspace]
+members = ["crates/*"]
+""",
+    )
+    member = package_root / "crates" / "member"
+    member.mkdir(parents=True)
+    _write_manifest(
+        member / "Cargo.toml",
+        """
+[package]
+name = "member"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+root_pkg = { path = "../.." }
+""",
+    )
+
+    package_result = introspect_cargo(package_root)
+
+    assert {node["id"] for node in package_result["nodes"]} == {
+        "crate:root_pkg",
+        "crate:member",
+    }
+    assert len(package_result["nodes"]) == 2
+    assert len(package_result["edges"]) == 1
+    assert {
+        "source": "crate:member",
+        "target": "crate:root_pkg",
+        "relation": "crate_depends_on",
+        "context": "cargo_dependency",
+        "weight": 1.0,
+        "confidence": "EXTRACTED",
+        "source_file": "crates/member/Cargo.toml",
+        "source_location": "L1",
+    } in package_result["edges"]
+
+
+def test_cargo_introspect_large_workspace_dependency_chain(tmp_path):
+    """Large deterministic workspace proves chain extraction scales by shape, not timing."""
+    # The exact 200-node/199-edge chain guards against truncation, glob misses, or
+    # accidental timing-based assertions that would make the test flaky.
+    crate_count = 200
+    _write_manifest(
+        tmp_path / "Cargo.toml",
+        """
+[workspace]
+members = ["crates/*"]
+""",
+    )
+
+    for index in range(crate_count):
+        crate_dir = tmp_path / "crates" / f"crate_{index:03d}"
+        crate_dir.mkdir(parents=True)
+        dependency_block = ""
+        if index + 1 < crate_count:
+            dependency_block = f"""
+
+[dependencies]
+crate_{index + 1:03d} = {{ path = "../crate_{index + 1:03d}" }}
+"""
+        _write_manifest(
+            crate_dir / "Cargo.toml",
+            f'''
+[package]
+name = "crate_{index:03d}"
+version = "0.1.0"
+edition = "2021"{dependency_block}
+''',
+        )
+
+    result = introspect_cargo(tmp_path)
+
+    assert len(result["nodes"]) == crate_count
+    assert len(result["edges"]) == crate_count - 1
+    assert {node["id"] for node in result["nodes"]} == {
+        f"crate:crate_{index:03d}" for index in range(crate_count)
+    }
+    assert {
+        "source": "crate:crate_000",
+        "target": "crate:crate_001",
+        "relation": "crate_depends_on",
+        "context": "cargo_dependency",
+        "weight": 1.0,
+        "confidence": "EXTRACTED",
+        "source_file": "crates/crate_000/Cargo.toml",
+        "source_location": "L1",
+    } in result["edges"]
+    assert {
+        "source": "crate:crate_198",
+        "target": "crate:crate_199",
+        "relation": "crate_depends_on",
+        "context": "cargo_dependency",
+        "weight": 1.0,
+        "confidence": "EXTRACTED",
+        "source_file": "crates/crate_198/Cargo.toml",
+        "source_location": "L1",
+    } in result["edges"]

tests/test_watch.py

@@ -424,9 +424,11 @@ def test_watch_handler_honors_graphifyignore(tmp_path, monkeypatch):
     import threading
     from graphify import watch as watch_mod
 
-    (tmp_path / ".graphifyignore").write_text("node_modules/\nbuild/\n", encoding="utf-8")
+    watch_root = tmp_path / ".hidden-parent" / "corpus"
-    (tmp_path / "node_modules").mkdir()
+    watch_root.mkdir(parents=True)
-    (tmp_path / "build").mkdir()
+    (watch_root / ".graphifyignore").write_text("node_modules/\nbuild/\n", encoding="utf-8")
+    (watch_root / "node_modules").mkdir()
+    (watch_root / "build").mkdir()
 
     rebuild_calls: list[Path] = []
     notify_calls: list[Path] = []
@@ -435,19 +437,24 @@ def test_watch_handler_honors_graphifyignore(tmp_path, monkeypatch):
 
     # Run watch() in a thread with a short debounce so we can verify the
     # post-debounce dispatch path actually runs on real events.
-    t = threading.Thread(target=watch_mod.watch, args=(tmp_path,), kwargs={"debounce": 0.2}, daemon=True)
+    t = threading.Thread(
+        target=watch_mod.watch,
+        args=(watch_root,),
+        kwargs={"debounce": 0.2},
+        daemon=True,
+    )
     t.start()
     time.sleep(0.5)  # let observer.start() settle
 
     # Ignored writes — handler must drop these.
-    (tmp_path / "node_modules" / "junk.js").write_text("// noise\n", encoding="utf-8")
+    (watch_root / "node_modules" / "junk.js").write_text("// noise\n", encoding="utf-8")
-    (tmp_path / "build" / "out.py").write_text("x = 1\n", encoding="utf-8")
+    (watch_root / "build" / "out.py").write_text("x = 1\n", encoding="utf-8")
     time.sleep(1.0)
     assert rebuild_calls == [], "ignored writes triggered a rebuild"
     assert notify_calls == [], "ignored writes triggered a notify"
 
     # Non-ignored write — handler must accept and (after debounce) dispatch.
-    (tmp_path / "app.py").write_text("def f():\n    return 1\n", encoding="utf-8")
+    (watch_root / "app.py").write_text("def f():\n    return 1\n", encoding="utf-8")
     deadline = time.monotonic() + 5.0
     while time.monotonic() < deadline and not rebuild_calls:
         time.sleep(0.1)