From 200edecea54098133f69c7eb9ec23e3ba71c0ee6 Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 5 Jun 2026 21:38:56 +0100 Subject: [PATCH] feat(extract): add Terraform/HCL AST extraction via tree-sitter-hcl (#1129) Adds extract_terraform() for .tf/.tfvars/.hcl files. Nodes: resources, data sources, modules, variables, outputs, providers, locals. Edges: contains, references (interpolation), depends_on. Node IDs are directory-scoped for cross-file resolution. tree-sitter-hcl added as optional extra (graphify[terraform]) matching sql/dm precedent. 8 tests including .tfvars documented no-op behaviour. Co-Authored-By: Claude Sonnet 4.6 --- graphify/detect.py | 2 +- graphify/extract.py | 181 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 4 +- tests/test_terraform.py | 158 +++++++++++++++++++++++++++++++++++ 4 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 tests/test_terraform.py diff --git a/graphify/detect.py b/graphify/detect.py index 67dcf6e..e15a413 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -25,7 +25,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index a56a780..6ec23a0 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -10526,6 +10526,184 @@ def extract_dmf(path: Path) -> dict: return {"nodes": nodes, "edges": edges} +# Head tokens in an HCL traversal that are meta/builtins, not references to a +# block defined in the corpus (count.index, each.key, self.*, path.module, ...). +_TF_META_HEADS = frozenset({"count", "each", "self", "path", "terraform"}) + + +def extract_terraform(path: Path) -> dict: + """Extract Terraform/HCL blocks and the references between them via tree-sitter. + + Nodes: resources, data sources, modules, variables, outputs, providers, and + locals. Edges: `contains` (file -> block), `references` (block -> the blocks + it interpolates, e.g. `aws_instance.web` -> `var.region`), and `depends_on` + (explicit dependency edges). + + Node IDs are scoped by the parent directory, not the file stem, because + Terraform resources are module(directory)-scoped: a resource defined in + main.tf is referenced from other .tf files in the same directory. Directory + scoping lets those cross-file references resolve when per-file extractions + are merged (stem scoping would split a definition from its references). + """ + try: + import tree_sitter_hcl as tshcl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree_sitter_hcl not installed. Run: pip install tree-sitter-hcl"} + + try: + language = Language(tshcl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + str_path = str(path) + file_nid = _make_id(str_path) + scope = path.parent.name or "tf" + + nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code", + "source_file": str_path, "source_location": None}] + edges: list[dict] = [] + seen_ids: set[str] = {file_nid} + seen_edges: set[tuple[str, str, str]] = set() + + def _read(n) -> str: + return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") + + def _label_text(n) -> str: + return _read(n).strip().strip('"') + + def _add_node(address: str, label: str, line: int) -> str: + nid = _make_id(scope, address) + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + edges.append({"source": file_nid, "target": nid, "relation": "contains", + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0}) + return nid + + def _add_edge(src: str, address: str, relation: str, line: int) -> None: + tgt = _make_id(scope, address) + if src == tgt: + return + key = (src, tgt, relation) + if key in seen_edges: + return + seen_edges.add(key) + edges.append({"source": src, "target": tgt, "relation": relation, + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0}) + + def _block_parts(block) -> tuple: + btype = None + labels: list[str] = [] + for c in block.children: + if c.type in ("block_start", "body", "block_end"): + break + if c.type == "identifier" and btype is None: + btype = _read(c) + elif c.type in ("string_lit", "identifier"): + labels.append(_label_text(c)) + return btype, labels + + def _ref_address(expr): + head = _read(expr) + parent = expr.parent + attrs: list[str] = [] + if parent is not None: + seen_self = False + for c in parent.children: + if c.id == expr.id: + seen_self = True + continue + if seen_self and c.type == "get_attr": + name = None + for gc in c.children: + if gc.type == "identifier": + name = _read(gc) + break + if name is None: + break + attrs.append(name) + elif seen_self and c.type not in ("get_attr",): + break + if head in _TF_META_HEADS or not head: + return None + if head == "var": + return f"var.{attrs[0]}" if attrs else None + if head == "local": + return f"local.{attrs[0]}" if attrs else None + if head == "module": + return f"module.{attrs[0]}" if attrs else None + if head == "data": + return f"data.{attrs[0]}.{attrs[1]}" if len(attrs) >= 2 else None + return f"{head}.{attrs[0]}" if attrs else None + + def _collect_refs(node, owner_nid: str, relation: str) -> None: + rel = relation + if node.type == "attribute": + key_node = node.child_by_field_name("key") or ( + node.children[0] if node.children else None + ) + if key_node is not None and _read(key_node) == "depends_on": + rel = "depends_on" + if node.type == "variable_expr": + addr = _ref_address(node) + if addr: + _add_edge(owner_nid, addr, rel, node.start_point[0] + 1) + for c in node.children: + if c.is_named: + _collect_refs(c, owner_nid, rel) + + def _body_of(block): + for c in block.children: + if c.type == "body": + return c + return None + + body = next((c for c in root.children if c.type == "body"), root) + for block in body.children: + if block.type != "block": + continue + btype, labels = _block_parts(block) + line = block.start_point[0] + 1 + blk_body = _body_of(block) + if btype == "resource" and len(labels) >= 2: + owner = _add_node(f"{labels[0]}.{labels[1]}", f"{labels[0]}.{labels[1]}", line) + elif btype == "data" and len(labels) >= 2: + owner = _add_node(f"data.{labels[0]}.{labels[1]}", f"data.{labels[0]}.{labels[1]}", line) + elif btype == "module" and labels: + owner = _add_node(f"module.{labels[0]}", f"module.{labels[0]}", line) + elif btype == "variable" and labels: + owner = _add_node(f"var.{labels[0]}", f"var.{labels[0]}", line) + elif btype == "output" and labels: + owner = _add_node(f"output.{labels[0]}", f"output.{labels[0]}", line) + elif btype == "provider" and labels: + owner = _add_node(f"provider.{labels[0]}", f"provider.{labels[0]}", line) + elif btype == "locals" and blk_body is not None: + for attr in blk_body.children: + if attr.type != "attribute": + continue + key_node = attr.children[0] if attr.children else None + if key_node is None: + continue + key = _read(key_node) + lnid = _add_node(f"local.{key}", f"local.{key}", attr.start_point[0] + 1) + _collect_refs(attr, lnid, "references") + continue + else: + continue + if blk_body is not None: + _collect_refs(blk_body, owner, "references") + + return {"nodes": nodes, "edges": edges} + + _DISPATCH: dict[str, Any] = { ".py": extract_python, ".js": extract_js, @@ -10594,6 +10772,9 @@ _DISPATCH: dict[str, Any] = { ".sh": extract_bash, ".bash": extract_bash, ".json": extract_json, + ".tf": extract_terraform, + ".tfvars": extract_terraform, + ".hcl": extract_terraform, ".dm": extract_dm, ".dme": extract_dm, ".dmi": extract_dmi, diff --git a/pyproject.toml b/pyproject.toml index ca32346..462c908 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,8 @@ sql = ["tree-sitter-sql"] # must compile from source (needs a C toolchain + python3-dev). Keeping it optional # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] -all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp", "matplotlib", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm"] +terraform = ["tree-sitter-hcl"] +all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp", "matplotlib", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl"] [project.scripts] graphify = "graphify.__main__:main" @@ -91,6 +92,7 @@ dev = [ "setuptools>=82.0.1", "wheel>=0.47.0", "tomli>=2.0 ; python_version < '3.11'", + "tree-sitter-hcl>=1.2.0", ] [tool.uv] diff --git a/tests/test_terraform.py b/tests/test_terraform.py new file mode 100644 index 0000000..2049078 --- /dev/null +++ b/tests/test_terraform.py @@ -0,0 +1,158 @@ +"""Tests for the Terraform/HCL extractor (graphify/extract.py, issue #187).""" +from __future__ import annotations + +from pathlib import Path + +from graphify.build import build_from_json +from graphify.extract import extract_terraform + + +def _write(tmp_path: Path, name: str, body: str) -> Path: + p = tmp_path / name + p.write_text(body, encoding="utf-8") + return p + + +def _labels(r) -> list[str]: + return [n["label"] for n in r["nodes"]] + + +def _rel_pairs(r, relation: str) -> set[tuple[str, str]]: + lab = {n["id"]: n["label"] for n in r["nodes"]} + return { + (lab.get(e["source"], e["source"]), lab.get(e["target"], e["target"])) + for e in r["edges"] + if e["relation"] == relation + } + + +SAMPLE = """\ +# leading comment so the body is not children[0] +terraform { + required_providers { azurerm = { source = "hashicorp/azurerm" } } +} + +variable "region" { default = "us-east-1" } + +provider "aws" { region = var.region } + +data "aws_ami" "ubuntu" { most_recent = true } + +resource "aws_instance" "web" { + ami = data.aws_ami.ubuntu.id + subnet_id = var.region + depends_on = [aws_security_group.sg] +} + +resource "aws_security_group" "sg" { name = "sg" } + +module "vpc" { + source = "./modules/vpc" + cidr = local.cidr +} + +locals { cidr = "10.0.0.0/16" } + +output "ip" { value = aws_instance.web.private_ip } +""" + + +def test_no_error_and_all_block_types_become_nodes(tmp_path): + r = extract_terraform(_write(tmp_path, "main.tf", SAMPLE)) + assert r.get("error") is None + labels = set(_labels(r)) + # one node per block type (the terraform{} settings block is intentionally skipped) + for expected in ( + "var.region", + "provider.aws", + "data.aws_ami.ubuntu", + "aws_instance.web", + "aws_security_group.sg", + "module.vpc", + "local.cidr", + "output.ip", + ): + assert expected in labels, f"missing node {expected!r}" + + +def test_reference_edges(tmp_path): + r = extract_terraform(_write(tmp_path, "main.tf", SAMPLE)) + refs = _rel_pairs(r, "references") + assert ("provider.aws", "var.region") in refs + assert ("aws_instance.web", "data.aws_ami.ubuntu") in refs + assert ("aws_instance.web", "var.region") in refs + assert ("module.vpc", "local.cidr") in refs + assert ("output.ip", "aws_instance.web") in refs + + +def test_depends_on_edge(tmp_path): + r = extract_terraform(_write(tmp_path, "main.tf", SAMPLE)) + assert ("aws_instance.web", "aws_security_group.sg") in _rel_pairs(r, "depends_on") + + +def test_file_contains_blocks(tmp_path): + r = extract_terraform(_write(tmp_path, "main.tf", SAMPLE)) + contains = _rel_pairs(r, "contains") + assert ("main.tf", "aws_instance.web") in contains + assert ("main.tf", "var.region") in contains + + +def test_meta_heads_not_emitted(tmp_path): + # count.index / each.key / self.* / path.module are builtins, not references. + body = """\ +resource "aws_instance" "web" { + count = 2 + name = "web-${count.index}" + tags = each.value + dir = path.module +} +""" + r = extract_terraform(_write(tmp_path, "main.tf", body)) + targets = {t for _, t in _rel_pairs(r, "references")} + assert not any(t.startswith(("count", "each", "path")) for t in targets) + + +def test_cross_file_references_resolve_after_merge(tmp_path): + # A resource defined in one file is referenced from another in the same + # directory; directory-scoped IDs must let the edge resolve at build time. + defn = """\ +resource "azurerm_resource_group" "main" { name = "rg" } +""" + user = """\ +resource "azurerm_network_interface" "nic" { + resource_group_name = azurerm_resource_group.main.name +} +""" + r_defn = extract_terraform(_write(tmp_path, "main.tf", defn)) + r_user = extract_terraform(_write(tmp_path, "nic.tf", user)) + + # The cross-file edge target id equals the definition's node id. + rg_id = next(n["id"] for n in r_defn["nodes"] if n["label"] == "azurerm_resource_group.main") + nic_ref_targets = {e["target"] for e in r_user["edges"] if e["relation"] == "references"} + assert rg_id in nic_ref_targets + + # And it survives a real merge: the edge is present (not dropped as dangling). + G = build_from_json( + { + "nodes": r_defn["nodes"] + r_user["nodes"], + "edges": r_defn["edges"] + r_user["edges"], + } + ) + nic_id = next(n["id"] for n in r_user["nodes"] if n["label"] == "azurerm_network_interface.nic") + assert G.has_edge(nic_id, rg_id) + + +def test_empty_and_commentonly_files_are_safe(tmp_path): + assert extract_terraform(_write(tmp_path, "a.tf", "")).get("error") is None + r = extract_terraform(_write(tmp_path, "b.tf", "# just a comment\n")) + # only the file node, no crash + assert len(r["nodes"]) == 1 + + +def test_tfvars_key_value_is_safe(tmp_path): + # .tfvars files contain only key=value assignments (no block structure), + # so extract_terraform produces zero block nodes — only the file node. + # This is the documented intended behaviour for .tfvars. + r = extract_terraform(_write(tmp_path, "terraform.tfvars", 'region = "us-east-1"\nenv = "prod"\n')) + assert r.get("error") is None + assert len(r["nodes"]) == 1 # only the file node, no variable nodes