feat(extract): add Terraform/HCL AST extraction via tree-sitter-hcl (#1129)
Adds extract_terraform() for .tf/.tfvars/.hcl files. Nodes: resources, data sources, modules, variables, outputs, providers, locals. Edges: contains, references (interpolation), depends_on. Node IDs are directory-scoped for cross-file resolution. tree-sitter-hcl added as optional extra (graphify[terraform]) matching sql/dm precedent. 8 tests including .tfvars documented no-op behaviour. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Safi
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
fff1c980ac
commit
200edecea5
+1
-1
@@ -25,7 +25,7 @@ class FileType(str, Enum):
|
||||
|
||||
_MANIFEST_PATH = "graphify-out/manifest.json"
|
||||
|
||||
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml'}
|
||||
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml'}
|
||||
DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'}
|
||||
PAPER_EXTENSIONS = {'.pdf'}
|
||||
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
|
||||
|
||||
@@ -10526,6 +10526,184 @@ def extract_dmf(path: Path) -> dict:
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
# Head tokens in an HCL traversal that are meta/builtins, not references to a
|
||||
# block defined in the corpus (count.index, each.key, self.*, path.module, ...).
|
||||
_TF_META_HEADS = frozenset({"count", "each", "self", "path", "terraform"})
|
||||
|
||||
|
||||
def extract_terraform(path: Path) -> dict:
|
||||
"""Extract Terraform/HCL blocks and the references between them via tree-sitter.
|
||||
|
||||
Nodes: resources, data sources, modules, variables, outputs, providers, and
|
||||
locals. Edges: `contains` (file -> block), `references` (block -> the blocks
|
||||
it interpolates, e.g. `aws_instance.web` -> `var.region`), and `depends_on`
|
||||
(explicit dependency edges).
|
||||
|
||||
Node IDs are scoped by the parent directory, not the file stem, because
|
||||
Terraform resources are module(directory)-scoped: a resource defined in
|
||||
main.tf is referenced from other .tf files in the same directory. Directory
|
||||
scoping lets those cross-file references resolve when per-file extractions
|
||||
are merged (stem scoping would split a definition from its references).
|
||||
"""
|
||||
try:
|
||||
import tree_sitter_hcl as tshcl
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_hcl not installed. Run: pip install tree-sitter-hcl"}
|
||||
|
||||
try:
|
||||
language = Language(tshcl.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
str_path = str(path)
|
||||
file_nid = _make_id(str_path)
|
||||
scope = path.parent.name or "tf"
|
||||
|
||||
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str_path, "source_location": None}]
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = {file_nid}
|
||||
seen_edges: set[tuple[str, str, str]] = set()
|
||||
|
||||
def _read(n) -> str:
|
||||
return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
||||
|
||||
def _label_text(n) -> str:
|
||||
return _read(n).strip().strip('"')
|
||||
|
||||
def _add_node(address: str, label: str, line: int) -> str:
|
||||
nid = _make_id(scope, address)
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0})
|
||||
return nid
|
||||
|
||||
def _add_edge(src: str, address: str, relation: str, line: int) -> None:
|
||||
tgt = _make_id(scope, address)
|
||||
if src == tgt:
|
||||
return
|
||||
key = (src, tgt, relation)
|
||||
if key in seen_edges:
|
||||
return
|
||||
seen_edges.add(key)
|
||||
edges.append({"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0})
|
||||
|
||||
def _block_parts(block) -> tuple:
|
||||
btype = None
|
||||
labels: list[str] = []
|
||||
for c in block.children:
|
||||
if c.type in ("block_start", "body", "block_end"):
|
||||
break
|
||||
if c.type == "identifier" and btype is None:
|
||||
btype = _read(c)
|
||||
elif c.type in ("string_lit", "identifier"):
|
||||
labels.append(_label_text(c))
|
||||
return btype, labels
|
||||
|
||||
def _ref_address(expr):
|
||||
head = _read(expr)
|
||||
parent = expr.parent
|
||||
attrs: list[str] = []
|
||||
if parent is not None:
|
||||
seen_self = False
|
||||
for c in parent.children:
|
||||
if c.id == expr.id:
|
||||
seen_self = True
|
||||
continue
|
||||
if seen_self and c.type == "get_attr":
|
||||
name = None
|
||||
for gc in c.children:
|
||||
if gc.type == "identifier":
|
||||
name = _read(gc)
|
||||
break
|
||||
if name is None:
|
||||
break
|
||||
attrs.append(name)
|
||||
elif seen_self and c.type not in ("get_attr",):
|
||||
break
|
||||
if head in _TF_META_HEADS or not head:
|
||||
return None
|
||||
if head == "var":
|
||||
return f"var.{attrs[0]}" if attrs else None
|
||||
if head == "local":
|
||||
return f"local.{attrs[0]}" if attrs else None
|
||||
if head == "module":
|
||||
return f"module.{attrs[0]}" if attrs else None
|
||||
if head == "data":
|
||||
return f"data.{attrs[0]}.{attrs[1]}" if len(attrs) >= 2 else None
|
||||
return f"{head}.{attrs[0]}" if attrs else None
|
||||
|
||||
def _collect_refs(node, owner_nid: str, relation: str) -> None:
|
||||
rel = relation
|
||||
if node.type == "attribute":
|
||||
key_node = node.child_by_field_name("key") or (
|
||||
node.children[0] if node.children else None
|
||||
)
|
||||
if key_node is not None and _read(key_node) == "depends_on":
|
||||
rel = "depends_on"
|
||||
if node.type == "variable_expr":
|
||||
addr = _ref_address(node)
|
||||
if addr:
|
||||
_add_edge(owner_nid, addr, rel, node.start_point[0] + 1)
|
||||
for c in node.children:
|
||||
if c.is_named:
|
||||
_collect_refs(c, owner_nid, rel)
|
||||
|
||||
def _body_of(block):
|
||||
for c in block.children:
|
||||
if c.type == "body":
|
||||
return c
|
||||
return None
|
||||
|
||||
body = next((c for c in root.children if c.type == "body"), root)
|
||||
for block in body.children:
|
||||
if block.type != "block":
|
||||
continue
|
||||
btype, labels = _block_parts(block)
|
||||
line = block.start_point[0] + 1
|
||||
blk_body = _body_of(block)
|
||||
if btype == "resource" and len(labels) >= 2:
|
||||
owner = _add_node(f"{labels[0]}.{labels[1]}", f"{labels[0]}.{labels[1]}", line)
|
||||
elif btype == "data" and len(labels) >= 2:
|
||||
owner = _add_node(f"data.{labels[0]}.{labels[1]}", f"data.{labels[0]}.{labels[1]}", line)
|
||||
elif btype == "module" and labels:
|
||||
owner = _add_node(f"module.{labels[0]}", f"module.{labels[0]}", line)
|
||||
elif btype == "variable" and labels:
|
||||
owner = _add_node(f"var.{labels[0]}", f"var.{labels[0]}", line)
|
||||
elif btype == "output" and labels:
|
||||
owner = _add_node(f"output.{labels[0]}", f"output.{labels[0]}", line)
|
||||
elif btype == "provider" and labels:
|
||||
owner = _add_node(f"provider.{labels[0]}", f"provider.{labels[0]}", line)
|
||||
elif btype == "locals" and blk_body is not None:
|
||||
for attr in blk_body.children:
|
||||
if attr.type != "attribute":
|
||||
continue
|
||||
key_node = attr.children[0] if attr.children else None
|
||||
if key_node is None:
|
||||
continue
|
||||
key = _read(key_node)
|
||||
lnid = _add_node(f"local.{key}", f"local.{key}", attr.start_point[0] + 1)
|
||||
_collect_refs(attr, lnid, "references")
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
if blk_body is not None:
|
||||
_collect_refs(blk_body, owner, "references")
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
_DISPATCH: dict[str, Any] = {
|
||||
".py": extract_python,
|
||||
".js": extract_js,
|
||||
@@ -10594,6 +10772,9 @@ _DISPATCH: dict[str, Any] = {
|
||||
".sh": extract_bash,
|
||||
".bash": extract_bash,
|
||||
".json": extract_json,
|
||||
".tf": extract_terraform,
|
||||
".tfvars": extract_terraform,
|
||||
".hcl": extract_terraform,
|
||||
".dm": extract_dm,
|
||||
".dme": extract_dm,
|
||||
".dmi": extract_dmi,
|
||||
|
||||
+3
-1
@@ -69,7 +69,8 @@ sql = ["tree-sitter-sql"]
|
||||
# must compile from source (needs a C toolchain + python3-dev). Keeping it optional
|
||||
# avoids breaking the default `uv tool install graphifyy` for everyone (#1104).
|
||||
dm = ["tree-sitter-dm"]
|
||||
all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp", "matplotlib", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm"]
|
||||
terraform = ["tree-sitter-hcl"]
|
||||
all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp", "matplotlib", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl"]
|
||||
|
||||
[project.scripts]
|
||||
graphify = "graphify.__main__:main"
|
||||
@@ -91,6 +92,7 @@ dev = [
|
||||
"setuptools>=82.0.1",
|
||||
"wheel>=0.47.0",
|
||||
"tomli>=2.0 ; python_version < '3.11'",
|
||||
"tree-sitter-hcl>=1.2.0",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Tests for the Terraform/HCL extractor (graphify/extract.py, issue #187)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.build import build_from_json
|
||||
from graphify.extract import extract_terraform
|
||||
|
||||
|
||||
def _write(tmp_path: Path, name: str, body: str) -> Path:
|
||||
p = tmp_path / name
|
||||
p.write_text(body, encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def _labels(r) -> list[str]:
|
||||
return [n["label"] for n in r["nodes"]]
|
||||
|
||||
|
||||
def _rel_pairs(r, relation: str) -> set[tuple[str, str]]:
|
||||
lab = {n["id"]: n["label"] for n in r["nodes"]}
|
||||
return {
|
||||
(lab.get(e["source"], e["source"]), lab.get(e["target"], e["target"]))
|
||||
for e in r["edges"]
|
||||
if e["relation"] == relation
|
||||
}
|
||||
|
||||
|
||||
SAMPLE = """\
|
||||
# leading comment so the body is not children[0]
|
||||
terraform {
|
||||
required_providers { azurerm = { source = "hashicorp/azurerm" } }
|
||||
}
|
||||
|
||||
variable "region" { default = "us-east-1" }
|
||||
|
||||
provider "aws" { region = var.region }
|
||||
|
||||
data "aws_ami" "ubuntu" { most_recent = true }
|
||||
|
||||
resource "aws_instance" "web" {
|
||||
ami = data.aws_ami.ubuntu.id
|
||||
subnet_id = var.region
|
||||
depends_on = [aws_security_group.sg]
|
||||
}
|
||||
|
||||
resource "aws_security_group" "sg" { name = "sg" }
|
||||
|
||||
module "vpc" {
|
||||
source = "./modules/vpc"
|
||||
cidr = local.cidr
|
||||
}
|
||||
|
||||
locals { cidr = "10.0.0.0/16" }
|
||||
|
||||
output "ip" { value = aws_instance.web.private_ip }
|
||||
"""
|
||||
|
||||
|
||||
def test_no_error_and_all_block_types_become_nodes(tmp_path):
|
||||
r = extract_terraform(_write(tmp_path, "main.tf", SAMPLE))
|
||||
assert r.get("error") is None
|
||||
labels = set(_labels(r))
|
||||
# one node per block type (the terraform{} settings block is intentionally skipped)
|
||||
for expected in (
|
||||
"var.region",
|
||||
"provider.aws",
|
||||
"data.aws_ami.ubuntu",
|
||||
"aws_instance.web",
|
||||
"aws_security_group.sg",
|
||||
"module.vpc",
|
||||
"local.cidr",
|
||||
"output.ip",
|
||||
):
|
||||
assert expected in labels, f"missing node {expected!r}"
|
||||
|
||||
|
||||
def test_reference_edges(tmp_path):
|
||||
r = extract_terraform(_write(tmp_path, "main.tf", SAMPLE))
|
||||
refs = _rel_pairs(r, "references")
|
||||
assert ("provider.aws", "var.region") in refs
|
||||
assert ("aws_instance.web", "data.aws_ami.ubuntu") in refs
|
||||
assert ("aws_instance.web", "var.region") in refs
|
||||
assert ("module.vpc", "local.cidr") in refs
|
||||
assert ("output.ip", "aws_instance.web") in refs
|
||||
|
||||
|
||||
def test_depends_on_edge(tmp_path):
|
||||
r = extract_terraform(_write(tmp_path, "main.tf", SAMPLE))
|
||||
assert ("aws_instance.web", "aws_security_group.sg") in _rel_pairs(r, "depends_on")
|
||||
|
||||
|
||||
def test_file_contains_blocks(tmp_path):
|
||||
r = extract_terraform(_write(tmp_path, "main.tf", SAMPLE))
|
||||
contains = _rel_pairs(r, "contains")
|
||||
assert ("main.tf", "aws_instance.web") in contains
|
||||
assert ("main.tf", "var.region") in contains
|
||||
|
||||
|
||||
def test_meta_heads_not_emitted(tmp_path):
|
||||
# count.index / each.key / self.* / path.module are builtins, not references.
|
||||
body = """\
|
||||
resource "aws_instance" "web" {
|
||||
count = 2
|
||||
name = "web-${count.index}"
|
||||
tags = each.value
|
||||
dir = path.module
|
||||
}
|
||||
"""
|
||||
r = extract_terraform(_write(tmp_path, "main.tf", body))
|
||||
targets = {t for _, t in _rel_pairs(r, "references")}
|
||||
assert not any(t.startswith(("count", "each", "path")) for t in targets)
|
||||
|
||||
|
||||
def test_cross_file_references_resolve_after_merge(tmp_path):
|
||||
# A resource defined in one file is referenced from another in the same
|
||||
# directory; directory-scoped IDs must let the edge resolve at build time.
|
||||
defn = """\
|
||||
resource "azurerm_resource_group" "main" { name = "rg" }
|
||||
"""
|
||||
user = """\
|
||||
resource "azurerm_network_interface" "nic" {
|
||||
resource_group_name = azurerm_resource_group.main.name
|
||||
}
|
||||
"""
|
||||
r_defn = extract_terraform(_write(tmp_path, "main.tf", defn))
|
||||
r_user = extract_terraform(_write(tmp_path, "nic.tf", user))
|
||||
|
||||
# The cross-file edge target id equals the definition's node id.
|
||||
rg_id = next(n["id"] for n in r_defn["nodes"] if n["label"] == "azurerm_resource_group.main")
|
||||
nic_ref_targets = {e["target"] for e in r_user["edges"] if e["relation"] == "references"}
|
||||
assert rg_id in nic_ref_targets
|
||||
|
||||
# And it survives a real merge: the edge is present (not dropped as dangling).
|
||||
G = build_from_json(
|
||||
{
|
||||
"nodes": r_defn["nodes"] + r_user["nodes"],
|
||||
"edges": r_defn["edges"] + r_user["edges"],
|
||||
}
|
||||
)
|
||||
nic_id = next(n["id"] for n in r_user["nodes"] if n["label"] == "azurerm_network_interface.nic")
|
||||
assert G.has_edge(nic_id, rg_id)
|
||||
|
||||
|
||||
def test_empty_and_commentonly_files_are_safe(tmp_path):
|
||||
assert extract_terraform(_write(tmp_path, "a.tf", "")).get("error") is None
|
||||
r = extract_terraform(_write(tmp_path, "b.tf", "# just a comment\n"))
|
||||
# only the file node, no crash
|
||||
assert len(r["nodes"]) == 1
|
||||
|
||||
|
||||
def test_tfvars_key_value_is_safe(tmp_path):
|
||||
# .tfvars files contain only key=value assignments (no block structure),
|
||||
# so extract_terraform produces zero block nodes — only the file node.
|
||||
# This is the documented intended behaviour for .tfvars.
|
||||
r = extract_terraform(_write(tmp_path, "terraform.tfvars", 'region = "us-east-1"\nenv = "prod"\n'))
|
||||
assert r.get("error") is None
|
||||
assert len(r["nodes"]) == 1 # only the file node, no variable nodes
|
||||
Reference in New Issue
Block a user