feat(commonlisp): add Common Lisp .lisp/.cl/.lsp/.asd extractor (optional tree-sitter-commonlisp extra)

tree-sitter-commonlisp-backed extractor for packages, classes, functions,
methods, generics, macros, variable definers, and same-file calls. Handles the
grammar's dedicated defun node (defun/defmacro/defmethod/defgeneric via
defun_header) and the generic list_lit + leading-symbol forms (defvar/defclass/
defpackage), recurses into wrapper macros (eval-when/progn) and reader
conditionals, and maps CL operator chars (= < > ? ! + *) to readable id
suffixes. Wired into detect/extract dispatch, the extractor registry, a
[commonlisp] optional extra, and the README; ships a fixture and a 23-test suite
behind an importorskip guard.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
fade
2026-08-17 16:05:33 +01:00
committed by safishamsi
co-authored by Claude Opus 4.8
parent c44a6cf7f1
commit 038ada6f0f
9 changed files with 845 additions and 4 deletions
+2
View File
@@ -268,6 +268,7 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi
| `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` |
| `pascal` | Pascal / Delphi `.pas`/`.dpr`/`.dpk`/`.inc` AST extraction (more accurate `calls`/`inherits` edges; falls back to a regex extractor when absent) | `uv tool install "graphifyy[pascal]"` |
| `ocaml` | OCaml `.ml`/`.mli` AST extraction | `uv tool install "graphifyy[ocaml]"` |
| `commonlisp` | Common Lisp `.lisp`/`.cl`/`.lsp`/`.asd` AST extraction | `uv tool install "graphifyy[commonlisp]"` |
| `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` |
| `all` | Everything above | `uv tool install "graphifyy[all]"` |
@@ -341,6 +342,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg
| Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) |
| Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) |
| OCaml | `.ml .mli` (requires `uv tool install graphifyy[ocaml]`) |
| Common Lisp | `.lisp .cl .lsp .asd` (requires `uv tool install graphifyy[commonlisp]`) |
| MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements |
| Package manifests | `apm.yml` `pyproject.toml` `go.mod` `pom.xml` — one canonical package node per package (by name) plus `depends_on` edges, so a package referenced from many manifests is a single hub |
| Docs | `.md .mdx .qmd .html .txt .rst .yaml .yml` (markdown `[text](./other.md)` links and `[[wikilinks]]` become `references` edges between docs) |
+1 -1
View File
@@ -42,7 +42,7 @@ _MANIFEST_PATH = str(out_path("manifest.json"))
_MTIME_COARSE_S = 2.0
_MTIME_SUBSECOND_S = 0.05
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'}
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd'}
DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'}
PAPER_EXTENSIONS = {'.pdf'}
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
+9
View File
@@ -45,6 +45,7 @@ from graphify.extractors.elixir import extract_elixir # noqa: F401
from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401
from graphify.extractors.go import _GO_PREDECLARED_FUNCS, extract_go # noqa: F401
from graphify.extractors.json_config import extract_json # noqa: F401
from graphify.extractors.commonlisp import extract_commonlisp # noqa: F401
from graphify.extractors.markdown import extract_markdown # noqa: F401
from graphify.extractors.ocaml import extract_ocaml # noqa: F401
from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401
@@ -4924,6 +4925,10 @@ _DISPATCH: dict[str, Any] = {
".dart": extract_dart,
".ml": extract_ocaml,
".mli": extract_ocaml,
".lisp": extract_commonlisp,
".cl": extract_commonlisp,
".lsp": extract_commonlisp,
".asd": extract_commonlisp,
".v": extract_verilog,
".sv": extract_verilog,
".svh": extract_verilog,
@@ -4978,6 +4983,10 @@ _EXTRA_FOR_EXTENSION = {
".dme": "dm",
".ml": "ocaml",
".mli": "ocaml",
".lisp": "commonlisp",
".cl": "commonlisp",
".lsp": "commonlisp",
".asd": "commonlisp",
}
# Substrings an extractor's error carries to classify why a dependency-backed
+2
View File
@@ -13,6 +13,7 @@ from typing import Callable
from graphify.extractors.apex import extract_apex
from graphify.extractors.bash import extract_bash
from graphify.extractors.blade import extract_blade
from graphify.extractors.commonlisp import extract_commonlisp
from graphify.extractors.dart import extract_dart
from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm
from graphify.extractors.elixir import extract_elixir
@@ -37,6 +38,7 @@ LANGUAGE_EXTRACTORS: dict[str, Callable[[Path], dict]] = {
"apex": extract_apex,
"bash": extract_bash,
"blade": extract_blade,
"commonlisp": extract_commonlisp,
"dart": extract_dart,
"delphi_form": extract_delphi_form,
"dm": extract_dm,
+492
View File
@@ -0,0 +1,492 @@
"""Common Lisp extractor for .lisp/.cl/.lsp/.asd, backed by tree-sitter-commonlisp."""
from __future__ import annotations
import warnings
from pathlib import Path
from graphify.extractors.base import _make_id
# Standard CL definer forms that introduce data/type/variable bindings
# (not callable, no body to walk for calls)
_CL_DATA_DEFINERS = frozenset({
"defstruct", "deftype", "define-condition",
"defvar", "defparameter", "defconstant",
"define-symbol-macro",
})
# Subset of data definers that take a VALUE as their second argument, which
# may itself be a string literal. For these, we must not mistake the value
# for a docstring.
_CL_VALUE_DEFINERS = frozenset({"defvar", "defparameter", "defconstant"})
# Standard CL macro-style definers
_CL_MACRO_DEFINERS = frozenset({
"define-modify-macro", "define-compiler-macro",
"define-setf-expander", "defsetf",
})
# Symbols that start with "def" but are NOT definitions (denylist for the
# def-prefix heuristic that catches custom definers like definline-maybe)
_CL_NOT_DEFINERS = frozenset({
"default", "default-value", "defaults", "define",
"defer", "deferred", "deflate", "deftest",
})
# CL special forms / macros that should not count as "calls" in the graph
_CL_SPECIAL_FORMS = frozenset({
"let", "let*", "flet", "labels", "macrolet",
"if", "when", "unless", "cond", "case", "ecase", "typecase", "etypecase",
"progn", "prog1", "prog2", "block", "return-from", "tagbody", "go",
"lambda", "function", "funcall", "apply",
"setf", "setq", "psetf", "psetq", "incf", "decf", "push", "pop",
"loop", "do", "do*", "dolist", "dotimes", "map", "mapcar", "mapc",
"format", "print", "princ", "prin1", "write", "terpri",
"and", "or", "not", "null",
"values", "multiple-value-bind", "multiple-value-setq",
"declare", "the", "locally", "eval-when",
"quote", "backquote",
"error", "signal", "warn", "assert", "check-type",
"handler-case", "handler-bind", "restart-case", "ignore-errors",
"unwind-protect", "catch", "throw",
"with-open-file", "with-output-to-string", "with-input-from-string",
"with-slots", "with-accessors",
"defvar", "defparameter", "defconstant",
"in-package", "require", "provide", "use-package",
"t", "nil",
})
def extract_commonlisp(path: Path) -> dict:
"""Extract packages, classes, functions, methods, macros, and calls from a Common Lisp file."""
try:
import tree_sitter_commonlisp as tscl
from tree_sitter import Language, Parser
except ImportError:
return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"}
try:
# tree-sitter-commonlisp 0.4.1 (latest) ships the old binding that returns
# an int pointer from language(); tree_sitter.Language only accepts that
# int via a deprecated path. Silence that one warning at the call site
# until the grammar ships a PyCapsule binding.
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message="int argument support is deprecated",
category=DeprecationWarning,
)
language = Language(tscl.language())
parser = Parser(language)
source = path.read_bytes()
tree = parser.parse(source)
root = tree.root_node
except Exception as e:
return {"nodes": [], "edges": [], "error": str(e)}
stem = path.stem
str_path = str(path)
nodes: list[dict] = []
edges: list[dict] = []
seen_ids: set[str] = set()
function_bodies: list[tuple[str, object]] = [] # (nid, body_nodes)
current_package: str | None = None
def _text(node) -> str:
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
# CL symbols can contain operator chars like = < > ? ! + * / that the
# generic _make_id strips. Map them to readable suffixes so upi=, upi<,
# upi> become distinct ids.
_CL_CHAR_MAP = {
'=': '_eq', '<': '_lt', '>': '_gt', '?': '_p', '!': '_bang',
'+': '_plus', '*': '_star', '/': '_slash', '%': '_pct', '&': '_amp',
}
def _cl_id(*parts: str) -> str:
normalized = []
for p in parts:
normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p))
return _make_id(*normalized)
def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None:
if nid not in seen_ids:
seen_ids.add(nid)
nodes.append({
"id": nid,
"label": label,
"file_type": file_type,
"source_file": str_path,
"source_location": f"L{line}",
})
def add_edge(src: str, tgt: str, relation: str, line: int,
confidence: str = "EXTRACTED", weight: float = 1.0) -> None:
edges.append({
"source": src,
"target": tgt,
"relation": relation,
"confidence": confidence,
"source_file": str_path,
"source_location": f"L{line}",
"weight": weight,
})
file_nid = _cl_id(stem)
add_node(file_nid, path.name, 1)
def _first_sym(node) -> str | None:
"""Get the first sym_lit text from a list_lit's children."""
for child in node.children:
if child.type == "sym_lit":
return _text(child)
return None
def _kwd_text(node) -> str:
"""Extract the symbol name from a kwd_lit, stripping the leading colon."""
t = _text(node)
return t.lstrip(":").lstrip("#:")
def _handle_defpackage(node) -> None:
nonlocal current_package
children = node.children
pkg_name = None
for child in children:
if child.type == "kwd_lit" and pkg_name is None:
pkg_name = _kwd_text(child)
break
if child.type == "sym_lit" and _text(child) != "defpackage":
pkg_name = _text(child)
break
if not pkg_name:
return
current_package = pkg_name
pkg_nid = _cl_id(stem, pkg_name)
line = node.start_point[0] + 1
add_node(pkg_nid, pkg_name, line)
add_edge(file_nid, pkg_nid, "contains", line)
# Extract :use clauses as imports
for child in children:
if child.type == "list_lit":
first = _first_sym(child)
if first is None:
# Could be (:use ...) with kwd_lit
for gc in child.children:
if gc.type == "kwd_lit" and _kwd_text(gc) == "use":
for uc in child.children:
if uc.type == "kwd_lit" and uc != gc:
mod_name = _kwd_text(uc)
if mod_name != "use":
tgt_nid = _cl_id(mod_name)
add_edge(pkg_nid, tgt_nid, "imports",
child.start_point[0] + 1)
break
def _handle_defclass(node) -> None:
children = node.children
# Find class name: second sym_lit after "defclass"
sym_lits = [c for c in children if c.type == "sym_lit"]
if len(sym_lits) < 2:
return
class_name = _text(sym_lits[1])
line = node.start_point[0] + 1
class_nid = _cl_id(stem, class_name)
add_node(class_nid, class_name, line)
parent_nid = file_nid
if current_package:
pkg_nid = _cl_id(stem, current_package)
if pkg_nid in seen_ids:
parent_nid = pkg_nid
add_edge(parent_nid, class_nid, "contains", line)
# Superclasses (third element, a list)
list_lits = [c for c in children if c.type == "list_lit"]
if list_lits:
superclass_list = list_lits[0]
for sc in superclass_list.children:
if sc.type == "sym_lit":
sc_name = _text(sc)
sc_nid = _cl_id(stem, sc_name)
add_edge(class_nid, sc_nid, "inherits",
superclass_list.start_point[0] + 1)
def _handle_defun_node(node) -> None:
"""Handle a defun AST node (covers defun, defmethod, defgeneric, defmacro)."""
header = None
for child in node.children:
if child.type == "defun_header":
header = child
break
if not header:
return
# Determine keyword type
keyword_type = "defun"
func_name = None
for child in header.children:
if child.type == "defun_keyword":
for kc in child.children:
if kc.type in ("defun", "defmethod", "defgeneric", "defmacro"):
keyword_type = kc.type
break
if child.type == "sym_lit" and func_name is None:
func_name = _text(child)
if not func_name:
return
line = node.start_point[0] + 1
func_nid = _cl_id(stem, func_name)
if keyword_type == "defmethod":
label = f".{func_name}()"
elif keyword_type == "defmacro":
label = f"{func_name} (macro)"
elif keyword_type == "defgeneric":
label = f"{func_name} (generic)"
else:
label = f"{func_name}()"
add_node(func_nid, label, line)
parent_nid = file_nid
if current_package:
pkg_nid = _cl_id(stem, current_package)
if pkg_nid in seen_ids:
parent_nid = pkg_nid
if keyword_type == "defmethod":
add_edge(parent_nid, func_nid, "method", line)
else:
add_edge(parent_nid, func_nid, "contains", line)
# Method specializers: (defmethod name ((param class) ...))
if keyword_type == "defmethod":
for child in header.children:
if child.type == "list_lit":
# This is the parameter list
for param in child.children:
if param.type == "list_lit":
syms = [c for c in param.children if c.type == "sym_lit"]
if len(syms) >= 2:
specializer_name = _text(syms[1])
spec_nid = _cl_id(stem, specializer_name)
add_edge(func_nid, spec_nid, "specializes",
param.start_point[0] + 1)
break
# Docstring
for child in node.children:
if child.type == "str_lit":
doc_text = _text(child).strip('"')
if doc_text:
doc_nid = _cl_id(func_nid, "rationale")
add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale")
add_edge(doc_nid, func_nid, "rationale_for",
child.start_point[0] + 1)
break
# Collect body for call extraction
body_nodes = [c for c in node.children
if c.type not in ("(", ")", "defun_header", "str_lit")]
if body_nodes:
function_bodies.append((func_nid, body_nodes))
def _extract_def_name(node, def_keyword: str) -> str | None:
"""Get the name being defined. May be a sym_lit or a list_lit whose
first sym_lit is the name (e.g. (defstruct (foo :conc-name "BAR-") ...))."""
seen_keyword = False
for child in node.children:
if not seen_keyword:
if child.type == "sym_lit" and _text(child).lower() == def_keyword:
seen_keyword = True
continue
if child.type == "sym_lit":
return _text(child)
if child.type == "list_lit":
for gc in child.children:
if gc.type == "sym_lit":
return _text(gc)
return None
return None
def _collect_def_body(node, def_keyword: str) -> list:
"""Collect body forms from (DEFKEYWORD name (params) [doc] body...).
Skips the def keyword, the name, the params list (if present), and
a leading docstring str_lit."""
children = [c for c in node.children if c.type not in ("(", ")")]
idx = 0
# Skip def keyword
if idx < len(children) and children[idx].type == "sym_lit" \
and _text(children[idx]).lower() == def_keyword:
idx += 1
# Skip name (sym_lit or list_lit)
if idx < len(children) and children[idx].type in ("sym_lit", "list_lit"):
idx += 1
# Skip params list (the next list_lit, if any)
if idx < len(children) and children[idx].type == "list_lit":
idx += 1
# Skip leading docstring
if idx < len(children) and children[idx].type == "str_lit":
idx += 1
return children[idx:]
def _handle_def_form(node, def_keyword: str) -> None:
"""Handle a generic (DEFKEYWORD name ...) form. Covers standard CL
definers (defstruct, deftype, defvar, define-condition, etc.) and
custom def-prefixed macros (definline, definline-maybe, etc.)."""
name = _extract_def_name(node, def_keyword)
if not name:
return
line = node.start_point[0] + 1
nid = _cl_id(stem, name)
kw = def_keyword.lower()
if kw in _CL_DATA_DEFINERS:
label = name
is_callable = False
elif kw in _CL_MACRO_DEFINERS or "macro" in kw:
label = f"{name} (macro)"
is_callable = True
else:
# Custom def-prefixed: assume function-like
label = f"{name}()"
is_callable = True
add_node(nid, label, line)
parent_nid = file_nid
if current_package:
pkg_nid = _cl_id(stem, current_package)
if pkg_nid in seen_ids:
parent_nid = pkg_nid
add_edge(parent_nid, nid, "contains", line)
# Docstring. Skip for defvar/defparameter/defconstant — their second
# argument is a value (possibly a string literal) which would be
# wrongly captured as a docstring.
if kw not in _CL_VALUE_DEFINERS:
for child in node.children:
if child.type == "str_lit":
doc_text = _text(child).strip('"')
if doc_text:
doc_nid = _cl_id(nid, "rationale")
add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale")
add_edge(doc_nid, nid, "rationale_for",
child.start_point[0] + 1)
break
if is_callable:
body_nodes = _collect_def_body(node, def_keyword)
if body_nodes:
function_bodies.append((nid, body_nodes))
def _is_def_prefixed(sym: str) -> bool:
"""Heuristic: does this symbol look like a definition macro?"""
if sym in _CL_NOT_DEFINERS:
return False
return sym.startswith("def") or sym.startswith("define-")
def _process_form(top) -> bool:
"""Dispatch on a single list_lit form. Returns True if it was
recognized as a definition or package directive (caller must NOT
recurse into it). Returns False if unrecognized, so the caller can
recurse to find defs nested inside wrapper macros like
(optimizing ...), (eval-when ...), (progn ...), etc."""
nonlocal current_package
# Check for defun node type inside list_lit
for child in top.children:
if child.type == "defun":
_handle_defun_node(child)
return True
first = _first_sym(top)
if not first:
return False
first_lower = first.lower()
if first_lower == "defpackage":
_handle_defpackage(top)
return True
if first_lower == "in-package":
for child in top.children:
if child.type == "kwd_lit":
current_package = _kwd_text(child)
break
if child.type == "sym_lit" and _text(child) != "in-package":
current_package = _text(child)
break
return True
if first_lower == "defclass":
_handle_defclass(top)
return True
if first_lower in ("require", "ql:quickload"):
for child in top.children:
if child.type in ("kwd_lit", "str_lit"):
mod_name = _kwd_text(child) if child.type == "kwd_lit" else _text(child).strip('"')
if mod_name:
tgt_nid = _cl_id(mod_name)
add_edge(file_nid, tgt_nid, "imports",
top.start_point[0] + 1)
break
return True
if first_lower in _CL_DATA_DEFINERS or first_lower in _CL_MACRO_DEFINERS:
_handle_def_form(top, first_lower)
return True
if _is_def_prefixed(first_lower):
# Custom definer (definline, definline-maybe, defcomponent, etc.)
_handle_def_form(top, first_lower)
return True
return False
def _walk_forms(parent) -> None:
"""Walk list_lit children of `parent`, processing each. If a form
isn't recognized, recurse into it — many CL codebases wrap
definitions in macros like (optimizing ...), (eval-when ...), or
(progn ...) that aren't themselves definers but contain defs.
Also descends into reader conditionals (#+feature / #-feature),
which wrap their guarded form in an include_reader_macro node."""
for top in parent.children:
if top.type == "list_lit":
if not _process_form(top):
_walk_forms(top)
elif top.type == "include_reader_macro":
_walk_forms(top)
_walk_forms(root)
# Call extraction pass
label_to_nid: dict[str, str] = {}
for n in nodes:
raw = n["label"]
normalised = raw.replace(" (macro)", "").replace(" (generic)", "").strip("()").lstrip(".")
label_to_nid[normalised.lower()] = n["id"]
seen_call_pairs: set[tuple[str, str]] = set()
def walk_calls(node, caller_nid: str) -> None:
if node.type == "defun":
return
if node.type == "list_lit":
callee = _first_sym(node)
if callee and callee.lower() not in _CL_SPECIAL_FORMS:
tgt_nid = label_to_nid.get(callee.lower())
if tgt_nid and tgt_nid != caller_nid:
pair = (caller_nid, tgt_nid)
if pair not in seen_call_pairs:
seen_call_pairs.add(pair)
add_edge(caller_nid, tgt_nid, "calls",
node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0)
for child in node.children:
walk_calls(child, caller_nid)
for caller_nid, body_nodes in function_bodies:
for body_node in body_nodes:
walk_calls(body_node, caller_nid)
clean_edges = [e for e in edges if e["source"] in seen_ids and
(e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))]
return {"nodes": nodes, "edges": clean_edges}
+5 -1
View File
@@ -88,7 +88,10 @@ terraform = ["tree-sitter-hcl"]
# tree-sitter-ocaml ships prebuilt abi3 wheels for every platform, so no C
# toolchain is needed; kept optional because OCaml is a niche corpus language.
ocaml = ["tree-sitter-ocaml"]
all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml"]
# tree-sitter-commonlisp ships prebuilt abi3 wheels for every platform; optional
# because Common Lisp is a niche corpus language.
commonlisp = ["tree-sitter-commonlisp"]
all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml", "tree-sitter-commonlisp"]
[project.scripts]
graphify = "graphify.__main__:main"
@@ -112,6 +115,7 @@ dev = [
"tomli>=2.0 ; python_version < '3.11'",
"tree-sitter-hcl>=1.2.0",
"tree-sitter-ocaml>=0.25.0",
"tree-sitter-commonlisp>=0.4.1",
]
[tool.uv]
+70
View File
@@ -0,0 +1,70 @@
(defpackage :http-server
(:use :cl :alexandria)
(:export #:make-server #:start #:stop))
(in-package :http-server)
(deftype port-number () '(integer 1 65535))
(defstruct request-stats
bytes-in
bytes-out
duration-ms)
(defstruct (connection (:conc-name conn-))
id
socket
state)
(defvar *active-connections* nil)
(defparameter *default-port* 8080)
(defconstant +max-headers+ 100)
(define-condition server-error (error)
((reason :initarg :reason :reader error-reason)))
(defclass server ()
((host :initarg :host :accessor server-host)
(port :initarg :port :accessor server-port)
(handler :initarg :handler :accessor server-handler)))
(defclass ssl-server (server)
((cert-path :initarg :cert-path :accessor ssl-cert-path)))
(defgeneric process-request (server request))
(defmethod process-request ((srv server) (req string))
"Process an incoming HTTP request."
(let ((parsed (parse-headers req)))
(funcall (server-handler srv) parsed)))
;; Custom definer (Franz-style) — should be picked up by the def-prefix heuristic
(definline-maybe header= (a b)
"Fast header equality."
(string-equal a b))
(definline header< (a b)
(string< a b))
(defun make-server (host port handler)
"Create a new server instance."
(make-instance 'server :host host :port port :handler handler))
(defun start (server)
"Start the server listening on its configured port."
(format t "Starting server on ~a:~a~%" (server-host server) (server-port server))
(process-request server "GET / HTTP/1.1"))
(defun stop (server)
(format t "Stopping server~%"))
(defun compare-headers (h1 h2)
"Compare two headers using the custom definers."
(or (header= h1 h2) (header< h1 h2)))
(defmacro with-server ((var host port handler) &body body)
"Execute body with a running server bound to var."
`(let ((,var (make-server ,host ,port ,handler)))
(unwind-protect
(progn (start ,var) ,@body)
(stop ,var))))
+240 -1
View File
@@ -8,7 +8,7 @@ from graphify.extract import (
extract_swift, extract_go, extract_julia, extract_js, extract_fortran,
extract_groovy, extract_sln, extract_csproj, extract_xaml, extract_razor,
extract_dm, extract_dmi, extract_dmm, extract_dmf,
extract_powershell, extract_apex, extract_verilog,
extract_powershell, extract_apex, extract_commonlisp, extract_verilog,
extract_powershell_manifest,
)
@@ -22,6 +22,10 @@ _needs_dm = pytest.mark.skipif(
_ilu.find_spec("tree_sitter_dm") is None,
reason="tree-sitter-dm not installed (optional [dm] extra)",
)
_needs_commonlisp = pytest.mark.skipif(
_ilu.find_spec("tree_sitter_commonlisp") is None,
reason="tree-sitter-commonlisp not installed (optional [commonlisp] extra)",
)
def _labels(r):
@@ -3052,3 +3056,238 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files():
r = _corpus("cpp_samedir/Alpha.h", "cpp_samedir/Beta.h")
dups = _nodes_with_label(r, "Dup")
assert len(dups) == 2, f"same-dir distinct Dups must stay distinct, got {[n['id'] for n in dups]}"
# ── Common Lisp ──────────────────────────────────────────────────────────────
@_needs_commonlisp
def test_cl_finds_package():
r = extract_commonlisp(FIXTURES / "sample.lisp")
assert "error" not in r
assert "http-server" in _labels(r)
@_needs_commonlisp
def test_cl_finds_class():
r = extract_commonlisp(FIXTURES / "sample.lisp")
assert "server" in _labels(r)
assert "ssl-server" in _labels(r)
@_needs_commonlisp
def test_cl_finds_defun():
r = extract_commonlisp(FIXTURES / "sample.lisp")
labels = _labels(r)
assert any("make-server" in l for l in labels)
assert any("start" in l for l in labels)
assert any("stop" in l for l in labels)
@_needs_commonlisp
def test_cl_finds_generic():
r = extract_commonlisp(FIXTURES / "sample.lisp")
assert any("process-request" in l for l in _labels(r))
@_needs_commonlisp
def test_cl_finds_macro():
r = extract_commonlisp(FIXTURES / "sample.lisp")
assert any("with-server" in l and "macro" in l for l in _labels(r))
@_needs_commonlisp
def test_cl_emits_calls():
r = extract_commonlisp(FIXTURES / "sample.lisp")
calls = _calls(r)
# start() calls process-request
assert any("start" in src and "process-request" in tgt for src, tgt in calls)
# with-server macro calls make-server and start
assert any("with-server" in src and "make-server" in tgt for src, tgt in calls)
assert any("with-server" in src and "start" in tgt for src, tgt in calls)
@_needs_commonlisp
def test_cl_calls_are_extracted():
r = extract_commonlisp(FIXTURES / "sample.lisp")
for e in r["edges"]:
if e["relation"] == "calls":
assert e["confidence"] == "EXTRACTED"
@_needs_commonlisp
def test_cl_no_dangling_edges():
r = extract_commonlisp(FIXTURES / "sample.lisp")
node_ids = {n["id"] for n in r["nodes"]}
for e in r["edges"]:
if e["relation"] in ("contains", "method", "calls"):
assert e["source"] in node_ids
@_needs_commonlisp
def test_cl_docstrings():
r = extract_commonlisp(FIXTURES / "sample.lisp")
rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"]
assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings
labels = _labels(r)
assert any("Process an incoming" in l for l in labels)
@_needs_commonlisp
def test_cl_method_specializers():
r = extract_commonlisp(FIXTURES / "sample.lisp")
spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"]
assert len(spec_edges) >= 1
# process-request specializes on server
assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges)
@_needs_commonlisp
def test_cl_inherits():
r = extract_commonlisp(FIXTURES / "sample.lisp")
inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"]
assert len(inherit_edges) >= 1
assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges)
@_needs_commonlisp
def test_cl_imports():
r = extract_commonlisp(FIXTURES / "sample.lisp")
import_edges = [e for e in r["edges"] if e["relation"] == "imports"]
targets = {e["target"] for e in import_edges}
assert "cl" in targets
assert "alexandria" in targets
@_needs_commonlisp
def test_cl_finds_deftype():
r = extract_commonlisp(FIXTURES / "sample.lisp")
assert "port-number" in _labels(r)
@_needs_commonlisp
def test_cl_finds_defstruct():
r = extract_commonlisp(FIXTURES / "sample.lisp")
labels = _labels(r)
assert "request-stats" in labels
# defstruct with options form: (defstruct (name ...) ...)
assert "connection" in labels
@_needs_commonlisp
def test_cl_finds_defvar_defparameter_defconstant():
r = extract_commonlisp(FIXTURES / "sample.lisp")
labels = _labels(r)
assert "*active-connections*" in labels
assert "*default-port*" in labels
assert "+max-headers+" in labels
@_needs_commonlisp
def test_cl_finds_define_condition():
r = extract_commonlisp(FIXTURES / "sample.lisp")
assert "server-error" in _labels(r)
@_needs_commonlisp
def test_cl_finds_custom_definer():
"""The def-prefix heuristic should catch definline / definline-maybe."""
r = extract_commonlisp(FIXTURES / "sample.lisp")
labels = _labels(r)
# definline-maybe and definline should produce function-style nodes
assert "header=()" in labels
assert "header<()" in labels
@_needs_commonlisp
def test_cl_custom_definer_in_call_graph():
"""Functions defined via custom definers should appear in the call graph."""
r = extract_commonlisp(FIXTURES / "sample.lisp")
calls = _calls(r)
# compare-headers calls header= and header< (defined via definline-maybe / definline)
assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls)
assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls)
@_needs_commonlisp
def test_cl_operator_names_disambiguated():
"""upi=, upi<, upi> must produce distinct ids (operator chars matter)."""
r = extract_commonlisp(FIXTURES / "sample.lisp")
# header= and header< must have different ids
eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"]
lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"]
assert len(eq_ids) == 1
assert len(lt_ids) == 1
assert eq_ids[0] != lt_ids[0]
@_needs_commonlisp
def test_cl_default_value_not_treated_as_definition():
"""The def-prefix heuristic must not match denylisted symbols."""
import tempfile
code = "(in-package :cl-user)\n(default-value foo)\n"
with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f:
f.write(code)
path = Path(f.name)
try:
r = extract_commonlisp(path)
labels = _labels(r)
# default-value is denylisted, shouldn't create a "foo" node
assert "foo" not in labels
assert "foo()" not in labels
finally:
path.unlink()
@_needs_commonlisp
def test_cl_defs_inside_wrapper_macro():
"""Definitions nested inside wrapper macros like (optimizing ...) or
(eval-when ...) must be extracted. Many CL codebases wrap hot-path
inline functions in application-specific macros."""
import tempfile
code = """(in-package :cl-user)
(optimizing
(definline-maybe packet-type (p) (aref p 0))
(definline-maybe set-packet-type (p v) (setf (aref p 0) v)))
(eval-when (:compile-toplevel :load-toplevel :execute)
(defun helper () 42))
(progn
(defun progn-def () 'ok))
"""
with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f:
f.write(code)
path = Path(f.name)
try:
r = extract_commonlisp(path)
labels = _labels(r)
assert "packet-type()" in labels
assert "set-packet-type()" in labels
assert "helper()" in labels
assert "progn-def()" in labels
finally:
path.unlink()
@_needs_commonlisp
def test_cl_defs_inside_reader_conditional():
"""#+feature / #-feature reader conditionals wrap their guarded form
in an include_reader_macro AST node, which the walker must descend
into to find the nested definition."""
import tempfile
code = """(in-package :cl-user)
#+little-endian
(definline-maybe byte-hash= (a b) (eq a b))
#-sbcl
(defun only-on-non-sbcl () 1)
"""
with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f:
f.write(code)
path = Path(f.name)
try:
r = extract_commonlisp(path)
labels = _labels(r)
assert "byte-hash=()" in labels
assert "only-on-non-sbcl()" in labels
finally:
path.unlink()
@_needs_commonlisp
def test_cl_defparameter_string_value_not_docstring():
"""For defvar/defparameter/defconstant, a string literal in the VALUE
position must not be wrongly captured as a docstring node."""
import tempfile
code = '''(in-package :cl-user)
(defparameter *config-path* "/etc/app/config")
(defvar *greeting* "hello world")
'''
with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f:
f.write(code)
path = Path(f.name)
try:
r = extract_commonlisp(path)
labels = _labels(r)
assert "*config-path*" in labels
assert "*greeting*" in labels
# The string VALUES must not show up as rationale nodes
assert not any("/etc/app/config" in l for l in labels)
assert not any("hello world" in l for l in labels)
finally:
path.unlink()
Generated
+24 -1
View File
@@ -1145,6 +1145,7 @@ all = [
{ name = "python-docx" },
{ name = "starlette" },
{ name = "tiktoken" },
{ name = "tree-sitter-commonlisp" },
{ name = "tree-sitter-dm" },
{ name = "tree-sitter-hcl" },
{ name = "tree-sitter-ocaml" },
@@ -1162,6 +1163,9 @@ bedrock = [
chinese = [
{ name = "jieba" },
]
commonlisp = [
{ name = "tree-sitter-commonlisp" },
]
dm = [
{ name = "tree-sitter-dm" },
]
@@ -1246,6 +1250,7 @@ dev = [
{ name = "ruff" },
{ name = "setuptools" },
{ name = "tomli", marker = "python_full_version < '3.11'" },
{ name = "tree-sitter-commonlisp" },
{ name = "tree-sitter-hcl" },
{ name = "tree-sitter-ocaml" },
{ name = "wheel" },
@@ -1301,6 +1306,8 @@ requires-dist = [
{ name = "tree-sitter-bash", specifier = ">=0.23,<0.27" },
{ name = "tree-sitter-c", specifier = ">=0.23,<0.25" },
{ name = "tree-sitter-c-sharp", specifier = ">=0.23,<0.25" },
{ name = "tree-sitter-commonlisp", marker = "extra == 'all'" },
{ name = "tree-sitter-commonlisp", marker = "extra == 'commonlisp'" },
{ name = "tree-sitter-cpp", specifier = ">=0.23,<0.25" },
{ name = "tree-sitter-dm", marker = "extra == 'all'" },
{ name = "tree-sitter-dm", marker = "extra == 'dm'" },
@@ -1338,7 +1345,7 @@ requires-dist = [
{ name = "yt-dlp", marker = "extra == 'all'", specifier = ">=2026.6.9" },
{ name = "yt-dlp", marker = "extra == 'video'", specifier = ">=2026.6.9" },
]
provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "ocaml", "all"]
provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "ocaml", "commonlisp", "all"]
[package.metadata.requires-dev]
dev = [
@@ -1355,6 +1362,7 @@ dev = [
{ name = "ruff", specifier = ">=0.15.13" },
{ name = "setuptools", specifier = ">=82.0.1" },
{ name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" },
{ name = "tree-sitter-commonlisp", specifier = ">=0.4.1" },
{ name = "tree-sitter-hcl", specifier = ">=1.2.0" },
{ name = "tree-sitter-ocaml", specifier = ">=0.25.0" },
{ name = "wheel", specifier = ">=0.47.0" },
@@ -4533,6 +4541,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7f/fb/114ff43fdd256d0befed32f77c1dadee9517867181c70794571f718ed05c/tree_sitter_c_sharp-0.23.5-cp310-abi3-win_arm64.whl", hash = "sha256:2de4ebf95ddc2e92cd3105c8a8e0e7ec646bc82f52bfaf2f3acec0fa2401ec09", size = 337260, upload-time = "2026-04-14T16:11:20.849Z" },
]
[[package]]
name = "tree-sitter-commonlisp"
version = "0.4.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/06/c5/503a9cc9e8ca02387f4424742964242499a407fb8451501ec24e71babc8b/tree_sitter_commonlisp-0.4.1.tar.gz", hash = "sha256:4b8fc7e1ae7faf29d8f656970e25c660b13857e39b55d4a13bcee06ccf3e79c4", size = 238594, upload-time = "2025-03-16T15:42:23.711Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b5/24/7fda258e5aeb8665d7a5d0d6b94f262fb795f014c399f6ec51768b0bea23/tree_sitter_commonlisp-0.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:011ad2788ad8517cc7e96220b5442d32f0b95f002be1ae6db0491e123b9f16d2", size = 106983, upload-time = "2025-03-16T15:42:15.697Z" },
{ url = "https://files.pythonhosted.org/packages/10/e8/d241bf4d543fb982d3eb39dbda68a42f960a236f802dd012766bf305e041/tree_sitter_commonlisp-0.4.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:378a5b3597c0bcd9d65b79b4dae664f4966ccf4fa72fab0f8f08e351d861596d", size = 112788, upload-time = "2025-03-16T15:42:16.86Z" },
{ url = "https://files.pythonhosted.org/packages/3f/ef/0446ee7d5ebc25f384de6fd3a9bb203ff9b701b82ee604967dc11d0c9552/tree_sitter_commonlisp-0.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33ed313aa0d75172a84b816fad3d3d4aa803cc3bd672a3165c5a594e345c2c61", size = 132135, upload-time = "2025-03-16T15:42:17.896Z" },
{ url = "https://files.pythonhosted.org/packages/32/6f/6f42b794a2fa1d69dd34f9ce887839065a8b32405cd613733a6d2ef83da1/tree_sitter_commonlisp-0.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd130f1866009e0b9000d0f11db061fbc0499a0b5b053e4d1f36b9b5cd1f2ef8", size = 122885, upload-time = "2025-03-16T15:42:18.943Z" },
{ url = "https://files.pythonhosted.org/packages/50/29/b9b0519ad3b7cc39201241367c10aa28b1055b2a35d4f4d292d9b6bb1ccc/tree_sitter_commonlisp-0.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3870af93d02039116a8ee1e325bf9b892cb593819e6dbd16ba2971a8f245c0ab", size = 117108, upload-time = "2025-03-16T15:42:19.989Z" },
{ url = "https://files.pythonhosted.org/packages/df/a2/6824c0b84c67f3d4a3855073c7a17273d677ccafbe5779c9a49eb3ef29eb/tree_sitter_commonlisp-0.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:5990660d55567565fee2dd609b4500c6fd0049c82d5ea195779d7c6dd78a75cc", size = 110022, upload-time = "2025-03-16T15:42:20.993Z" },
{ url = "https://files.pythonhosted.org/packages/34/76/1022ce8ec204ef45ce0d2e7e5b42164d0729212135df49eed2fc3ffa115a/tree_sitter_commonlisp-0.4.1-cp38-abi3-win_arm64.whl", hash = "sha256:fca0ea03b60e7f940d466e9c3e757246e8e172d79d4903d2294ab5e73e235eb5", size = 108036, upload-time = "2025-03-16T15:42:22.323Z" },
]
[[package]]
name = "tree-sitter-cpp"
version = "0.23.4"