From 038ada6f0fb13510d9b1e483906e8f0682b2dac6 Mon Sep 17 00:00:00 2001 From: fade <88103+fade@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:05:33 +0100 Subject: [PATCH] feat(commonlisp): add Common Lisp .lisp/.cl/.lsp/.asd extractor (optional tree-sitter-commonlisp extra) tree-sitter-commonlisp-backed extractor for packages, classes, functions, methods, generics, macros, variable definers, and same-file calls. Handles the grammar's dedicated defun node (defun/defmacro/defmethod/defgeneric via defun_header) and the generic list_lit + leading-symbol forms (defvar/defclass/ defpackage), recurses into wrapper macros (eval-when/progn) and reader conditionals, and maps CL operator chars (= < > ? ! + *) to readable id suffixes. Wired into detect/extract dispatch, the extractor registry, a [commonlisp] optional extra, and the README; ships a fixture and a 23-test suite behind an importorskip guard. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 + graphify/detect.py | 2 +- graphify/extract.py | 9 + graphify/extractors/__init__.py | 2 + graphify/extractors/commonlisp.py | 492 ++++++++++++++++++++++++++++++ pyproject.toml | 6 +- tests/fixtures/sample.lisp | 70 +++++ tests/test_languages.py | 241 ++++++++++++++- uv.lock | 25 +- 9 files changed, 845 insertions(+), 4 deletions(-) create mode 100644 graphify/extractors/commonlisp.py create mode 100644 tests/fixtures/sample.lisp diff --git a/README.md b/README.md index e3a2cf8..272c5f6 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,7 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | | `pascal` | Pascal / Delphi `.pas`/`.dpr`/`.dpk`/`.inc` AST extraction (more accurate `calls`/`inherits` edges; falls back to a regex extractor when absent) | `uv tool install "graphifyy[pascal]"` | | `ocaml` | OCaml `.ml`/`.mli` AST extraction | `uv tool install "graphifyy[ocaml]"` | +| `commonlisp` | Common Lisp `.lisp`/`.cl`/`.lsp`/`.asd` AST extraction | `uv tool install "graphifyy[commonlisp]"` | | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | | `all` | Everything above | `uv tool install "graphifyy[all]"` | @@ -341,6 +342,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | OCaml | `.ml .mli` (requires `uv tool install graphifyy[ocaml]`) | +| Common Lisp | `.lisp .cl .lsp .asd` (requires `uv tool install graphifyy[commonlisp]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | | Package manifests | `apm.yml` `pyproject.toml` `go.mod` `pom.xml` — one canonical package node per package (by name) plus `depends_on` edges, so a package referenced from many manifests is a single hub | | Docs | `.md .mdx .qmd .html .txt .rst .yaml .yml` (markdown `[text](./other.md)` links and `[[wikilinks]]` become `references` edges between docs) | diff --git a/graphify/detect.py b/graphify/detect.py index e17809f..4cb1231 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -42,7 +42,7 @@ _MANIFEST_PATH = str(out_path("manifest.json")) _MTIME_COARSE_S = 2.0 _MTIME_SUBSECOND_S = 0.05 -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index 396aa26..a113d4a 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -45,6 +45,7 @@ from graphify.extractors.elixir import extract_elixir # noqa: F401 from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401 from graphify.extractors.go import _GO_PREDECLARED_FUNCS, extract_go # noqa: F401 from graphify.extractors.json_config import extract_json # noqa: F401 +from graphify.extractors.commonlisp import extract_commonlisp # noqa: F401 from graphify.extractors.markdown import extract_markdown # noqa: F401 from graphify.extractors.ocaml import extract_ocaml # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 @@ -4924,6 +4925,10 @@ _DISPATCH: dict[str, Any] = { ".dart": extract_dart, ".ml": extract_ocaml, ".mli": extract_ocaml, + ".lisp": extract_commonlisp, + ".cl": extract_commonlisp, + ".lsp": extract_commonlisp, + ".asd": extract_commonlisp, ".v": extract_verilog, ".sv": extract_verilog, ".svh": extract_verilog, @@ -4978,6 +4983,10 @@ _EXTRA_FOR_EXTENSION = { ".dme": "dm", ".ml": "ocaml", ".mli": "ocaml", + ".lisp": "commonlisp", + ".cl": "commonlisp", + ".lsp": "commonlisp", + ".asd": "commonlisp", } # Substrings an extractor's error carries to classify why a dependency-backed diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada5170..68ff334 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -13,6 +13,7 @@ from typing import Callable from graphify.extractors.apex import extract_apex from graphify.extractors.bash import extract_bash from graphify.extractors.blade import extract_blade +from graphify.extractors.commonlisp import extract_commonlisp from graphify.extractors.dart import extract_dart from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm from graphify.extractors.elixir import extract_elixir @@ -37,6 +38,7 @@ LANGUAGE_EXTRACTORS: dict[str, Callable[[Path], dict]] = { "apex": extract_apex, "bash": extract_bash, "blade": extract_blade, + "commonlisp": extract_commonlisp, "dart": extract_dart, "delphi_form": extract_delphi_form, "dm": extract_dm, diff --git a/graphify/extractors/commonlisp.py b/graphify/extractors/commonlisp.py new file mode 100644 index 0000000..1508809 --- /dev/null +++ b/graphify/extractors/commonlisp.py @@ -0,0 +1,492 @@ +"""Common Lisp extractor for .lisp/.cl/.lsp/.asd, backed by tree-sitter-commonlisp.""" +from __future__ import annotations + +import warnings +from pathlib import Path + +from graphify.extractors.base import _make_id + + +# Standard CL definer forms that introduce data/type/variable bindings +# (not callable, no body to walk for calls) +_CL_DATA_DEFINERS = frozenset({ + "defstruct", "deftype", "define-condition", + "defvar", "defparameter", "defconstant", + "define-symbol-macro", +}) + +# Subset of data definers that take a VALUE as their second argument, which +# may itself be a string literal. For these, we must not mistake the value +# for a docstring. +_CL_VALUE_DEFINERS = frozenset({"defvar", "defparameter", "defconstant"}) + +# Standard CL macro-style definers +_CL_MACRO_DEFINERS = frozenset({ + "define-modify-macro", "define-compiler-macro", + "define-setf-expander", "defsetf", +}) + +# Symbols that start with "def" but are NOT definitions (denylist for the +# def-prefix heuristic that catches custom definers like definline-maybe) +_CL_NOT_DEFINERS = frozenset({ + "default", "default-value", "defaults", "define", + "defer", "deferred", "deflate", "deftest", +}) + +# CL special forms / macros that should not count as "calls" in the graph +_CL_SPECIAL_FORMS = frozenset({ + "let", "let*", "flet", "labels", "macrolet", + "if", "when", "unless", "cond", "case", "ecase", "typecase", "etypecase", + "progn", "prog1", "prog2", "block", "return-from", "tagbody", "go", + "lambda", "function", "funcall", "apply", + "setf", "setq", "psetf", "psetq", "incf", "decf", "push", "pop", + "loop", "do", "do*", "dolist", "dotimes", "map", "mapcar", "mapc", + "format", "print", "princ", "prin1", "write", "terpri", + "and", "or", "not", "null", + "values", "multiple-value-bind", "multiple-value-setq", + "declare", "the", "locally", "eval-when", + "quote", "backquote", + "error", "signal", "warn", "assert", "check-type", + "handler-case", "handler-bind", "restart-case", "ignore-errors", + "unwind-protect", "catch", "throw", + "with-open-file", "with-output-to-string", "with-input-from-string", + "with-slots", "with-accessors", + "defvar", "defparameter", "defconstant", + "in-package", "require", "provide", "use-package", + "t", "nil", +}) + + +def extract_commonlisp(path: Path) -> dict: + """Extract packages, classes, functions, methods, macros, and calls from a Common Lisp file.""" + try: + import tree_sitter_commonlisp as tscl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} + + try: + # tree-sitter-commonlisp 0.4.1 (latest) ships the old binding that returns + # an int pointer from language(); tree_sitter.Language only accepts that + # int via a deprecated path. Silence that one warning at the call site + # until the grammar ships a PyCapsule binding. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="int argument support is deprecated", + category=DeprecationWarning, + ) + language = Language(tscl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = path.stem + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, object]] = [] # (nid, body_nodes) + current_package: str | None = None + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + # CL symbols can contain operator chars like = < > ? ! + * / that the + # generic _make_id strips. Map them to readable suffixes so upi=, upi<, + # upi> become distinct ids. + _CL_CHAR_MAP = { + '=': '_eq', '<': '_lt', '>': '_gt', '?': '_p', '!': '_bang', + '+': '_plus', '*': '_star', '/': '_slash', '%': '_pct', '&': '_amp', + } + + def _cl_id(*parts: str) -> str: + normalized = [] + for p in parts: + normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) + return _make_id(*normalized) + + def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": file_type, + "source_file": str_path, + "source_location": f"L{line}", + }) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({ + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + }) + + file_nid = _cl_id(stem) + add_node(file_nid, path.name, 1) + + def _first_sym(node) -> str | None: + """Get the first sym_lit text from a list_lit's children.""" + for child in node.children: + if child.type == "sym_lit": + return _text(child) + return None + + def _kwd_text(node) -> str: + """Extract the symbol name from a kwd_lit, stripping the leading colon.""" + t = _text(node) + return t.lstrip(":").lstrip("#:") + + def _handle_defpackage(node) -> None: + nonlocal current_package + children = node.children + pkg_name = None + for child in children: + if child.type == "kwd_lit" and pkg_name is None: + pkg_name = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "defpackage": + pkg_name = _text(child) + break + if not pkg_name: + return + current_package = pkg_name + pkg_nid = _cl_id(stem, pkg_name) + line = node.start_point[0] + 1 + add_node(pkg_nid, pkg_name, line) + add_edge(file_nid, pkg_nid, "contains", line) + + # Extract :use clauses as imports + for child in children: + if child.type == "list_lit": + first = _first_sym(child) + if first is None: + # Could be (:use ...) with kwd_lit + for gc in child.children: + if gc.type == "kwd_lit" and _kwd_text(gc) == "use": + for uc in child.children: + if uc.type == "kwd_lit" and uc != gc: + mod_name = _kwd_text(uc) + if mod_name != "use": + tgt_nid = _cl_id(mod_name) + add_edge(pkg_nid, tgt_nid, "imports", + child.start_point[0] + 1) + break + + def _handle_defclass(node) -> None: + children = node.children + # Find class name: second sym_lit after "defclass" + sym_lits = [c for c in children if c.type == "sym_lit"] + if len(sym_lits) < 2: + return + class_name = _text(sym_lits[1]) + line = node.start_point[0] + 1 + class_nid = _cl_id(stem, class_name) + add_node(class_nid, class_name, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, class_nid, "contains", line) + + # Superclasses (third element, a list) + list_lits = [c for c in children if c.type == "list_lit"] + if list_lits: + superclass_list = list_lits[0] + for sc in superclass_list.children: + if sc.type == "sym_lit": + sc_name = _text(sc) + sc_nid = _cl_id(stem, sc_name) + add_edge(class_nid, sc_nid, "inherits", + superclass_list.start_point[0] + 1) + + def _handle_defun_node(node) -> None: + """Handle a defun AST node (covers defun, defmethod, defgeneric, defmacro).""" + header = None + for child in node.children: + if child.type == "defun_header": + header = child + break + if not header: + return + + # Determine keyword type + keyword_type = "defun" + func_name = None + for child in header.children: + if child.type == "defun_keyword": + for kc in child.children: + if kc.type in ("defun", "defmethod", "defgeneric", "defmacro"): + keyword_type = kc.type + break + if child.type == "sym_lit" and func_name is None: + func_name = _text(child) + + if not func_name: + return + + line = node.start_point[0] + 1 + func_nid = _cl_id(stem, func_name) + + if keyword_type == "defmethod": + label = f".{func_name}()" + elif keyword_type == "defmacro": + label = f"{func_name} (macro)" + elif keyword_type == "defgeneric": + label = f"{func_name} (generic)" + else: + label = f"{func_name}()" + + add_node(func_nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + if keyword_type == "defmethod": + add_edge(parent_nid, func_nid, "method", line) + else: + add_edge(parent_nid, func_nid, "contains", line) + + # Method specializers: (defmethod name ((param class) ...)) + if keyword_type == "defmethod": + for child in header.children: + if child.type == "list_lit": + # This is the parameter list + for param in child.children: + if param.type == "list_lit": + syms = [c for c in param.children if c.type == "sym_lit"] + if len(syms) >= 2: + specializer_name = _text(syms[1]) + spec_nid = _cl_id(stem, specializer_name) + add_edge(func_nid, spec_nid, "specializes", + param.start_point[0] + 1) + break + + # Docstring + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(func_nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") + add_edge(doc_nid, func_nid, "rationale_for", + child.start_point[0] + 1) + break + + # Collect body for call extraction + body_nodes = [c for c in node.children + if c.type not in ("(", ")", "defun_header", "str_lit")] + if body_nodes: + function_bodies.append((func_nid, body_nodes)) + + def _extract_def_name(node, def_keyword: str) -> str | None: + """Get the name being defined. May be a sym_lit or a list_lit whose + first sym_lit is the name (e.g. (defstruct (foo :conc-name "BAR-") ...)).""" + seen_keyword = False + for child in node.children: + if not seen_keyword: + if child.type == "sym_lit" and _text(child).lower() == def_keyword: + seen_keyword = True + continue + if child.type == "sym_lit": + return _text(child) + if child.type == "list_lit": + for gc in child.children: + if gc.type == "sym_lit": + return _text(gc) + return None + return None + + def _collect_def_body(node, def_keyword: str) -> list: + """Collect body forms from (DEFKEYWORD name (params) [doc] body...). + Skips the def keyword, the name, the params list (if present), and + a leading docstring str_lit.""" + children = [c for c in node.children if c.type not in ("(", ")")] + idx = 0 + # Skip def keyword + if idx < len(children) and children[idx].type == "sym_lit" \ + and _text(children[idx]).lower() == def_keyword: + idx += 1 + # Skip name (sym_lit or list_lit) + if idx < len(children) and children[idx].type in ("sym_lit", "list_lit"): + idx += 1 + # Skip params list (the next list_lit, if any) + if idx < len(children) and children[idx].type == "list_lit": + idx += 1 + # Skip leading docstring + if idx < len(children) and children[idx].type == "str_lit": + idx += 1 + return children[idx:] + + def _handle_def_form(node, def_keyword: str) -> None: + """Handle a generic (DEFKEYWORD name ...) form. Covers standard CL + definers (defstruct, deftype, defvar, define-condition, etc.) and + custom def-prefixed macros (definline, definline-maybe, etc.).""" + name = _extract_def_name(node, def_keyword) + if not name: + return + line = node.start_point[0] + 1 + nid = _cl_id(stem, name) + + kw = def_keyword.lower() + if kw in _CL_DATA_DEFINERS: + label = name + is_callable = False + elif kw in _CL_MACRO_DEFINERS or "macro" in kw: + label = f"{name} (macro)" + is_callable = True + else: + # Custom def-prefixed: assume function-like + label = f"{name}()" + is_callable = True + + add_node(nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, nid, "contains", line) + + # Docstring. Skip for defvar/defparameter/defconstant — their second + # argument is a value (possibly a string literal) which would be + # wrongly captured as a docstring. + if kw not in _CL_VALUE_DEFINERS: + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") + add_edge(doc_nid, nid, "rationale_for", + child.start_point[0] + 1) + break + + if is_callable: + body_nodes = _collect_def_body(node, def_keyword) + if body_nodes: + function_bodies.append((nid, body_nodes)) + + def _is_def_prefixed(sym: str) -> bool: + """Heuristic: does this symbol look like a definition macro?""" + if sym in _CL_NOT_DEFINERS: + return False + return sym.startswith("def") or sym.startswith("define-") + + def _process_form(top) -> bool: + """Dispatch on a single list_lit form. Returns True if it was + recognized as a definition or package directive (caller must NOT + recurse into it). Returns False if unrecognized, so the caller can + recurse to find defs nested inside wrapper macros like + (optimizing ...), (eval-when ...), (progn ...), etc.""" + nonlocal current_package + + # Check for defun node type inside list_lit + for child in top.children: + if child.type == "defun": + _handle_defun_node(child) + return True + + first = _first_sym(top) + if not first: + return False + first_lower = first.lower() + + if first_lower == "defpackage": + _handle_defpackage(top) + return True + if first_lower == "in-package": + for child in top.children: + if child.type == "kwd_lit": + current_package = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "in-package": + current_package = _text(child) + break + return True + if first_lower == "defclass": + _handle_defclass(top) + return True + if first_lower in ("require", "ql:quickload"): + for child in top.children: + if child.type in ("kwd_lit", "str_lit"): + mod_name = _kwd_text(child) if child.type == "kwd_lit" else _text(child).strip('"') + if mod_name: + tgt_nid = _cl_id(mod_name) + add_edge(file_nid, tgt_nid, "imports", + top.start_point[0] + 1) + break + return True + if first_lower in _CL_DATA_DEFINERS or first_lower in _CL_MACRO_DEFINERS: + _handle_def_form(top, first_lower) + return True + if _is_def_prefixed(first_lower): + # Custom definer (definline, definline-maybe, defcomponent, etc.) + _handle_def_form(top, first_lower) + return True + + return False + + def _walk_forms(parent) -> None: + """Walk list_lit children of `parent`, processing each. If a form + isn't recognized, recurse into it — many CL codebases wrap + definitions in macros like (optimizing ...), (eval-when ...), or + (progn ...) that aren't themselves definers but contain defs. + Also descends into reader conditionals (#+feature / #-feature), + which wrap their guarded form in an include_reader_macro node.""" + for top in parent.children: + if top.type == "list_lit": + if not _process_form(top): + _walk_forms(top) + elif top.type == "include_reader_macro": + _walk_forms(top) + + _walk_forms(root) + + # Call extraction pass + label_to_nid: dict[str, str] = {} + for n in nodes: + raw = n["label"] + normalised = raw.replace(" (macro)", "").replace(" (generic)", "").strip("()").lstrip(".") + label_to_nid[normalised.lower()] = n["id"] + + seen_call_pairs: set[tuple[str, str]] = set() + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "defun": + return + if node.type == "list_lit": + callee = _first_sym(node) + if callee and callee.lower() not in _CL_SPECIAL_FORMS: + tgt_nid = label_to_nid.get(callee.lower()) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid) + if pair not in seen_call_pairs: + seen_call_pairs.add(pair) + add_edge(caller_nid, tgt_nid, "calls", + node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + for child in node.children: + walk_calls(child, caller_nid) + + for caller_nid, body_nodes in function_bodies: + for body_node in body_nodes: + walk_calls(body_node, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] + return {"nodes": nodes, "edges": clean_edges} diff --git a/pyproject.toml b/pyproject.toml index 72d9ff6..e636830 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,10 @@ terraform = ["tree-sitter-hcl"] # tree-sitter-ocaml ships prebuilt abi3 wheels for every platform, so no C # toolchain is needed; kept optional because OCaml is a niche corpus language. ocaml = ["tree-sitter-ocaml"] -all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml"] +# tree-sitter-commonlisp ships prebuilt abi3 wheels for every platform; optional +# because Common Lisp is a niche corpus language. +commonlisp = ["tree-sitter-commonlisp"] +all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml", "tree-sitter-commonlisp"] [project.scripts] graphify = "graphify.__main__:main" @@ -112,6 +115,7 @@ dev = [ "tomli>=2.0 ; python_version < '3.11'", "tree-sitter-hcl>=1.2.0", "tree-sitter-ocaml>=0.25.0", + "tree-sitter-commonlisp>=0.4.1", ] [tool.uv] diff --git a/tests/fixtures/sample.lisp b/tests/fixtures/sample.lisp new file mode 100644 index 0000000..4dc2dc4 --- /dev/null +++ b/tests/fixtures/sample.lisp @@ -0,0 +1,70 @@ +(defpackage :http-server + (:use :cl :alexandria) + (:export #:make-server #:start #:stop)) + +(in-package :http-server) + +(deftype port-number () '(integer 1 65535)) + +(defstruct request-stats + bytes-in + bytes-out + duration-ms) + +(defstruct (connection (:conc-name conn-)) + id + socket + state) + +(defvar *active-connections* nil) +(defparameter *default-port* 8080) +(defconstant +max-headers+ 100) + +(define-condition server-error (error) + ((reason :initarg :reason :reader error-reason))) + +(defclass server () + ((host :initarg :host :accessor server-host) + (port :initarg :port :accessor server-port) + (handler :initarg :handler :accessor server-handler))) + +(defclass ssl-server (server) + ((cert-path :initarg :cert-path :accessor ssl-cert-path))) + +(defgeneric process-request (server request)) + +(defmethod process-request ((srv server) (req string)) + "Process an incoming HTTP request." + (let ((parsed (parse-headers req))) + (funcall (server-handler srv) parsed))) + +;; Custom definer (Franz-style) — should be picked up by the def-prefix heuristic +(definline-maybe header= (a b) + "Fast header equality." + (string-equal a b)) + +(definline header< (a b) + (string< a b)) + +(defun make-server (host port handler) + "Create a new server instance." + (make-instance 'server :host host :port port :handler handler)) + +(defun start (server) + "Start the server listening on its configured port." + (format t "Starting server on ~a:~a~%" (server-host server) (server-port server)) + (process-request server "GET / HTTP/1.1")) + +(defun stop (server) + (format t "Stopping server~%")) + +(defun compare-headers (h1 h2) + "Compare two headers using the custom definers." + (or (header= h1 h2) (header< h1 h2))) + +(defmacro with-server ((var host port handler) &body body) + "Execute body with a running server bound to var." + `(let ((,var (make-server ,host ,port ,handler))) + (unwind-protect + (progn (start ,var) ,@body) + (stop ,var)))) diff --git a/tests/test_languages.py b/tests/test_languages.py index 37466ec..fcadfcd 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -8,7 +8,7 @@ from graphify.extract import ( extract_swift, extract_go, extract_julia, extract_js, extract_fortran, extract_groovy, extract_sln, extract_csproj, extract_xaml, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, - extract_powershell, extract_apex, extract_verilog, + extract_powershell, extract_apex, extract_commonlisp, extract_verilog, extract_powershell_manifest, ) @@ -22,6 +22,10 @@ _needs_dm = pytest.mark.skipif( _ilu.find_spec("tree_sitter_dm") is None, reason="tree-sitter-dm not installed (optional [dm] extra)", ) +_needs_commonlisp = pytest.mark.skipif( + _ilu.find_spec("tree_sitter_commonlisp") is None, + reason="tree-sitter-commonlisp not installed (optional [commonlisp] extra)", +) def _labels(r): @@ -3052,3 +3056,238 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): r = _corpus("cpp_samedir/Alpha.h", "cpp_samedir/Beta.h") dups = _nodes_with_label(r, "Dup") assert len(dups) == 2, f"same-dir distinct Dups must stay distinct, got {[n['id'] for n in dups]}" +# ── Common Lisp ────────────────────────────────────────────────────────────── + +@_needs_commonlisp +def test_cl_finds_package(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "error" not in r + assert "http-server" in _labels(r) + +@_needs_commonlisp +def test_cl_finds_class(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server" in _labels(r) + assert "ssl-server" in _labels(r) + +@_needs_commonlisp +def test_cl_finds_defun(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert any("make-server" in l for l in labels) + assert any("start" in l for l in labels) + assert any("stop" in l for l in labels) + +@_needs_commonlisp +def test_cl_finds_generic(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("process-request" in l for l in _labels(r)) + +@_needs_commonlisp +def test_cl_finds_macro(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("with-server" in l and "macro" in l for l in _labels(r)) + +@_needs_commonlisp +def test_cl_emits_calls(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # start() calls process-request + assert any("start" in src and "process-request" in tgt for src, tgt in calls) + # with-server macro calls make-server and start + assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) + assert any("with-server" in src and "start" in tgt for src, tgt in calls) + +@_needs_commonlisp +def test_cl_calls_are_extracted(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + for e in r["edges"]: + if e["relation"] == "calls": + assert e["confidence"] == "EXTRACTED" + +@_needs_commonlisp +def test_cl_no_dangling_edges(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + if e["relation"] in ("contains", "method", "calls"): + assert e["source"] in node_ids + +@_needs_commonlisp +def test_cl_docstrings(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] + assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings + labels = _labels(r) + assert any("Process an incoming" in l for l in labels) + +@_needs_commonlisp +def test_cl_method_specializers(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] + assert len(spec_edges) >= 1 + # process-request specializes on server + assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) + +@_needs_commonlisp +def test_cl_inherits(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherit_edges) >= 1 + assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) + +@_needs_commonlisp +def test_cl_imports(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + targets = {e["target"] for e in import_edges} + assert "cl" in targets + assert "alexandria" in targets + +@_needs_commonlisp +def test_cl_finds_deftype(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "port-number" in _labels(r) + +@_needs_commonlisp +def test_cl_finds_defstruct(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "request-stats" in labels + # defstruct with options form: (defstruct (name ...) ...) + assert "connection" in labels + +@_needs_commonlisp +def test_cl_finds_defvar_defparameter_defconstant(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "*active-connections*" in labels + assert "*default-port*" in labels + assert "+max-headers+" in labels + +@_needs_commonlisp +def test_cl_finds_define_condition(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server-error" in _labels(r) + +@_needs_commonlisp +def test_cl_finds_custom_definer(): + """The def-prefix heuristic should catch definline / definline-maybe.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + # definline-maybe and definline should produce function-style nodes + assert "header=()" in labels + assert "header<()" in labels + +@_needs_commonlisp +def test_cl_custom_definer_in_call_graph(): + """Functions defined via custom definers should appear in the call graph.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # compare-headers calls header= and header< (defined via definline-maybe / definline) + assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) + assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) + +@_needs_commonlisp +def test_cl_operator_names_disambiguated(): + """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + # header= and header< must have different ids + eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] + lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] + assert len(eq_ids) == 1 + assert len(lt_ids) == 1 + assert eq_ids[0] != lt_ids[0] + +@_needs_commonlisp +def test_cl_default_value_not_treated_as_definition(): + """The def-prefix heuristic must not match denylisted symbols.""" + import tempfile + code = "(in-package :cl-user)\n(default-value foo)\n" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + # default-value is denylisted, shouldn't create a "foo" node + assert "foo" not in labels + assert "foo()" not in labels + finally: + path.unlink() + +@_needs_commonlisp +def test_cl_defs_inside_wrapper_macro(): + """Definitions nested inside wrapper macros like (optimizing ...) or + (eval-when ...) must be extracted. Many CL codebases wrap hot-path + inline functions in application-specific macros.""" + import tempfile + code = """(in-package :cl-user) +(optimizing + (definline-maybe packet-type (p) (aref p 0)) + (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) +(eval-when (:compile-toplevel :load-toplevel :execute) + (defun helper () 42)) +(progn + (defun progn-def () 'ok)) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "packet-type()" in labels + assert "set-packet-type()" in labels + assert "helper()" in labels + assert "progn-def()" in labels + finally: + path.unlink() + +@_needs_commonlisp +def test_cl_defs_inside_reader_conditional(): + """#+feature / #-feature reader conditionals wrap their guarded form + in an include_reader_macro AST node, which the walker must descend + into to find the nested definition.""" + import tempfile + code = """(in-package :cl-user) +#+little-endian +(definline-maybe byte-hash= (a b) (eq a b)) +#-sbcl +(defun only-on-non-sbcl () 1) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "byte-hash=()" in labels + assert "only-on-non-sbcl()" in labels + finally: + path.unlink() + +@_needs_commonlisp +def test_cl_defparameter_string_value_not_docstring(): + """For defvar/defparameter/defconstant, a string literal in the VALUE + position must not be wrongly captured as a docstring node.""" + import tempfile + code = '''(in-package :cl-user) +(defparameter *config-path* "/etc/app/config") +(defvar *greeting* "hello world") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "*config-path*" in labels + assert "*greeting*" in labels + # The string VALUES must not show up as rationale nodes + assert not any("/etc/app/config" in l for l in labels) + assert not any("hello world" in l for l in labels) + finally: + path.unlink() + + diff --git a/uv.lock b/uv.lock index 126ff23..1850521 100644 --- a/uv.lock +++ b/uv.lock @@ -1145,6 +1145,7 @@ all = [ { name = "python-docx" }, { name = "starlette" }, { name = "tiktoken" }, + { name = "tree-sitter-commonlisp" }, { name = "tree-sitter-dm" }, { name = "tree-sitter-hcl" }, { name = "tree-sitter-ocaml" }, @@ -1162,6 +1163,9 @@ bedrock = [ chinese = [ { name = "jieba" }, ] +commonlisp = [ + { name = "tree-sitter-commonlisp" }, +] dm = [ { name = "tree-sitter-dm" }, ] @@ -1246,6 +1250,7 @@ dev = [ { name = "ruff" }, { name = "setuptools" }, { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tree-sitter-commonlisp" }, { name = "tree-sitter-hcl" }, { name = "tree-sitter-ocaml" }, { name = "wheel" }, @@ -1301,6 +1306,8 @@ requires-dist = [ { name = "tree-sitter-bash", specifier = ">=0.23,<0.27" }, { name = "tree-sitter-c", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-c-sharp", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-commonlisp", marker = "extra == 'all'" }, + { name = "tree-sitter-commonlisp", marker = "extra == 'commonlisp'" }, { name = "tree-sitter-cpp", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-dm", marker = "extra == 'all'" }, { name = "tree-sitter-dm", marker = "extra == 'dm'" }, @@ -1338,7 +1345,7 @@ requires-dist = [ { name = "yt-dlp", marker = "extra == 'all'", specifier = ">=2026.6.9" }, { name = "yt-dlp", marker = "extra == 'video'", specifier = ">=2026.6.9" }, ] -provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "ocaml", "all"] +provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "ocaml", "commonlisp", "all"] [package.metadata.requires-dev] dev = [ @@ -1355,6 +1362,7 @@ dev = [ { name = "ruff", specifier = ">=0.15.13" }, { name = "setuptools", specifier = ">=82.0.1" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" }, + { name = "tree-sitter-commonlisp", specifier = ">=0.4.1" }, { name = "tree-sitter-hcl", specifier = ">=1.2.0" }, { name = "tree-sitter-ocaml", specifier = ">=0.25.0" }, { name = "wheel", specifier = ">=0.47.0" }, @@ -4533,6 +4541,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/fb/114ff43fdd256d0befed32f77c1dadee9517867181c70794571f718ed05c/tree_sitter_c_sharp-0.23.5-cp310-abi3-win_arm64.whl", hash = "sha256:2de4ebf95ddc2e92cd3105c8a8e0e7ec646bc82f52bfaf2f3acec0fa2401ec09", size = 337260, upload-time = "2026-04-14T16:11:20.849Z" }, ] +[[package]] +name = "tree-sitter-commonlisp" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/c5/503a9cc9e8ca02387f4424742964242499a407fb8451501ec24e71babc8b/tree_sitter_commonlisp-0.4.1.tar.gz", hash = "sha256:4b8fc7e1ae7faf29d8f656970e25c660b13857e39b55d4a13bcee06ccf3e79c4", size = 238594, upload-time = "2025-03-16T15:42:23.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/24/7fda258e5aeb8665d7a5d0d6b94f262fb795f014c399f6ec51768b0bea23/tree_sitter_commonlisp-0.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:011ad2788ad8517cc7e96220b5442d32f0b95f002be1ae6db0491e123b9f16d2", size = 106983, upload-time = "2025-03-16T15:42:15.697Z" }, + { url = "https://files.pythonhosted.org/packages/10/e8/d241bf4d543fb982d3eb39dbda68a42f960a236f802dd012766bf305e041/tree_sitter_commonlisp-0.4.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:378a5b3597c0bcd9d65b79b4dae664f4966ccf4fa72fab0f8f08e351d861596d", size = 112788, upload-time = "2025-03-16T15:42:16.86Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ef/0446ee7d5ebc25f384de6fd3a9bb203ff9b701b82ee604967dc11d0c9552/tree_sitter_commonlisp-0.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33ed313aa0d75172a84b816fad3d3d4aa803cc3bd672a3165c5a594e345c2c61", size = 132135, upload-time = "2025-03-16T15:42:17.896Z" }, + { url = "https://files.pythonhosted.org/packages/32/6f/6f42b794a2fa1d69dd34f9ce887839065a8b32405cd613733a6d2ef83da1/tree_sitter_commonlisp-0.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd130f1866009e0b9000d0f11db061fbc0499a0b5b053e4d1f36b9b5cd1f2ef8", size = 122885, upload-time = "2025-03-16T15:42:18.943Z" }, + { url = "https://files.pythonhosted.org/packages/50/29/b9b0519ad3b7cc39201241367c10aa28b1055b2a35d4f4d292d9b6bb1ccc/tree_sitter_commonlisp-0.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3870af93d02039116a8ee1e325bf9b892cb593819e6dbd16ba2971a8f245c0ab", size = 117108, upload-time = "2025-03-16T15:42:19.989Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/6824c0b84c67f3d4a3855073c7a17273d677ccafbe5779c9a49eb3ef29eb/tree_sitter_commonlisp-0.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:5990660d55567565fee2dd609b4500c6fd0049c82d5ea195779d7c6dd78a75cc", size = 110022, upload-time = "2025-03-16T15:42:20.993Z" }, + { url = "https://files.pythonhosted.org/packages/34/76/1022ce8ec204ef45ce0d2e7e5b42164d0729212135df49eed2fc3ffa115a/tree_sitter_commonlisp-0.4.1-cp38-abi3-win_arm64.whl", hash = "sha256:fca0ea03b60e7f940d466e9c3e757246e8e172d79d4903d2294ab5e73e235eb5", size = 108036, upload-time = "2025-03-16T15:42:22.323Z" }, +] + [[package]] name = "tree-sitter-cpp" version = "0.23.4"