feat(extract): add .astro support (#850)
Astro files have a `---...---` TypeScript frontmatter block at the top containing nearly all imports, followed by an HTML-with-expressions template body, and optionally `<script>` blocks for client-side JS. Feeding the whole file to the JS tree-sitter parser produces a top-level ERROR node because the template isn't valid JS, so `import_statement` nodes are never reached. Mirrors `extract_svelte` (#713) — same regex-rescue strategy: - Dynamic `import('...')` pass over the full source. - Static `import X from '...'` pass over the frontmatter block plus any `<script>` blocks. Both static and dynamic imports honor tsconfig.json path aliases and resolve relative paths through `_resolve_js_module_path` so node ids match the targets that `_extract_generic` produces for sibling files. Wired in: - `detect.py:27` — `.astro` added to `CODE_EXTENSIONS`. - `extract.py:5492` — `".astro": extract_astro` in the dispatch table. Tests in `tests/test_astro_extraction.py` cover: extension registration, frontmatter static imports, dynamic import in frontmatter, client-side `<script>` block imports, frontmatter-less pages (must not crash), and tsconfig.json path-alias resolution.
This commit is contained in:
+1
-1
@@ -24,7 +24,7 @@ class FileType(str, Enum):
|
|||||||
|
|
||||||
_MANIFEST_PATH = "graphify-out/manifest.json"
|
_MANIFEST_PATH = "graphify-out/manifest.json"
|
||||||
|
|
||||||
CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.dart', '.v', '.sv', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk'}
|
CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk'}
|
||||||
DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'}
|
DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'}
|
||||||
PAPER_EXTENSIONS = {'.pdf'}
|
PAPER_EXTENSIONS = {'.pdf'}
|
||||||
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
|
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
|
||||||
|
|||||||
@@ -2123,6 +2123,142 @@ def extract_svelte(path: Path) -> dict:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def extract_astro(path: Path) -> dict:
|
||||||
|
"""Extract imports from .astro files: frontmatter (TS) + template regex fallback.
|
||||||
|
|
||||||
|
Astro files start with a ``---\\n...\\n---`` frontmatter block of TypeScript
|
||||||
|
setup code (where almost all imports live), followed by an HTML-with-expressions
|
||||||
|
template body, and optionally ``<script>`` blocks for client-side JS. Tree-sitter
|
||||||
|
only sees the file usefully through the frontmatter — feeding the whole file to
|
||||||
|
the JS parser produces a top-level ERROR node because the template is not valid
|
||||||
|
JS, so ``import_statement`` nodes are never reached and static imports are
|
||||||
|
silently dropped (#850). Mirrors :func:`extract_svelte` — same regex-rescue
|
||||||
|
approach, scanning the frontmatter block and any client-side ``<script>`` blocks
|
||||||
|
for static and dynamic imports.
|
||||||
|
"""
|
||||||
|
result = _extract_generic(path, _JS_CONFIG)
|
||||||
|
try:
|
||||||
|
import re as _re
|
||||||
|
src = path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
existing_ids = {n["id"] for n in result.get("nodes", [])}
|
||||||
|
file_node_id = _make_id(str(path))
|
||||||
|
aliases = _load_tsconfig_aliases(path.parent)
|
||||||
|
# Dynamic imports anywhere in the file: `import('./X.astro')` is legal in
|
||||||
|
# frontmatter setup code and inside expression slots.
|
||||||
|
for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
|
||||||
|
raw = m.group(1)
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
if raw.startswith("."):
|
||||||
|
resolved = Path(os.path.normpath(path.parent / raw))
|
||||||
|
resolved = _resolve_js_module_path(resolved)
|
||||||
|
node_id = _make_id(str(resolved))
|
||||||
|
stub_source_file = str(resolved)
|
||||||
|
else:
|
||||||
|
resolved_alias = None
|
||||||
|
for alias_prefix, alias_base in aliases.items():
|
||||||
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
||||||
|
rest = raw[len(alias_prefix):].lstrip("/")
|
||||||
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
||||||
|
break
|
||||||
|
if resolved_alias is not None:
|
||||||
|
resolved_alias = _resolve_js_module_path(resolved_alias)
|
||||||
|
node_id = _make_id(str(resolved_alias))
|
||||||
|
stub_source_file = str(resolved_alias)
|
||||||
|
else:
|
||||||
|
module_name = raw.split("/")[-1]
|
||||||
|
if not module_name:
|
||||||
|
continue
|
||||||
|
node_id = _make_id(module_name)
|
||||||
|
stub_source_file = raw
|
||||||
|
if node_id in existing_ids:
|
||||||
|
result.setdefault("edges", []).append({
|
||||||
|
"source": file_node_id, "target": node_id,
|
||||||
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
||||||
|
"source_file": str(path),
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
result.setdefault("nodes", []).append({
|
||||||
|
"id": node_id, "label": raw,
|
||||||
|
"file_type": "code", "source_file": stub_source_file,
|
||||||
|
"confidence": "EXTRACTED",
|
||||||
|
})
|
||||||
|
result.setdefault("edges", []).append({
|
||||||
|
"source": file_node_id, "target": node_id,
|
||||||
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
||||||
|
"source_file": str(path),
|
||||||
|
})
|
||||||
|
existing_ids.add(node_id)
|
||||||
|
# Static imports: scan the `---...---` frontmatter at the file head plus any
|
||||||
|
# client-side <script> blocks. Both are TS/JS regions but live inside a file
|
||||||
|
# the JS tree-sitter parser cannot validate as a whole.
|
||||||
|
frontmatter_re = _re.compile(
|
||||||
|
r"\A\s*---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|\Z)"
|
||||||
|
)
|
||||||
|
script_re = _re.compile(
|
||||||
|
r"<script\b[^>]*>([\s\S]*?)</script\s*>", _re.IGNORECASE
|
||||||
|
)
|
||||||
|
static_import_re = _re.compile(
|
||||||
|
r"""import\s+(?:[^'"`;]+?\s+from\s+)?['"]([^'"]+)['"]"""
|
||||||
|
)
|
||||||
|
regions: list[str] = []
|
||||||
|
fm = frontmatter_re.search(src)
|
||||||
|
if fm:
|
||||||
|
regions.append(fm.group(1))
|
||||||
|
for script_match in script_re.finditer(src):
|
||||||
|
regions.append(script_match.group(1))
|
||||||
|
for region in regions:
|
||||||
|
for m in static_import_re.finditer(region):
|
||||||
|
raw = m.group(1)
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
if raw.startswith("."):
|
||||||
|
resolved = Path(os.path.normpath(path.parent / raw))
|
||||||
|
if resolved.suffix == ".js":
|
||||||
|
resolved = resolved.with_suffix(".ts")
|
||||||
|
elif resolved.suffix == ".jsx":
|
||||||
|
resolved = resolved.with_suffix(".tsx")
|
||||||
|
node_id = _make_id(str(resolved))
|
||||||
|
stub_source_file = str(resolved)
|
||||||
|
else:
|
||||||
|
resolved_alias = None
|
||||||
|
for alias_prefix, alias_base in aliases.items():
|
||||||
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
||||||
|
rest = raw[len(alias_prefix):].lstrip("/")
|
||||||
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
||||||
|
break
|
||||||
|
if resolved_alias is not None:
|
||||||
|
node_id = _make_id(str(resolved_alias))
|
||||||
|
stub_source_file = str(resolved_alias)
|
||||||
|
else:
|
||||||
|
module_name = raw.split("/")[-1]
|
||||||
|
if not module_name:
|
||||||
|
continue
|
||||||
|
node_id = _make_id(module_name)
|
||||||
|
stub_source_file = raw
|
||||||
|
if node_id in existing_ids:
|
||||||
|
result.setdefault("edges", []).append({
|
||||||
|
"source": file_node_id, "target": node_id,
|
||||||
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
||||||
|
"source_file": str(path),
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
result.setdefault("nodes", []).append({
|
||||||
|
"id": node_id, "label": raw,
|
||||||
|
"file_type": "code", "source_file": stub_source_file,
|
||||||
|
"confidence": "EXTRACTED",
|
||||||
|
})
|
||||||
|
result.setdefault("edges", []).append({
|
||||||
|
"source": file_node_id, "target": node_id,
|
||||||
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
||||||
|
"source_file": str(path),
|
||||||
|
})
|
||||||
|
existing_ids.add(node_id)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def extract_java(path: Path) -> dict:
|
def extract_java(path: Path) -> dict:
|
||||||
"""Extract classes, interfaces, methods, constructors, and imports from a .java file."""
|
"""Extract classes, interfaces, methods, constructors, and imports from a .java file."""
|
||||||
return _extract_generic(path, _JAVA_CONFIG)
|
return _extract_generic(path, _JAVA_CONFIG)
|
||||||
@@ -5490,6 +5626,7 @@ _DISPATCH: dict[str, Any] = {
|
|||||||
".F08": extract_fortran,
|
".F08": extract_fortran,
|
||||||
".vue": extract_js,
|
".vue": extract_js,
|
||||||
".svelte": extract_svelte,
|
".svelte": extract_svelte,
|
||||||
|
".astro": extract_astro,
|
||||||
".dart": extract_dart,
|
".dart": extract_dart,
|
||||||
".v": extract_verilog,
|
".v": extract_verilog,
|
||||||
".sv": extract_verilog,
|
".sv": extract_verilog,
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
"""Tests for `.astro` extraction (#850).
|
||||||
|
|
||||||
|
Astro files have a TypeScript frontmatter block (`---...---`) at the top where
|
||||||
|
nearly all imports live, followed by an HTML-with-expressions template and
|
||||||
|
optionally `<script>` blocks. Tree-sitter-javascript fed the whole file produces
|
||||||
|
a top-level ERROR node because the template is not valid JS, so the JS AST pass
|
||||||
|
recovers nothing. The :func:`extract_astro` regex pass salvages imports from the
|
||||||
|
frontmatter and any `<script>` blocks — same strategy as :func:`extract_svelte`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from graphify.detect import CODE_EXTENSIONS
|
||||||
|
from graphify.extract import (
|
||||||
|
_make_id,
|
||||||
|
extract_astro,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write(path: Path, body: str) -> Path:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(body, encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _import_targets(result: dict, *, relation: str | None = None) -> set[str]:
|
||||||
|
return {
|
||||||
|
str(e.get("target") or "")
|
||||||
|
for e in result.get("edges", [])
|
||||||
|
if relation is None or e.get("relation") == relation
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_astro_is_in_code_extensions():
|
||||||
|
"""Without this, detect.py silently drops `.astro` from the AST pass (#850)."""
|
||||||
|
assert ".astro" in CODE_EXTENSIONS
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_astro_picks_up_frontmatter_static_imports(tmp_path):
|
||||||
|
page = _write(
|
||||||
|
tmp_path / "src/pages/index.astro",
|
||||||
|
"""---
|
||||||
|
import Layout from '../layouts/Layout.astro';
|
||||||
|
import Hero from '../components/Hero.astro';
|
||||||
|
const { title } = Astro.props;
|
||||||
|
---
|
||||||
|
|
||||||
|
<Layout title={title}>
|
||||||
|
<Hero />
|
||||||
|
</Layout>
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
# Sibling files so the resolver lands on real node ids, not phantoms.
|
||||||
|
layout = _write(tmp_path / "src/layouts/Layout.astro", "---\n---\n<slot />\n")
|
||||||
|
hero = _write(tmp_path / "src/components/Hero.astro", "---\n---\n<h1>hi</h1>\n")
|
||||||
|
|
||||||
|
result = extract_astro(page)
|
||||||
|
targets = _import_targets(result, relation="imports_from")
|
||||||
|
assert _make_id(str(layout)) in targets
|
||||||
|
assert _make_id(str(hero)) in targets
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_astro_handles_dynamic_import_in_frontmatter(tmp_path):
|
||||||
|
page = _write(
|
||||||
|
tmp_path / "src/pages/lazy.astro",
|
||||||
|
"""---
|
||||||
|
const Mod = await import('./Other.astro');
|
||||||
|
---
|
||||||
|
|
||||||
|
<div>{Mod.default}</div>
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
other = _write(tmp_path / "src/pages/Other.astro", "---\n---\n<p>o</p>\n")
|
||||||
|
|
||||||
|
result = extract_astro(page)
|
||||||
|
targets = _import_targets(result, relation="dynamic_import")
|
||||||
|
assert _make_id(str(other)) in targets
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_astro_picks_up_client_side_script_imports(tmp_path):
|
||||||
|
page = _write(
|
||||||
|
tmp_path / "src/pages/with-script.astro",
|
||||||
|
"""---
|
||||||
|
import Layout from '../layouts/Layout.astro';
|
||||||
|
---
|
||||||
|
|
||||||
|
<Layout>
|
||||||
|
<button id="b">click</button>
|
||||||
|
</Layout>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
import { hydrate } from '../client/hydrate.ts';
|
||||||
|
hydrate(document.getElementById('b'));
|
||||||
|
</script>
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
layout = _write(tmp_path / "src/layouts/Layout.astro", "---\n---\n<slot />\n")
|
||||||
|
hydrate = _write(tmp_path / "src/client/hydrate.ts", "export function hydrate(){}\n")
|
||||||
|
|
||||||
|
result = extract_astro(page)
|
||||||
|
targets = _import_targets(result, relation="imports_from")
|
||||||
|
assert _make_id(str(layout)) in targets
|
||||||
|
assert _make_id(str(hydrate)) in targets
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_astro_no_frontmatter_does_not_crash(tmp_path):
|
||||||
|
"""Astro permits frontmatter-less files (pure-HTML pages). Must not raise."""
|
||||||
|
page = _write(
|
||||||
|
tmp_path / "src/pages/plain.astro",
|
||||||
|
"<h1>no frontmatter here</h1>\n",
|
||||||
|
)
|
||||||
|
result = extract_astro(page)
|
||||||
|
# Empty/no-imports result is acceptable; the extractor must just not crash.
|
||||||
|
assert isinstance(result, dict)
|
||||||
|
assert _import_targets(result, relation="imports_from") == set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_astro_handles_tsconfig_path_alias(tmp_path):
|
||||||
|
_write(
|
||||||
|
tmp_path / "tsconfig.json",
|
||||||
|
"""{
|
||||||
|
"compilerOptions": {
|
||||||
|
"baseUrl": ".",
|
||||||
|
"paths": { "@components/*": ["src/components/*"] }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
page = _write(
|
||||||
|
tmp_path / "src/pages/alias.astro",
|
||||||
|
"""---
|
||||||
|
import Hero from '@components/Hero.astro';
|
||||||
|
---
|
||||||
|
|
||||||
|
<Hero />
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
hero = _write(tmp_path / "src/components/Hero.astro", "---\n---\n<h1>h</h1>\n")
|
||||||
|
|
||||||
|
result = extract_astro(page)
|
||||||
|
targets = _import_targets(result, relation="imports_from")
|
||||||
|
assert _make_id(str(hero)) in targets
|
||||||
Reference in New Issue
Block a user