fix(detect): forward follow_symlinks from detect_incremental to detect

`detect_incremental(root)` always called `detect(root)` without forwarding
the `follow_symlinks` kwarg. As a result, corpora that include symlinked
sub-trees pointing to directories outside the scan root (e.g. a
`state_of_truth/` symlink pointing at `~/.hermes/state_of_truth/`) were
visible to a full `detect()` run with `follow_symlinks=True` but invisible
to any subsequent `--update` run. The incremental scan would then either
report no changes (silently dropping legitimate new files) or repeatedly
re-extract a phantom subset, depending on what was reachable without
crossing symlinks.

Add a keyword-only `follow_symlinks` parameter to `detect_incremental()`
and forward it. Default stays `False` for backwards compatibility — only
callers that already opt in to symlink following on `detect()` pick up
the new behaviour for incremental runs too.

Test: a corpus with a symlinked directory is invisible with
`follow_symlinks=False`, fully indexed with `follow_symlinks=True`, and
correctly reports zero new files on a second incremental scan after the
manifest is saved.
This commit is contained in:
Alpha Nury
2026-05-05 15:25:08 +02:00
parent ee85bbfbfd
commit 64585cf889
3 changed files with 44 additions and 3 deletions
+4
View File
@@ -2,6 +2,10 @@
Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases)
## Unreleased
- Fix: `detect_incremental()` now accepts and forwards `follow_symlinks` to `detect()`. Without this, `--update` runs silently miss any files reached through a symlinked sub-tree (e.g. `state_of_truth/` symlinking to a directory outside the corpus root), even when the original full run had detected them. Previously the flag was on `detect()` and `collect_files()` only.
## 0.7.5 (2026-05-04)
- Feat: `graphify extract` now runs incrementally - auto-detects prior `manifest.json` and re-extracts only changed/new files; semantic results cached by content hash so unchanged docs cost zero LLM tokens on repeat runs (#698)
+12 -2
View File
@@ -771,7 +771,12 @@ def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PA
Path(manifest_path).write_text(json.dumps(manifest, indent=2), encoding="utf-8")
def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict:
def detect_incremental(
root: Path,
manifest_path: str = _MANIFEST_PATH,
*,
follow_symlinks: bool = False,
) -> dict:
"""Like detect(), but returns only new or modified files since the last run.
Fast path: mtime unchanged → unchanged (free, no hash).
@@ -779,8 +784,13 @@ def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict:
treat as unchanged. Different hash = actually changed, re-extract.
Backwards compatible with legacy manifests storing plain float mtime values.
The ``follow_symlinks`` flag is forwarded to :func:`detect` so corpora that
rely on symlinked sub-trees (e.g. a ``state_of_truth/`` symlink pointing to a
directory outside the scan root) are scanned consistently between full and
incremental runs.
"""
full = detect(root)
full = detect(root, follow_symlinks=follow_symlinks)
manifest = load_manifest(manifest_path)
if not manifest:
+28 -1
View File
@@ -1,5 +1,5 @@
from pathlib import Path
from graphify.detect import classify_file, count_words, detect, FileType, _looks_like_paper, _is_ignored, _load_graphifyignore
from graphify.detect import classify_file, count_words, detect, detect_incremental, save_manifest, FileType, _looks_like_paper, _is_ignored, _load_graphifyignore
FIXTURES = Path(__file__).parent / "fixtures"
@@ -220,6 +220,33 @@ def test_detect_handles_circular_symlinks(tmp_path):
assert any("main.py" in f for f in result["files"]["code"])
def test_detect_incremental_propagates_follow_symlinks(tmp_path, monkeypatch):
"""detect_incremental must forward follow_symlinks so symlinked sub-trees
appear in incremental scans the same way they appear in full scans."""
monkeypatch.chdir(tmp_path)
real_dir = tmp_path / "real_corpus"
real_dir.mkdir()
(real_dir / "note.md").write_text("# real note\n\nsome content")
(tmp_path / "linked_corpus").symlink_to(real_dir)
manifest_path = str(tmp_path / "manifest.json")
# Without following symlinks, the symlinked dir contents are invisible.
no_link = detect_incremental(tmp_path, manifest_path, follow_symlinks=False)
assert not any("linked_corpus" in f for f in no_link["files"]["document"])
# With follow_symlinks=True, the symlinked dir contents appear and are new.
yes_link = detect_incremental(tmp_path, manifest_path, follow_symlinks=True)
assert any("linked_corpus" in f for f in yes_link["files"]["document"])
assert yes_link["new_total"] >= 2 # real + linked
# After saving manifest, a second incremental scan should see no changes.
save_manifest(yes_link["files"], manifest_path)
second = detect_incremental(tmp_path, manifest_path, follow_symlinks=True)
assert second["new_total"] == 0
def test_classify_video_extensions():
"""Video and audio file extensions should classify as VIDEO."""
from graphify.detect import FileType