fix(cache): scope the incremental checkpoint write too (#1757 followup)

The #1835 fix scoped save_semantic_cache's final CLI write to an
allowed_source_files allowlist, but the per-chunk incremental checkpoint
in llm.py `_checkpoint_chunk` — the write that actually runs on every
`graphify extract`/`update` via extract_corpus_parallel — still called
save_semantic_cache with no allowlist. A chunk whose model result
mis-attributes a node's source_file to another corpus file would merge
that stray fragment into the victim's cache entry (merge_existing=True).

Scope the checkpoint write to the chunk's own dispatched files (FileSlice
-> .rel, bare Path -> the relative source_file). Also hoist the
`import warnings` in cache.py to module level.

Adds an extract_corpus_parallel integration test: a chunk dispatching
only A.py that returns a node attributed to already-cached B.py must
leave B.py's cache entry untouched.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
safishamsi
2026-07-13 00:47:10 +01:00
co-authored by Claude Opus 4.8
parent 579ba1da3c
commit da9616d99e
4 changed files with 61 additions and 5 deletions
+2
View File
@@ -8,6 +8,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu
- Fix: the CLI no longer crashes with exit code 255 when a downstream reader closes the pipe early (#1807 / #1811, thanks @varuntej07). Truncating output with `head`, PowerShell's `Select-Object -First N`, or `sed q` disconnected the reader mid-write, graphify hit an unhandled `BrokenPipeError` (or `OSError(EINVAL)` on Windows) and exited 255 — so CI wrappers and agent harnesses that both trim output and check the exit code read a successful query as a command failure. An early-closing reader is now treated as success: stdout is flushed inside the guard (piped stdout is block-buffered, so a small output would otherwise only flush at interpreter shutdown, where the error escapes as a noisy "Exception ignored" and a nonzero exit), then redirected to devnull so the shutdown flush can't raise again, and the process exits 0.
- Fix: semantic cache writes are now scoped to the files actually dispatched in each extraction batch (#1757 / #1835, thanks @TPAteeq). A model can attribute a node's `source_file` to another corpus file; `save_semantic_cache` would then replace (or, mid-run, pollute) that other file's complete cache entry with a stray fragment — silently, with no shrink guard. Writes now honor an `allowed_source_files` allowlist: the final CLI write is scoped to the uncached file set, and the per-chunk incremental checkpoint (`extract_corpus_parallel`, the default path for `extract`/`update`) is scoped to that chunk's own files, so an out-of-scope attribution is skipped with a warning instead of clobbering a legitimate entry.
- Fix: `graphify export graphml` no longer crashes on dict- or list-valued attributes (#1831 / #1830, thanks @hofmockel). `nx.write_graphml` only accepts scalar values, so a per-node `metadata` dict or the graph-level `hyperedges` list raised `GraphML does not support type <class 'dict'/'list'>` and failed the entire export — on a real ~2,300-node graph, every attempt. `to_graphml` now coerces `None -> ""` and JSON-serializes non-scalars across graph/node/edge scopes (GraphML-native int/float/bool/str pass through unchanged), and writes atomically via a temp file so a failed export no longer leaves a 0-byte `.graphml` that downstream tooling mistakes for a completed one.
- Fix: `.nox/` (nox virtualenvs) is now skipped during detection alongside `.tox/` (#1804, thanks @igorregoir-lgtm). nox is tox's successor and creates a `.nox/` tree of the same shape, but only `.tox` was in the skip set — so a repo with a nox env got its site-packages fully indexed (one real repo came out 91% venv noise: 6,720 of 7,365 nodes from `.nox/`) and semantic extraction burned tokens reading venv docs.
+1 -2
View File
@@ -7,6 +7,7 @@ import json
import os
import re
import tempfile
import warnings
from collections.abc import Iterable
from pathlib import Path
@@ -600,8 +601,6 @@ def save_semantic_cache(
p = resolved_source_path(fpath)
if p.is_file():
if allowed_paths is not None and p not in allowed_paths:
import warnings
warnings.warn(
"semantic cache skipped out-of-scope source_file "
f"{fpath!r}; the file was not dispatched for extraction",
+11 -3
View File
@@ -1903,7 +1903,7 @@ def extract_corpus_parallel(
# over session state. Force serial unless the user explicitly opts in.
if backend == "claude-cli" and os.environ.get("GRAPHIFY_CLAUDE_CLI_PARALLEL", "").strip() != "1":
max_concurrency = 1
def _checkpoint_chunk(result: dict) -> None:
def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None:
# Persist each chunk's semantic results to the cache as soon as it
# completes. Without this, the semantic cache is only written once, at
# the very end of the run (in __main__), so a run interrupted partway
@@ -1914,12 +1914,20 @@ def extract_corpus_parallel(
return
try:
from .cache import save_semantic_cache as _scs
# Scope the write to the files actually dispatched in this chunk
# (#1757). The model can attribute a node's source_file to another
# corpus file; without this bound, that stray node would clobber the
# other file's complete cache entry (or, with merge_existing, pollute
# it). A FileSlice reports its file via `.rel`; a bare Path is the
# relative source_file itself.
allowed = [getattr(item, "rel", None) or item for item in chunk]
_scs(
result.get("nodes", []),
result.get("edges", []),
result.get("hyperedges", []),
root=root,
merge_existing=True,
allowed_source_files=allowed,
)
except Exception as _exc: # noqa: BLE001 — checkpoint is best-effort
print(f"[graphify] incremental cache checkpoint failed: {_exc}", file=sys.stderr)
@@ -1936,7 +1944,7 @@ def extract_corpus_parallel(
continue
assert result is not None
_merge_into(merged, result)
_checkpoint_chunk(result)
_checkpoint_chunk(result, chunk)
if callable(on_chunk_done):
on_chunk_done(idx, total, result)
else:
@@ -1961,7 +1969,7 @@ def extract_corpus_parallel(
continue
assert result is not None
results_by_idx[idx] = result
_checkpoint_chunk(result)
_checkpoint_chunk(result, chunks[idx])
if callable(on_chunk_done):
on_chunk_done(idx, total, result)
for idx in sorted(results_by_idx):
+47
View File
@@ -279,6 +279,53 @@ def test_corpus_parallel_continues_after_chunk_failure(tmp_path, capsys):
assert "failed" in err and "simulated API error" in err
def test_checkpoint_scopes_cache_writes_to_chunk_files(tmp_path):
"""#1757: the per-chunk incremental checkpoint must not let a chunk's
mis-attributed node clobber another corpus file's semantic cache. A chunk
processing only A.py that returns a node attributed to B.py must leave B.py's
existing cache entry untouched. Guards the call site the original fix missed."""
from graphify.llm import extract_corpus_parallel
from graphify.cache import save_semantic_cache, load_cached
a = tmp_path / "A.py"; a.write_text("def a(): pass")
b = tmp_path / "B.py"; b.write_text("def b(): pass")
# Seed B.py's legitimate semantic cache (a full, correct entry).
save_semantic_cache(
[{"id": "b_real", "source_file": "B.py", "file_type": "code"}],
[], [], root=tmp_path,
)
before = load_cached(b, tmp_path, kind="semantic")
assert before and [n["id"] for n in before["nodes"]] == ["b_real"]
# The chunk dispatches only A.py, but the (untrusted) model result attributes
# a stray node to B.py — the #1757 mis-attribution.
def stray(chunk, **kwargs):
return {
"nodes": [
{"id": "a_ok", "source_file": "A.py", "file_type": "code"},
{"id": "b_stray", "source_file": "B.py", "file_type": "code"},
],
"edges": [], "hyperedges": [],
"input_tokens": 1, "output_tokens": 1,
}
with patch("graphify.llm.extract_files_direct", side_effect=stray):
extract_corpus_parallel(
[a], backend="kimi", root=tmp_path,
token_budget=None, chunk_size=1, max_concurrency=1,
)
# B.py's cache is unchanged: the stray node was rejected, not merged in.
after = load_cached(b, tmp_path, kind="semantic")
assert [n["id"] for n in after["nodes"]] == ["b_real"], (
f"B.py cache was clobbered by an out-of-chunk node: {after}"
)
# A.py (the actual chunk file) was legitimately cached.
a_cache = load_cached(a, tmp_path, kind="semantic")
assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"])
def test_corpus_parallel_legacy_mode_when_token_budget_is_none(tmp_path):
"""token_budget=None should fall back to legacy fixed-count chunking."""
from graphify.llm import extract_corpus_parallel