From da9616d99e234087052cadc9918e7e0c17916c00 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Mon, 13 Jul 2026 00:47:10 +0100 Subject: [PATCH] fix(cache): scope the incremental checkpoint write too (#1757 followup) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #1835 fix scoped save_semantic_cache's final CLI write to an allowed_source_files allowlist, but the per-chunk incremental checkpoint in llm.py `_checkpoint_chunk` — the write that actually runs on every `graphify extract`/`update` via extract_corpus_parallel — still called save_semantic_cache with no allowlist. A chunk whose model result mis-attributes a node's source_file to another corpus file would merge that stray fragment into the victim's cache entry (merge_existing=True). Scope the checkpoint write to the chunk's own dispatched files (FileSlice -> .rel, bare Path -> the relative source_file). Also hoist the `import warnings` in cache.py to module level. Adds an extract_corpus_parallel integration test: a chunk dispatching only A.py that returns a node attributed to already-cached B.py must leave B.py's cache entry untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 ++ graphify/cache.py | 3 +-- graphify/llm.py | 14 ++++++++++--- tests/test_chunking.py | 47 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30c2407..e58aa9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: the CLI no longer crashes with exit code 255 when a downstream reader closes the pipe early (#1807 / #1811, thanks @varuntej07). Truncating output with `head`, PowerShell's `Select-Object -First N`, or `sed q` disconnected the reader mid-write, graphify hit an unhandled `BrokenPipeError` (or `OSError(EINVAL)` on Windows) and exited 255 — so CI wrappers and agent harnesses that both trim output and check the exit code read a successful query as a command failure. An early-closing reader is now treated as success: stdout is flushed inside the guard (piped stdout is block-buffered, so a small output would otherwise only flush at interpreter shutdown, where the error escapes as a noisy "Exception ignored" and a nonzero exit), then redirected to devnull so the shutdown flush can't raise again, and the process exits 0. +- Fix: semantic cache writes are now scoped to the files actually dispatched in each extraction batch (#1757 / #1835, thanks @TPAteeq). A model can attribute a node's `source_file` to another corpus file; `save_semantic_cache` would then replace (or, mid-run, pollute) that other file's complete cache entry with a stray fragment — silently, with no shrink guard. Writes now honor an `allowed_source_files` allowlist: the final CLI write is scoped to the uncached file set, and the per-chunk incremental checkpoint (`extract_corpus_parallel`, the default path for `extract`/`update`) is scoped to that chunk's own files, so an out-of-scope attribution is skipped with a warning instead of clobbering a legitimate entry. + - Fix: `graphify export graphml` no longer crashes on dict- or list-valued attributes (#1831 / #1830, thanks @hofmockel). `nx.write_graphml` only accepts scalar values, so a per-node `metadata` dict or the graph-level `hyperedges` list raised `GraphML does not support type ` and failed the entire export — on a real ~2,300-node graph, every attempt. `to_graphml` now coerces `None -> ""` and JSON-serializes non-scalars across graph/node/edge scopes (GraphML-native int/float/bool/str pass through unchanged), and writes atomically via a temp file so a failed export no longer leaves a 0-byte `.graphml` that downstream tooling mistakes for a completed one. - Fix: `.nox/` (nox virtualenvs) is now skipped during detection alongside `.tox/` (#1804, thanks @igorregoir-lgtm). nox is tox's successor and creates a `.nox/` tree of the same shape, but only `.tox` was in the skip set — so a repo with a nox env got its site-packages fully indexed (one real repo came out 91% venv noise: 6,720 of 7,365 nodes from `.nox/`) and semantic extraction burned tokens reading venv docs. diff --git a/graphify/cache.py b/graphify/cache.py index 34928a4..3970540 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -7,6 +7,7 @@ import json import os import re import tempfile +import warnings from collections.abc import Iterable from pathlib import Path @@ -600,8 +601,6 @@ def save_semantic_cache( p = resolved_source_path(fpath) if p.is_file(): if allowed_paths is not None and p not in allowed_paths: - import warnings - warnings.warn( "semantic cache skipped out-of-scope source_file " f"{fpath!r}; the file was not dispatched for extraction", diff --git a/graphify/llm.py b/graphify/llm.py index 560bbcb..23b3020 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -1903,7 +1903,7 @@ def extract_corpus_parallel( # over session state. Force serial unless the user explicitly opts in. if backend == "claude-cli" and os.environ.get("GRAPHIFY_CLAUDE_CLI_PARALLEL", "").strip() != "1": max_concurrency = 1 - def _checkpoint_chunk(result: dict) -> None: + def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # Persist each chunk's semantic results to the cache as soon as it # completes. Without this, the semantic cache is only written once, at # the very end of the run (in __main__), so a run interrupted partway @@ -1914,12 +1914,20 @@ def extract_corpus_parallel( return try: from .cache import save_semantic_cache as _scs + # Scope the write to the files actually dispatched in this chunk + # (#1757). The model can attribute a node's source_file to another + # corpus file; without this bound, that stray node would clobber the + # other file's complete cache entry (or, with merge_existing, pollute + # it). A FileSlice reports its file via `.rel`; a bare Path is the + # relative source_file itself. + allowed = [getattr(item, "rel", None) or item for item in chunk] _scs( result.get("nodes", []), result.get("edges", []), result.get("hyperedges", []), root=root, merge_existing=True, + allowed_source_files=allowed, ) except Exception as _exc: # noqa: BLE001 — checkpoint is best-effort print(f"[graphify] incremental cache checkpoint failed: {_exc}", file=sys.stderr) @@ -1936,7 +1944,7 @@ def extract_corpus_parallel( continue assert result is not None _merge_into(merged, result) - _checkpoint_chunk(result) + _checkpoint_chunk(result, chunk) if callable(on_chunk_done): on_chunk_done(idx, total, result) else: @@ -1961,7 +1969,7 @@ def extract_corpus_parallel( continue assert result is not None results_by_idx[idx] = result - _checkpoint_chunk(result) + _checkpoint_chunk(result, chunks[idx]) if callable(on_chunk_done): on_chunk_done(idx, total, result) for idx in sorted(results_by_idx): diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 2706dfd..b39262a 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -279,6 +279,53 @@ def test_corpus_parallel_continues_after_chunk_failure(tmp_path, capsys): assert "failed" in err and "simulated API error" in err +def test_checkpoint_scopes_cache_writes_to_chunk_files(tmp_path): + """#1757: the per-chunk incremental checkpoint must not let a chunk's + mis-attributed node clobber another corpus file's semantic cache. A chunk + processing only A.py that returns a node attributed to B.py must leave B.py's + existing cache entry untouched. Guards the call site the original fix missed.""" + from graphify.llm import extract_corpus_parallel + from graphify.cache import save_semantic_cache, load_cached + + a = tmp_path / "A.py"; a.write_text("def a(): pass") + b = tmp_path / "B.py"; b.write_text("def b(): pass") + + # Seed B.py's legitimate semantic cache (a full, correct entry). + save_semantic_cache( + [{"id": "b_real", "source_file": "B.py", "file_type": "code"}], + [], [], root=tmp_path, + ) + before = load_cached(b, tmp_path, kind="semantic") + assert before and [n["id"] for n in before["nodes"]] == ["b_real"] + + # The chunk dispatches only A.py, but the (untrusted) model result attributes + # a stray node to B.py — the #1757 mis-attribution. + def stray(chunk, **kwargs): + return { + "nodes": [ + {"id": "a_ok", "source_file": "A.py", "file_type": "code"}, + {"id": "b_stray", "source_file": "B.py", "file_type": "code"}, + ], + "edges": [], "hyperedges": [], + "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=stray): + extract_corpus_parallel( + [a], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + # B.py's cache is unchanged: the stray node was rejected, not merged in. + after = load_cached(b, tmp_path, kind="semantic") + assert [n["id"] for n in after["nodes"]] == ["b_real"], ( + f"B.py cache was clobbered by an out-of-chunk node: {after}" + ) + # A.py (the actual chunk file) was legitimately cached. + a_cache = load_cached(a, tmp_path, kind="semantic") + assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"]) + + def test_corpus_parallel_legacy_mode_when_token_budget_is_none(tmp_path): """token_budget=None should fall back to legacy fixed-count chunking.""" from graphify.llm import extract_corpus_parallel