fix(cache): scope the incremental checkpoint write too (#1757 followup)
The #1835 fix scoped save_semantic_cache's final CLI write to an allowed_source_files allowlist, but the per-chunk incremental checkpoint in llm.py `_checkpoint_chunk` — the write that actually runs on every `graphify extract`/`update` via extract_corpus_parallel — still called save_semantic_cache with no allowlist. A chunk whose model result mis-attributes a node's source_file to another corpus file would merge that stray fragment into the victim's cache entry (merge_existing=True). Scope the checkpoint write to the chunk's own dispatched files (FileSlice -> .rel, bare Path -> the relative source_file). Also hoist the `import warnings` in cache.py to module level. Adds an extract_corpus_parallel integration test: a chunk dispatching only A.py that returns a node attributed to already-cached B.py must leave B.py's cache entry untouched. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
579ba1da3c
commit
da9616d99e
@@ -8,6 +8,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu
|
||||
|
||||
- Fix: the CLI no longer crashes with exit code 255 when a downstream reader closes the pipe early (#1807 / #1811, thanks @varuntej07). Truncating output with `head`, PowerShell's `Select-Object -First N`, or `sed q` disconnected the reader mid-write, graphify hit an unhandled `BrokenPipeError` (or `OSError(EINVAL)` on Windows) and exited 255 — so CI wrappers and agent harnesses that both trim output and check the exit code read a successful query as a command failure. An early-closing reader is now treated as success: stdout is flushed inside the guard (piped stdout is block-buffered, so a small output would otherwise only flush at interpreter shutdown, where the error escapes as a noisy "Exception ignored" and a nonzero exit), then redirected to devnull so the shutdown flush can't raise again, and the process exits 0.
|
||||
|
||||
- Fix: semantic cache writes are now scoped to the files actually dispatched in each extraction batch (#1757 / #1835, thanks @TPAteeq). A model can attribute a node's `source_file` to another corpus file; `save_semantic_cache` would then replace (or, mid-run, pollute) that other file's complete cache entry with a stray fragment — silently, with no shrink guard. Writes now honor an `allowed_source_files` allowlist: the final CLI write is scoped to the uncached file set, and the per-chunk incremental checkpoint (`extract_corpus_parallel`, the default path for `extract`/`update`) is scoped to that chunk's own files, so an out-of-scope attribution is skipped with a warning instead of clobbering a legitimate entry.
|
||||
|
||||
- Fix: `graphify export graphml` no longer crashes on dict- or list-valued attributes (#1831 / #1830, thanks @hofmockel). `nx.write_graphml` only accepts scalar values, so a per-node `metadata` dict or the graph-level `hyperedges` list raised `GraphML does not support type <class 'dict'/'list'>` and failed the entire export — on a real ~2,300-node graph, every attempt. `to_graphml` now coerces `None -> ""` and JSON-serializes non-scalars across graph/node/edge scopes (GraphML-native int/float/bool/str pass through unchanged), and writes atomically via a temp file so a failed export no longer leaves a 0-byte `.graphml` that downstream tooling mistakes for a completed one.
|
||||
|
||||
- Fix: `.nox/` (nox virtualenvs) is now skipped during detection alongside `.tox/` (#1804, thanks @igorregoir-lgtm). nox is tox's successor and creates a `.nox/` tree of the same shape, but only `.tox` was in the skip set — so a repo with a nox env got its site-packages fully indexed (one real repo came out 91% venv noise: 6,720 of 7,365 nodes from `.nox/`) and semantic extraction burned tokens reading venv docs.
|
||||
|
||||
+1
-2
@@ -7,6 +7,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import warnings
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
@@ -600,8 +601,6 @@ def save_semantic_cache(
|
||||
p = resolved_source_path(fpath)
|
||||
if p.is_file():
|
||||
if allowed_paths is not None and p not in allowed_paths:
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"semantic cache skipped out-of-scope source_file "
|
||||
f"{fpath!r}; the file was not dispatched for extraction",
|
||||
|
||||
+11
-3
@@ -1903,7 +1903,7 @@ def extract_corpus_parallel(
|
||||
# over session state. Force serial unless the user explicitly opts in.
|
||||
if backend == "claude-cli" and os.environ.get("GRAPHIFY_CLAUDE_CLI_PARALLEL", "").strip() != "1":
|
||||
max_concurrency = 1
|
||||
def _checkpoint_chunk(result: dict) -> None:
|
||||
def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None:
|
||||
# Persist each chunk's semantic results to the cache as soon as it
|
||||
# completes. Without this, the semantic cache is only written once, at
|
||||
# the very end of the run (in __main__), so a run interrupted partway
|
||||
@@ -1914,12 +1914,20 @@ def extract_corpus_parallel(
|
||||
return
|
||||
try:
|
||||
from .cache import save_semantic_cache as _scs
|
||||
# Scope the write to the files actually dispatched in this chunk
|
||||
# (#1757). The model can attribute a node's source_file to another
|
||||
# corpus file; without this bound, that stray node would clobber the
|
||||
# other file's complete cache entry (or, with merge_existing, pollute
|
||||
# it). A FileSlice reports its file via `.rel`; a bare Path is the
|
||||
# relative source_file itself.
|
||||
allowed = [getattr(item, "rel", None) or item for item in chunk]
|
||||
_scs(
|
||||
result.get("nodes", []),
|
||||
result.get("edges", []),
|
||||
result.get("hyperedges", []),
|
||||
root=root,
|
||||
merge_existing=True,
|
||||
allowed_source_files=allowed,
|
||||
)
|
||||
except Exception as _exc: # noqa: BLE001 — checkpoint is best-effort
|
||||
print(f"[graphify] incremental cache checkpoint failed: {_exc}", file=sys.stderr)
|
||||
@@ -1936,7 +1944,7 @@ def extract_corpus_parallel(
|
||||
continue
|
||||
assert result is not None
|
||||
_merge_into(merged, result)
|
||||
_checkpoint_chunk(result)
|
||||
_checkpoint_chunk(result, chunk)
|
||||
if callable(on_chunk_done):
|
||||
on_chunk_done(idx, total, result)
|
||||
else:
|
||||
@@ -1961,7 +1969,7 @@ def extract_corpus_parallel(
|
||||
continue
|
||||
assert result is not None
|
||||
results_by_idx[idx] = result
|
||||
_checkpoint_chunk(result)
|
||||
_checkpoint_chunk(result, chunks[idx])
|
||||
if callable(on_chunk_done):
|
||||
on_chunk_done(idx, total, result)
|
||||
for idx in sorted(results_by_idx):
|
||||
|
||||
@@ -279,6 +279,53 @@ def test_corpus_parallel_continues_after_chunk_failure(tmp_path, capsys):
|
||||
assert "failed" in err and "simulated API error" in err
|
||||
|
||||
|
||||
def test_checkpoint_scopes_cache_writes_to_chunk_files(tmp_path):
|
||||
"""#1757: the per-chunk incremental checkpoint must not let a chunk's
|
||||
mis-attributed node clobber another corpus file's semantic cache. A chunk
|
||||
processing only A.py that returns a node attributed to B.py must leave B.py's
|
||||
existing cache entry untouched. Guards the call site the original fix missed."""
|
||||
from graphify.llm import extract_corpus_parallel
|
||||
from graphify.cache import save_semantic_cache, load_cached
|
||||
|
||||
a = tmp_path / "A.py"; a.write_text("def a(): pass")
|
||||
b = tmp_path / "B.py"; b.write_text("def b(): pass")
|
||||
|
||||
# Seed B.py's legitimate semantic cache (a full, correct entry).
|
||||
save_semantic_cache(
|
||||
[{"id": "b_real", "source_file": "B.py", "file_type": "code"}],
|
||||
[], [], root=tmp_path,
|
||||
)
|
||||
before = load_cached(b, tmp_path, kind="semantic")
|
||||
assert before and [n["id"] for n in before["nodes"]] == ["b_real"]
|
||||
|
||||
# The chunk dispatches only A.py, but the (untrusted) model result attributes
|
||||
# a stray node to B.py — the #1757 mis-attribution.
|
||||
def stray(chunk, **kwargs):
|
||||
return {
|
||||
"nodes": [
|
||||
{"id": "a_ok", "source_file": "A.py", "file_type": "code"},
|
||||
{"id": "b_stray", "source_file": "B.py", "file_type": "code"},
|
||||
],
|
||||
"edges": [], "hyperedges": [],
|
||||
"input_tokens": 1, "output_tokens": 1,
|
||||
}
|
||||
|
||||
with patch("graphify.llm.extract_files_direct", side_effect=stray):
|
||||
extract_corpus_parallel(
|
||||
[a], backend="kimi", root=tmp_path,
|
||||
token_budget=None, chunk_size=1, max_concurrency=1,
|
||||
)
|
||||
|
||||
# B.py's cache is unchanged: the stray node was rejected, not merged in.
|
||||
after = load_cached(b, tmp_path, kind="semantic")
|
||||
assert [n["id"] for n in after["nodes"]] == ["b_real"], (
|
||||
f"B.py cache was clobbered by an out-of-chunk node: {after}"
|
||||
)
|
||||
# A.py (the actual chunk file) was legitimately cached.
|
||||
a_cache = load_cached(a, tmp_path, kind="semantic")
|
||||
assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"])
|
||||
|
||||
|
||||
def test_corpus_parallel_legacy_mode_when_token_budget_is_none(tmp_path):
|
||||
"""token_budget=None should fall back to legacy fixed-count chunking."""
|
||||
from graphify.llm import extract_corpus_parallel
|
||||
|
||||
Reference in New Issue
Block a user