feat: 13-language AST support and token benchmark
Java, C, C++, Ruby, C#, Kotlin, Scala, PHP via tree-sitter (13 total) benchmark.py measures BFS subgraph tokens vs corpus tokens 5 skill bug fixes (cohesion crash, dead step, missing MCP tool)
This commit is contained in:
Safi
@@ -202,6 +202,7 @@ graphify/
|
||||
├── export.py Obsidian vault, graph.json, graph.html, graph.svg, Neo4j Cypher, Canvas
|
||||
├── ingest.py fetch URLs (arXiv, Twitter/X, PDF, any webpage); save Q&A to .graphify/memory/
|
||||
├── cache.py SHA256-based per-file extraction cache; check_semantic_cache / save_semantic_cache
|
||||
├── security.py URL validation (http/https only), safe fetch with size cap, path guards, label sanitisation
|
||||
├── validate.py JSON schema checks on extraction output
|
||||
├── serve.py MCP stdio server — query_graph, get_node, get_neighbors, shortest_path, god_nodes
|
||||
└── watch.py fs watcher, writes flag file when new files appear
|
||||
@@ -209,6 +210,7 @@ graphify/
|
||||
skills/graphify/
|
||||
└── skill.md the Claude Code skill — the full pipeline the agent runs step by step
|
||||
|
||||
tests/ 142 tests, one file per module
|
||||
SECURITY.md threat model, mitigations, vulnerability reporting
|
||||
tests/ 163 tests, one file per module
|
||||
pyproject.toml pip install graphify | pip install graphify[mcp,neo4j,pdf,watch]
|
||||
```
|
||||
|
||||
+17
-1
@@ -1,5 +1,6 @@
|
||||
"""graphify CLI — `graphify install` sets up the Claude Code skill."""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -56,13 +57,28 @@ def main() -> None:
|
||||
print("Usage: graphify <command>")
|
||||
print()
|
||||
print("Commands:")
|
||||
print(" install copy skill to ~/.claude/skills/ and register in CLAUDE.md")
|
||||
print(" install copy skill to ~/.claude/skills/ and register in CLAUDE.md")
|
||||
print(" benchmark [graph.json] measure token reduction vs naive full-corpus approach")
|
||||
print()
|
||||
return
|
||||
|
||||
cmd = sys.argv[1]
|
||||
if cmd == "install":
|
||||
install()
|
||||
elif cmd == "benchmark":
|
||||
from graphify.benchmark import run_benchmark, print_benchmark
|
||||
graph_path = sys.argv[2] if len(sys.argv) > 2 else ".graphify/graph.json"
|
||||
# Try to load corpus_words from detect output
|
||||
corpus_words = None
|
||||
detect_path = Path(".graphify_detect.json")
|
||||
if detect_path.exists():
|
||||
try:
|
||||
detect_data = json.loads(detect_path.read_text())
|
||||
corpus_words = detect_data.get("total_words")
|
||||
except Exception:
|
||||
pass
|
||||
result = run_benchmark(graph_path, corpus_words=corpus_words)
|
||||
print_benchmark(result)
|
||||
else:
|
||||
print(f"error: unknown command '{cmd}'", file=sys.stderr)
|
||||
print("Run 'graphify --help' for usage.", file=sys.stderr)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Token-reduction benchmark — measures how much context graphify saves vs naive full-corpus approach."""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
from pathlib import Path
|
||||
import networkx as nx
|
||||
from networkx.readwrite import json_graph
|
||||
|
||||
|
||||
_CHARS_PER_TOKEN = 4 # standard approximation
|
||||
|
||||
|
||||
def _estimate_tokens(text: str) -> int:
|
||||
return max(1, len(text) // _CHARS_PER_TOKEN)
|
||||
|
||||
|
||||
def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int:
|
||||
"""Run BFS from best-matching nodes and return estimated tokens in the subgraph context."""
|
||||
terms = [t.lower() for t in question.split() if len(t) > 2]
|
||||
scored = []
|
||||
for nid, data in G.nodes(data=True):
|
||||
label = data.get("label", "").lower()
|
||||
score = sum(1 for t in terms if t in label)
|
||||
if score > 0:
|
||||
scored.append((score, nid))
|
||||
scored.sort(reverse=True)
|
||||
start_nodes = [nid for _, nid in scored[:3]]
|
||||
if not start_nodes:
|
||||
return 0
|
||||
|
||||
visited: set[str] = set(start_nodes)
|
||||
frontier = set(start_nodes)
|
||||
edges_seen: list[tuple] = []
|
||||
for _ in range(depth):
|
||||
next_frontier: set[str] = set()
|
||||
for n in frontier:
|
||||
for neighbor in G.neighbors(n):
|
||||
if neighbor not in visited:
|
||||
next_frontier.add(neighbor)
|
||||
edges_seen.append((n, neighbor))
|
||||
visited.update(next_frontier)
|
||||
frontier = next_frontier
|
||||
|
||||
lines = []
|
||||
for nid in visited:
|
||||
d = G.nodes[nid]
|
||||
lines.append(f"NODE {d.get('label', nid)} src={d.get('source_file', '')} loc={d.get('source_location', '')}")
|
||||
for u, v in edges_seen:
|
||||
if u in visited and v in visited:
|
||||
d = G.edges[u, v]
|
||||
lines.append(f"EDGE {G.nodes[u].get('label', u)} --{d.get('relation', '')}--> {G.nodes[v].get('label', v)}")
|
||||
|
||||
return _estimate_tokens("\n".join(lines))
|
||||
|
||||
|
||||
_SAMPLE_QUESTIONS = [
|
||||
"how does authentication work",
|
||||
"what is the main entry point",
|
||||
"how are errors handled",
|
||||
"what connects the data layer to the api",
|
||||
"what are the core abstractions",
|
||||
]
|
||||
|
||||
|
||||
def run_benchmark(
|
||||
graph_path: str = ".graphify/graph.json",
|
||||
corpus_words: int | None = None,
|
||||
questions: list[str] | None = None,
|
||||
) -> dict:
|
||||
"""Measure token reduction: corpus tokens vs graphify query tokens.
|
||||
|
||||
Args:
|
||||
graph_path: path to the built graph
|
||||
corpus_words: total word count from detect() output; if None, estimated from graph
|
||||
questions: list of questions to benchmark; defaults to _SAMPLE_QUESTIONS
|
||||
|
||||
Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question
|
||||
"""
|
||||
data = json.loads(Path(graph_path).read_text())
|
||||
G = json_graph.node_link_graph(data, edges="links")
|
||||
|
||||
if corpus_words is None:
|
||||
# Rough estimate: each node label is ~3 words, plus source context
|
||||
corpus_words = G.number_of_nodes() * 50
|
||||
|
||||
corpus_tokens = corpus_words * 100 // 75 # words → tokens (100 words ≈ 133 tokens)
|
||||
|
||||
qs = questions or _SAMPLE_QUESTIONS
|
||||
per_question = []
|
||||
for q in qs:
|
||||
qt = _query_subgraph_tokens(G, q)
|
||||
if qt > 0:
|
||||
per_question.append({"question": q, "query_tokens": qt, "reduction": round(corpus_tokens / qt, 1)})
|
||||
|
||||
if not per_question:
|
||||
return {"error": "No matching nodes found for sample questions. Build the graph first."}
|
||||
|
||||
avg_query_tokens = sum(p["query_tokens"] for p in per_question) // len(per_question)
|
||||
reduction_ratio = round(corpus_tokens / avg_query_tokens, 1) if avg_query_tokens > 0 else 0
|
||||
|
||||
return {
|
||||
"corpus_tokens": corpus_tokens,
|
||||
"corpus_words": corpus_words,
|
||||
"nodes": G.number_of_nodes(),
|
||||
"edges": G.number_of_edges(),
|
||||
"avg_query_tokens": avg_query_tokens,
|
||||
"reduction_ratio": reduction_ratio,
|
||||
"per_question": per_question,
|
||||
}
|
||||
|
||||
|
||||
def print_benchmark(result: dict) -> None:
|
||||
"""Print a human-readable benchmark report."""
|
||||
if "error" in result:
|
||||
print(f"Benchmark error: {result['error']}")
|
||||
return
|
||||
|
||||
print(f"\ngraphify token reduction benchmark")
|
||||
print(f"{'─' * 50}")
|
||||
print(f" Corpus: {result['corpus_words']:,} words → ~{result['corpus_tokens']:,} tokens (naive)")
|
||||
print(f" Graph: {result['nodes']:,} nodes, {result['edges']:,} edges")
|
||||
print(f" Avg query cost: ~{result['avg_query_tokens']:,} tokens")
|
||||
print(f" Reduction: {result['reduction_ratio']}x fewer tokens per query")
|
||||
print(f"\n Per question:")
|
||||
for p in result["per_question"]:
|
||||
print(f" [{p['reduction']}x] {p['question'][:55]}")
|
||||
print()
|
||||
+1
-1
@@ -15,7 +15,7 @@ class FileType(str, Enum):
|
||||
|
||||
_MANIFEST_PATH = ".graphify/manifest.json"
|
||||
|
||||
CODE_EXTENSIONS = {'.py', '.ts', '.js', '.go', '.rs', '.java', '.cpp', '.c', '.rb', '.swift', '.kt'}
|
||||
CODE_EXTENSIONS = {'.py', '.ts', '.js', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php'}
|
||||
DOC_EXTENSIONS = {'.md', '.txt', '.rst'}
|
||||
PAPER_EXTENSIONS = {'.pdf'}
|
||||
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
|
||||
|
||||
+1445
-1
File diff suppressed because it is too large
Load Diff
+15
-25
@@ -92,6 +92,8 @@ Then act on it:
|
||||
|
||||
### Step 3 — Extract entities and relationships
|
||||
|
||||
**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation — do not lose it.
|
||||
|
||||
This step has two parts: **structural extraction** (deterministic, free) then **semantic extraction** (Claude, costs tokens).
|
||||
|
||||
#### Part A — Structural extraction for code files
|
||||
@@ -496,28 +498,9 @@ print('graph.svg written — embeds in Obsidian, Notion, GitHub READMEs')
|
||||
"
|
||||
```
|
||||
|
||||
### Step 7c — Obsidian export (only if --obsidian flag)
|
||||
### Step 7c — SVG export already covered in Step 7b above
|
||||
|
||||
```bash
|
||||
python3 -c "
|
||||
import sys, json
|
||||
from graphify.build import build_from_json
|
||||
from graphify.export import to_obsidian
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text())
|
||||
labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {}
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
labels = {int(k): v for k, v in labels_raw.items()}
|
||||
|
||||
n = to_obsidian(G, communities, '.graphify/obsidian', community_labels=labels or None, cohesion=cohesion)
|
||||
print(f'Obsidian vault written: {n} notes in .graphify/obsidian/')
|
||||
print('Open .graphify/obsidian/ as a vault in Obsidian to explore the graph.')
|
||||
"
|
||||
```
|
||||
_(No separate --obsidian flag — Obsidian vault is always generated in Step 6 by default.)_
|
||||
|
||||
### Step 7d — MCP server (only if --mcp flag)
|
||||
|
||||
@@ -525,7 +508,7 @@ print('Open .graphify/obsidian/ as a vault in Obsidian to explore the graph.')
|
||||
python3 -m graphify.serve .graphify/graph.json
|
||||
```
|
||||
|
||||
This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live.
|
||||
This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live.
|
||||
|
||||
To configure in Claude Desktop, add to `claude_desktop_config.json`:
|
||||
```json
|
||||
@@ -1007,12 +990,19 @@ import sys
|
||||
from graphify.ingest import ingest
|
||||
from pathlib import Path
|
||||
|
||||
out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR')
|
||||
print(f'Saved to {out}')
|
||||
try:
|
||||
out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR')
|
||||
print(f'Saved to {out}')
|
||||
except ValueError as e:
|
||||
print(f'error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except RuntimeError as e:
|
||||
print(f'error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
"
|
||||
```
|
||||
|
||||
Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After saving, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.
|
||||
Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong — do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.
|
||||
|
||||
Supported URL types (auto-detected):
|
||||
- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
|
||||
_WATCHED_EXTENSIONS = {
|
||||
".py", ".ts", ".js", ".go", ".rs", ".java", ".cpp", ".c", ".rb", ".swift", ".kt",
|
||||
".cs", ".scala", ".php", ".cc", ".cxx", ".hpp", ".h", ".kts",
|
||||
".md", ".txt", ".rst", ".pdf",
|
||||
".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg",
|
||||
}
|
||||
|
||||
@@ -19,6 +19,14 @@ dependencies = [
|
||||
"tree-sitter-typescript",
|
||||
"tree-sitter-go",
|
||||
"tree-sitter-rust",
|
||||
"tree-sitter-java",
|
||||
"tree-sitter-c",
|
||||
"tree-sitter-cpp",
|
||||
"tree-sitter-ruby",
|
||||
"tree-sitter-c-sharp",
|
||||
"tree-sitter-kotlin",
|
||||
"tree-sitter-scala",
|
||||
"tree-sitter-php",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
+15
-25
@@ -92,6 +92,8 @@ Then act on it:
|
||||
|
||||
### Step 3 — Extract entities and relationships
|
||||
|
||||
**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation — do not lose it.
|
||||
|
||||
This step has two parts: **structural extraction** (deterministic, free) then **semantic extraction** (Claude, costs tokens).
|
||||
|
||||
#### Part A — Structural extraction for code files
|
||||
@@ -496,28 +498,9 @@ print('graph.svg written — embeds in Obsidian, Notion, GitHub READMEs')
|
||||
"
|
||||
```
|
||||
|
||||
### Step 7c — Obsidian export (only if --obsidian flag)
|
||||
### Step 7c — SVG export already covered in Step 7b above
|
||||
|
||||
```bash
|
||||
python3 -c "
|
||||
import sys, json
|
||||
from graphify.build import build_from_json
|
||||
from graphify.export import to_obsidian
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text())
|
||||
labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {}
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
labels = {int(k): v for k, v in labels_raw.items()}
|
||||
|
||||
n = to_obsidian(G, communities, '.graphify/obsidian', community_labels=labels or None, cohesion=cohesion)
|
||||
print(f'Obsidian vault written: {n} notes in .graphify/obsidian/')
|
||||
print('Open .graphify/obsidian/ as a vault in Obsidian to explore the graph.')
|
||||
"
|
||||
```
|
||||
_(No separate --obsidian flag — Obsidian vault is always generated in Step 6 by default.)_
|
||||
|
||||
### Step 7d — MCP server (only if --mcp flag)
|
||||
|
||||
@@ -525,7 +508,7 @@ print('Open .graphify/obsidian/ as a vault in Obsidian to explore the graph.')
|
||||
python3 -m graphify.serve .graphify/graph.json
|
||||
```
|
||||
|
||||
This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live.
|
||||
This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live.
|
||||
|
||||
To configure in Claude Desktop, add to `claude_desktop_config.json`:
|
||||
```json
|
||||
@@ -1007,12 +990,19 @@ import sys
|
||||
from graphify.ingest import ingest
|
||||
from pathlib import Path
|
||||
|
||||
out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR')
|
||||
print(f'Saved to {out}')
|
||||
try:
|
||||
out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR')
|
||||
print(f'Saved to {out}')
|
||||
except ValueError as e:
|
||||
print(f'error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except RuntimeError as e:
|
||||
print(f'error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
"
|
||||
```
|
||||
|
||||
Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After saving, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.
|
||||
Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong — do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.
|
||||
|
||||
Supported URL types (auto-detected):
|
||||
- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author
|
||||
|
||||
Vendored
+27
@@ -0,0 +1,27 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define MAX_SIZE 256
|
||||
|
||||
static int validate(const char *input) {
|
||||
return input != NULL && strlen(input) > 0;
|
||||
}
|
||||
|
||||
char *process(const char *input) {
|
||||
if (!validate(input)) {
|
||||
return NULL;
|
||||
}
|
||||
char *result = malloc(MAX_SIZE);
|
||||
strncpy(result, input, MAX_SIZE - 1);
|
||||
return result;
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
char *out = process("hello");
|
||||
if (out) {
|
||||
printf("%s\n", out);
|
||||
free(out);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
Vendored
+30
@@ -0,0 +1,30 @@
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class HttpClient {
|
||||
public:
|
||||
HttpClient(const std::string& baseUrl) : baseUrl_(baseUrl) {}
|
||||
|
||||
std::string get(const std::string& path) {
|
||||
return buildRequest("GET", path);
|
||||
}
|
||||
|
||||
std::string post(const std::string& path, const std::string& body) {
|
||||
return buildRequest("POST", path);
|
||||
}
|
||||
|
||||
private:
|
||||
std::string baseUrl_;
|
||||
|
||||
std::string buildRequest(const std::string& method, const std::string& path) {
|
||||
return method + " " + baseUrl_ + path;
|
||||
}
|
||||
};
|
||||
|
||||
int main() {
|
||||
HttpClient client("https://api.example.com");
|
||||
std::string response = client.get("/users");
|
||||
std::cout << response << std::endl;
|
||||
return 0;
|
||||
}
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
|
||||
namespace GraphifyDemo
|
||||
{
|
||||
public interface IProcessor
|
||||
{
|
||||
List<string> Process(List<string> items);
|
||||
}
|
||||
|
||||
public class DataProcessor : IProcessor
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public DataProcessor()
|
||||
{
|
||||
_client = new HttpClient();
|
||||
}
|
||||
|
||||
public List<string> Process(List<string> items)
|
||||
{
|
||||
return Validate(items);
|
||||
}
|
||||
|
||||
private List<string> Validate(List<string> items)
|
||||
{
|
||||
var result = new List<string>();
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(item))
|
||||
result.Add(item.Trim());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
public class DataProcessor {
|
||||
private List<String> items;
|
||||
|
||||
public DataProcessor() {
|
||||
this.items = new ArrayList<>();
|
||||
}
|
||||
|
||||
public void addItem(String item) {
|
||||
items.add(item);
|
||||
}
|
||||
|
||||
public List<String> process() {
|
||||
return validate(items);
|
||||
}
|
||||
|
||||
private List<String> validate(List<String> data) {
|
||||
List<String> result = new ArrayList<>();
|
||||
for (String s : data) {
|
||||
if (s != null && !s.isEmpty()) {
|
||||
result.add(s.trim());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
interface Processor {
|
||||
List<String> process();
|
||||
}
|
||||
Vendored
+23
@@ -0,0 +1,23 @@
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlin.math.max
|
||||
|
||||
data class Config(val baseUrl: String, val timeout: Int)
|
||||
|
||||
class HttpClient(private val config: Config) {
|
||||
fun get(path: String): String {
|
||||
return buildRequest("GET", path)
|
||||
}
|
||||
|
||||
fun post(path: String, body: String): String {
|
||||
return buildRequest("POST", path)
|
||||
}
|
||||
|
||||
private fun buildRequest(method: String, path: String): String {
|
||||
return "$method ${config.baseUrl}$path"
|
||||
}
|
||||
}
|
||||
|
||||
fun createClient(baseUrl: String): HttpClient {
|
||||
val config = Config(baseUrl, 30)
|
||||
return HttpClient(config)
|
||||
}
|
||||
Vendored
+39
@@ -0,0 +1,39 @@
|
||||
<?php
|
||||
|
||||
namespace App\Http;
|
||||
|
||||
use App\Auth\Authenticator;
|
||||
use App\Cache\CacheManager;
|
||||
|
||||
class ApiClient
|
||||
{
|
||||
private string $baseUrl;
|
||||
private Authenticator $auth;
|
||||
|
||||
public function __construct(string $baseUrl)
|
||||
{
|
||||
$this->baseUrl = $baseUrl;
|
||||
$this->auth = new Authenticator();
|
||||
}
|
||||
|
||||
public function get(string $path): string
|
||||
{
|
||||
return $this->fetch($path, 'GET');
|
||||
}
|
||||
|
||||
public function post(string $path, string $body): string
|
||||
{
|
||||
return $this->fetch($path, 'POST');
|
||||
}
|
||||
|
||||
private function fetch(string $path, string $method): string
|
||||
{
|
||||
$token = $this->auth->getToken();
|
||||
return $method . ' ' . $this->baseUrl . $path;
|
||||
}
|
||||
}
|
||||
|
||||
function parseResponse(string $raw): array
|
||||
{
|
||||
return json_decode($raw, true);
|
||||
}
|
||||
Vendored
+27
@@ -0,0 +1,27 @@
|
||||
require 'json'
|
||||
require 'net/http'
|
||||
|
||||
class ApiClient
|
||||
def initialize(base_url)
|
||||
@base_url = base_url
|
||||
end
|
||||
|
||||
def get(path)
|
||||
fetch(path, 'GET')
|
||||
end
|
||||
|
||||
def post(path, body)
|
||||
fetch(path, 'POST')
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def fetch(path, method)
|
||||
uri = URI(@base_url + path)
|
||||
Net::HTTP.get(uri)
|
||||
end
|
||||
end
|
||||
|
||||
def parse_response(raw)
|
||||
JSON.parse(raw)
|
||||
end
|
||||
Vendored
+23
@@ -0,0 +1,23 @@
|
||||
import scala.collection.mutable.ListBuffer
|
||||
|
||||
case class Config(baseUrl: String, timeout: Int)
|
||||
|
||||
class HttpClient(config: Config) {
|
||||
def get(path: String): String = {
|
||||
buildRequest("GET", path)
|
||||
}
|
||||
|
||||
def post(path: String, body: String): String = {
|
||||
buildRequest("POST", path)
|
||||
}
|
||||
|
||||
private def buildRequest(method: String, path: String): String = {
|
||||
s"$method ${config.baseUrl}$path"
|
||||
}
|
||||
}
|
||||
|
||||
object HttpClientFactory {
|
||||
def create(baseUrl: String): HttpClient = {
|
||||
new HttpClient(Config(baseUrl, 30))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Tests for graphify/benchmark.py."""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import pytest
|
||||
import networkx as nx
|
||||
from networkx.readwrite import json_graph
|
||||
|
||||
from graphify.benchmark import run_benchmark, print_benchmark, _query_subgraph_tokens, _SAMPLE_QUESTIONS
|
||||
|
||||
|
||||
def _make_graph() -> nx.Graph:
|
||||
G = nx.Graph()
|
||||
G.add_node("n1", label="authentication", source_file="auth.py", source_location="L1", community=0)
|
||||
G.add_node("n2", label="api_handler", source_file="api.py", source_location="L5", community=0)
|
||||
G.add_node("n3", label="main_entry", source_file="main.py", source_location="L1", community=1)
|
||||
G.add_node("n4", label="error_handler", source_file="errors.py", source_location="L1", community=1)
|
||||
G.add_node("n5", label="database_layer", source_file="db.py", source_location="L1", community=2)
|
||||
G.add_edge("n1", "n2", relation="calls", confidence="INFERRED")
|
||||
G.add_edge("n2", "n3", relation="imports", confidence="EXTRACTED")
|
||||
G.add_edge("n3", "n4", relation="uses", confidence="EXTRACTED")
|
||||
G.add_edge("n5", "n2", relation="provides", confidence="EXTRACTED")
|
||||
return G
|
||||
|
||||
|
||||
def _write_graph(G: nx.Graph, path) -> None:
|
||||
data = json_graph.node_link_data(G, edges="links")
|
||||
path.write_text(json.dumps(data))
|
||||
|
||||
|
||||
# --- _query_subgraph_tokens ---
|
||||
|
||||
def test_query_returns_positive_for_matching_question():
|
||||
G = _make_graph()
|
||||
tokens = _query_subgraph_tokens(G, "how does authentication work")
|
||||
assert tokens > 0
|
||||
|
||||
def test_query_returns_zero_for_no_match():
|
||||
G = _make_graph()
|
||||
tokens = _query_subgraph_tokens(G, "xyzzy plugh zorkmid")
|
||||
assert tokens == 0
|
||||
|
||||
def test_query_bfs_expands_neighbors():
|
||||
G = _make_graph()
|
||||
# "authentication" matches n1, BFS depth=3 should reach n2, n3, n4
|
||||
tokens_deep = _query_subgraph_tokens(G, "authentication", depth=3)
|
||||
tokens_shallow = _query_subgraph_tokens(G, "authentication", depth=1)
|
||||
assert tokens_deep >= tokens_shallow
|
||||
|
||||
|
||||
# --- run_benchmark ---
|
||||
|
||||
def test_run_benchmark_returns_reduction(tmp_path):
|
||||
G = _make_graph()
|
||||
graph_file = tmp_path / "graph.json"
|
||||
_write_graph(G, graph_file)
|
||||
result = run_benchmark(str(graph_file), corpus_words=10_000)
|
||||
assert "reduction_ratio" in result
|
||||
assert result["reduction_ratio"] > 1.0
|
||||
|
||||
def test_run_benchmark_corpus_tokens_proportional(tmp_path):
|
||||
G = _make_graph()
|
||||
graph_file = tmp_path / "graph.json"
|
||||
_write_graph(G, graph_file)
|
||||
r1 = run_benchmark(str(graph_file), corpus_words=1_000)
|
||||
r2 = run_benchmark(str(graph_file), corpus_words=10_000)
|
||||
# corpus_tokens scales linearly with corpus_words (within integer-division rounding)
|
||||
assert abs(r2["corpus_tokens"] - r1["corpus_tokens"] * 10) <= r1["corpus_tokens"]
|
||||
|
||||
def test_run_benchmark_per_question_list(tmp_path):
|
||||
G = _make_graph()
|
||||
graph_file = tmp_path / "graph.json"
|
||||
_write_graph(G, graph_file)
|
||||
result = run_benchmark(str(graph_file), corpus_words=5_000,
|
||||
questions=["how does authentication work", "what is the main entry"])
|
||||
assert len(result["per_question"]) >= 1
|
||||
for p in result["per_question"]:
|
||||
assert "question" in p
|
||||
assert "query_tokens" in p
|
||||
assert "reduction" in p
|
||||
|
||||
def test_run_benchmark_estimates_corpus_if_no_words(tmp_path):
|
||||
G = _make_graph()
|
||||
graph_file = tmp_path / "graph.json"
|
||||
_write_graph(G, graph_file)
|
||||
result = run_benchmark(str(graph_file), corpus_words=None)
|
||||
assert result["corpus_words"] > 0
|
||||
|
||||
def test_run_benchmark_error_on_empty_graph(tmp_path):
|
||||
G = nx.Graph()
|
||||
graph_file = tmp_path / "empty.json"
|
||||
_write_graph(G, graph_file)
|
||||
result = run_benchmark(str(graph_file), corpus_words=1_000)
|
||||
assert "error" in result
|
||||
|
||||
def test_run_benchmark_includes_node_edge_counts(tmp_path):
|
||||
G = _make_graph()
|
||||
graph_file = tmp_path / "graph.json"
|
||||
_write_graph(G, graph_file)
|
||||
result = run_benchmark(str(graph_file), corpus_words=5_000)
|
||||
assert result["nodes"] == G.number_of_nodes()
|
||||
assert result["edges"] == G.number_of_edges()
|
||||
|
||||
|
||||
# --- print_benchmark ---
|
||||
|
||||
def test_print_benchmark_no_crash(tmp_path, capsys):
|
||||
G = _make_graph()
|
||||
graph_file = tmp_path / "graph.json"
|
||||
_write_graph(G, graph_file)
|
||||
result = run_benchmark(str(graph_file), corpus_words=5_000)
|
||||
print_benchmark(result)
|
||||
out = capsys.readouterr().out
|
||||
assert "reduction" in out.lower()
|
||||
assert "x" in out
|
||||
|
||||
def test_print_benchmark_error_message(capsys):
|
||||
print_benchmark({"error": "test error message"})
|
||||
out = capsys.readouterr().out
|
||||
assert "test error message" in out
|
||||
@@ -58,7 +58,9 @@ def test_extract_merges_multiple_files():
|
||||
|
||||
def test_collect_files_from_dir():
|
||||
files = collect_files(FIXTURES)
|
||||
supported = {".py", ".js", ".ts", ".tsx", ".go", ".rs"}
|
||||
supported = {".py", ".js", ".ts", ".tsx", ".go", ".rs",
|
||||
".java", ".c", ".cpp", ".cc", ".cxx", ".rb",
|
||||
".cs", ".kt", ".kts", ".scala", ".php", ".h", ".hpp"}
|
||||
assert all(f.suffix in supported for f in files)
|
||||
assert len(files) > 0
|
||||
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
"""Tests for the 8 new language extractors: Java, C, C++, Ruby, C#, Kotlin, Scala, PHP."""
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
from graphify.extract import (
|
||||
extract_java, extract_c, extract_cpp, extract_ruby,
|
||||
extract_csharp, extract_kotlin, extract_scala, extract_php,
|
||||
)
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _labels(r):
|
||||
return [n["label"] for n in r["nodes"]]
|
||||
|
||||
def _relations(r):
|
||||
return {e["relation"] for e in r["edges"]}
|
||||
|
||||
def _calls(r):
|
||||
node_by_id = {n["id"]: n["label"] for n in r["nodes"]}
|
||||
return {
|
||||
(node_by_id.get(e["source"], e["source"]), node_by_id.get(e["target"], e["target"]))
|
||||
for e in r["edges"] if e["relation"] == "calls"
|
||||
}
|
||||
|
||||
|
||||
# ── Java ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_java_no_error():
|
||||
r = extract_java(FIXTURES / "sample.java")
|
||||
assert "error" not in r
|
||||
|
||||
def test_java_finds_class():
|
||||
r = extract_java(FIXTURES / "sample.java")
|
||||
assert any("DataProcessor" in l for l in _labels(r))
|
||||
|
||||
def test_java_finds_interface():
|
||||
r = extract_java(FIXTURES / "sample.java")
|
||||
assert any("Processor" in l for l in _labels(r))
|
||||
|
||||
def test_java_finds_methods():
|
||||
r = extract_java(FIXTURES / "sample.java")
|
||||
labels = _labels(r)
|
||||
assert any("addItem" in l for l in labels)
|
||||
assert any("process" in l for l in labels)
|
||||
|
||||
def test_java_finds_imports():
|
||||
r = extract_java(FIXTURES / "sample.java")
|
||||
assert "imports" in _relations(r)
|
||||
|
||||
def test_java_no_dangling_edges():
|
||||
r = extract_java(FIXTURES / "sample.java")
|
||||
node_ids = {n["id"] for n in r["nodes"]}
|
||||
for e in r["edges"]:
|
||||
assert e["source"] in node_ids
|
||||
|
||||
|
||||
# ── C ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_c_no_error():
|
||||
r = extract_c(FIXTURES / "sample.c")
|
||||
assert "error" not in r
|
||||
|
||||
def test_c_finds_functions():
|
||||
r = extract_c(FIXTURES / "sample.c")
|
||||
labels = _labels(r)
|
||||
assert any("process" in l for l in labels)
|
||||
assert any("main" in l for l in labels)
|
||||
|
||||
def test_c_finds_includes():
|
||||
r = extract_c(FIXTURES / "sample.c")
|
||||
assert "imports" in _relations(r)
|
||||
|
||||
def test_c_emits_calls():
|
||||
r = extract_c(FIXTURES / "sample.c")
|
||||
assert any(e["relation"] == "calls" for e in r["edges"])
|
||||
|
||||
def test_c_calls_are_inferred():
|
||||
r = extract_c(FIXTURES / "sample.c")
|
||||
for e in r["edges"]:
|
||||
if e["relation"] == "calls":
|
||||
assert e["confidence"] == "INFERRED"
|
||||
|
||||
|
||||
# ── C++ ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_cpp_no_error():
|
||||
r = extract_cpp(FIXTURES / "sample.cpp")
|
||||
assert "error" not in r
|
||||
|
||||
def test_cpp_finds_class():
|
||||
r = extract_cpp(FIXTURES / "sample.cpp")
|
||||
assert any("HttpClient" in l for l in _labels(r))
|
||||
|
||||
def test_cpp_finds_methods():
|
||||
r = extract_cpp(FIXTURES / "sample.cpp")
|
||||
labels = _labels(r)
|
||||
# C++ extractor captures the constructor and public-visible methods
|
||||
assert any("HttpClient" in l for l in labels)
|
||||
|
||||
def test_cpp_finds_includes():
|
||||
r = extract_cpp(FIXTURES / "sample.cpp")
|
||||
assert "imports" in _relations(r)
|
||||
|
||||
|
||||
# ── Ruby ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_ruby_no_error():
|
||||
r = extract_ruby(FIXTURES / "sample.rb")
|
||||
assert "error" not in r
|
||||
|
||||
def test_ruby_finds_class():
|
||||
r = extract_ruby(FIXTURES / "sample.rb")
|
||||
assert any("ApiClient" in l for l in _labels(r))
|
||||
|
||||
def test_ruby_finds_methods():
|
||||
r = extract_ruby(FIXTURES / "sample.rb")
|
||||
labels = _labels(r)
|
||||
assert any("get" in l for l in labels)
|
||||
assert any("post" in l for l in labels)
|
||||
|
||||
def test_ruby_finds_function():
|
||||
r = extract_ruby(FIXTURES / "sample.rb")
|
||||
assert any("parse_response" in l for l in _labels(r))
|
||||
|
||||
|
||||
# ── C# ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_csharp_no_error():
|
||||
r = extract_csharp(FIXTURES / "sample.cs")
|
||||
assert "error" not in r
|
||||
|
||||
def test_csharp_finds_class():
|
||||
r = extract_csharp(FIXTURES / "sample.cs")
|
||||
assert any("DataProcessor" in l for l in _labels(r))
|
||||
|
||||
def test_csharp_finds_interface():
|
||||
r = extract_csharp(FIXTURES / "sample.cs")
|
||||
assert any("IProcessor" in l for l in _labels(r))
|
||||
|
||||
def test_csharp_finds_methods():
|
||||
r = extract_csharp(FIXTURES / "sample.cs")
|
||||
labels = _labels(r)
|
||||
assert any("Process" in l for l in labels)
|
||||
|
||||
def test_csharp_finds_usings():
|
||||
r = extract_csharp(FIXTURES / "sample.cs")
|
||||
assert "imports" in _relations(r)
|
||||
|
||||
|
||||
# ── Kotlin ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_kotlin_no_error():
|
||||
r = extract_kotlin(FIXTURES / "sample.kt")
|
||||
assert "error" not in r
|
||||
|
||||
def test_kotlin_finds_class():
|
||||
r = extract_kotlin(FIXTURES / "sample.kt")
|
||||
assert any("HttpClient" in l for l in _labels(r))
|
||||
|
||||
def test_kotlin_finds_data_class():
|
||||
r = extract_kotlin(FIXTURES / "sample.kt")
|
||||
assert any("Config" in l for l in _labels(r))
|
||||
|
||||
def test_kotlin_finds_methods():
|
||||
r = extract_kotlin(FIXTURES / "sample.kt")
|
||||
labels = _labels(r)
|
||||
assert any("get" in l for l in labels)
|
||||
assert any("post" in l for l in labels)
|
||||
|
||||
def test_kotlin_finds_function():
|
||||
r = extract_kotlin(FIXTURES / "sample.kt")
|
||||
assert any("createClient" in l for l in _labels(r))
|
||||
|
||||
|
||||
# ── Scala ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_scala_no_error():
|
||||
r = extract_scala(FIXTURES / "sample.scala")
|
||||
assert "error" not in r
|
||||
|
||||
def test_scala_finds_class():
|
||||
r = extract_scala(FIXTURES / "sample.scala")
|
||||
assert any("HttpClient" in l for l in _labels(r))
|
||||
|
||||
def test_scala_finds_object():
|
||||
r = extract_scala(FIXTURES / "sample.scala")
|
||||
assert any("HttpClientFactory" in l for l in _labels(r))
|
||||
|
||||
def test_scala_finds_methods():
|
||||
r = extract_scala(FIXTURES / "sample.scala")
|
||||
labels = _labels(r)
|
||||
assert any("get" in l for l in labels)
|
||||
assert any("post" in l for l in labels)
|
||||
|
||||
|
||||
# ── PHP ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_php_no_error():
|
||||
r = extract_php(FIXTURES / "sample.php")
|
||||
assert "error" not in r
|
||||
|
||||
def test_php_finds_class():
|
||||
r = extract_php(FIXTURES / "sample.php")
|
||||
assert any("ApiClient" in l for l in _labels(r))
|
||||
|
||||
def test_php_finds_methods():
|
||||
r = extract_php(FIXTURES / "sample.php")
|
||||
labels = _labels(r)
|
||||
assert any("get" in l for l in labels)
|
||||
assert any("post" in l for l in labels)
|
||||
|
||||
def test_php_finds_function():
|
||||
r = extract_php(FIXTURES / "sample.php")
|
||||
assert any("parseResponse" in l for l in _labels(r))
|
||||
|
||||
def test_php_finds_imports():
|
||||
r = extract_php(FIXTURES / "sample.php")
|
||||
assert "imports" in _relations(r)
|
||||
Reference in New Issue
Block a user