beyhan/bifrost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Beyhan Oğur
2026-04-26 21:52:23 +03:00
commit 880f412e2c
2662 changed files with 866266 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

837
tests/e2e/api/newman-reporter-dbverify/index.js Normal file
View File

@@ -0,0 +1,837 @@
'use strict';
/**
* Newman DB Verifier Reporter
*
* After each 2xx API response, fires SQL queries to verify that CRUD operations
* are correctly reflected in the database (PostgreSQL or SQLite).
*
* Main DB connection is resolved in this order:
* 1. --reporter-dbverify-db-url (explicit DSN)
* 2. BIFROST_DB_URL env var (explicit DSN)
* 3. --reporter-dbverify-config (path to Bifrost config.json; auto-detects type + DSN)
* 4. ./config.json (auto-discovered in cwd)
*
* Logs DB connection (for logs/mcp-logs endpoints) is resolved in this order:
* 1. --reporter-dbverify-logs-db-url (explicit DSN)
* 2. BIFROST_LOGS_DB_URL env var
* 3. Same config.json as above (reads logs_store section)
*
* Supported DSN formats:
* postgresql://user:pass@host:port/db[?sslmode=...]
* sqlite:///absolute/path/to/file.db
* sqlite://relative/path/to/file.db
* /absolute/path/to/file.db (bare path → treated as SQLite)
*
* Other options:
* --reporter-dbverify-silent Suppress per-request log lines
*/
const fs = require('fs');
const path = require('path');
// ─── Bifrost config.json reader ───────────────────────────────────────────────
/**
* Resolve an EnvVar field from Bifrost config.
* Values can be a plain string, an "env.KEY" reference, or an explicit
* {"value": "...", "env_var": "..."} object.
*/
function resolveEnvVar(val) {
if (val == null) return '';
if (typeof val === 'object') {
const v = val.value || '';
if (v.startsWith('env.')) return process.env[v.slice(4)] || '';
return v;
}
const s = String(val);
if (s.startsWith('env.')) return process.env[s.slice(4)] || '';
return s;
}
function sqliteUrlFromPath(filePath, configPath) {
const resolved = path.isAbsolute(filePath)
? filePath
: path.resolve(path.dirname(configPath), filePath);
return `sqlite://${resolved}`;
}
function postgresUrlFromConfig(c) {
const host = resolveEnvVar(c.host) || 'localhost';
const port = resolveEnvVar(c.port) || '5432';
const user = resolveEnvVar(c.user) || 'bifrost';
const password = resolveEnvVar(c.password) || '';
const dbName = resolveEnvVar(c.db_name) || 'bifrost';
const sslMode = resolveEnvVar(c.ssl_mode) || 'disable';
return `postgresql://${user}:${encodeURIComponent(password)}@${host}:${port}/${dbName}?sslmode=${sslMode}`;
}
/**
* Read a Bifrost config.json and return a DB connection URL for the main
* config_store, or null if not enabled / unreadable.
*/
function dbUrlFromBifrostConfig(configPath) {
let raw;
try { raw = fs.readFileSync(configPath, 'utf8'); }
catch (_) { return null; }
let cfg;
try { cfg = JSON.parse(raw); }
catch (_) { return null; }
const cs = cfg.config_store;
if (!cs || !cs.enabled) return null;
if (cs.type === 'sqlite') {
const filePath = resolveEnvVar(cs.config && cs.config.path);
if (!filePath) return null;
return sqliteUrlFromPath(filePath, configPath);
}
if (cs.type === 'postgres') {
return postgresUrlFromConfig(cs.config || {});
}
return null;
}
/**
* Read a Bifrost config.json and return a DB connection URL for the logs_store,
* or null if not enabled / unreadable.
*/
function logsDbUrlFromBifrostConfig(configPath) {
let raw;
try { raw = fs.readFileSync(configPath, 'utf8'); }
catch (_) { return null; }
let cfg;
try { cfg = JSON.parse(raw); }
catch (_) { return null; }
const ls = cfg.logs_store;
if (!ls || !ls.enabled) return null;
if (ls.type === 'sqlite') {
const filePath = resolveEnvVar(ls.config && ls.config.path);
if (!filePath) return null;
return sqliteUrlFromPath(filePath, configPath);
}
if (ls.type === 'postgres') {
return postgresUrlFromConfig(ls.config || {});
}
return null;
}
// ─── DB type detection ────────────────────────────────────────────────────────
function detectDbType(url) {
if (/^postgres(ql)?:\/\//i.test(url)) return 'postgres';
return 'sqlite';
}
function resolveSqlitePath(url) {
return url.replace(/^sqlite:\/\//i, '');
}
// ─── DB backend abstraction ───────────────────────────────────────────────────
async function createDbClient(url) {
const type = detectDbType(url);
if (type === 'postgres') {
let pg;
try { pg = require('pg'); }
catch (_) {
console.warn('[dbverify] pg module not found. Run: npm install in tests/e2e/api/');
return null;
}
const pgClient = new pg.Client({ connectionString: url });
await pgClient.connect();
return {
type: 'postgres',
query: async (sql, params) => {
const res = await pgClient.query(sql, params);
return { rows: res.rows, rowCount: res.rowCount };
},
close: () => pgClient.end().catch(() => {}),
};
}
// SQLite
let Database;
try { Database = require('better-sqlite3'); }
catch (_) {
console.warn('[dbverify] better-sqlite3 not found. Run: npm install in tests/e2e/api/');
return null;
}
const filePath = resolveSqlitePath(url);
const db = new Database(filePath, { readonly: true });
return {
type: 'sqlite',
query: async (sql, params) => {
const rows = db.prepare(sql.replace(/\$\d+/g, '?')).all(...params);
return { rows, rowCount: rows.length };
},
close: () => { try { db.close(); } catch (_) {} },
};
}
// ─── URL → Table mapping ──────────────────────────────────────────────────────
//
// logsDb: true → query is routed to the logs DB (logs_store) instead of the
// main config DB (config_store).
const URL_TABLE_MAP = [
// ── Specific (id-bearing) patterns — matched before collection patterns ────
{
pattern: /\/api\/governance\/customers\/([^/?#]+)/,
table: 'governance_customers', idParam: 1, idColumn: 'id',
verifyFields: ['id', 'name'],
bodyId: (b) => b && (b.id || (b.customer && b.customer.id)),
bodyFields: (b) => b && (b.customer || b),
},
{
pattern: /\/api\/governance\/teams\/([^/?#]+)/,
table: 'governance_teams', idParam: 1, idColumn: 'id',
verifyFields: ['id', 'name', 'customer_id'],
bodyId: (b) => b && (b.id || (b.team && b.team.id)),
bodyFields: (b) => b && (b.team || b),
},
{
pattern: /\/api\/governance\/virtual-keys\/([^/?#]+)/,
table: 'governance_virtual_keys', idParam: 1, idColumn: 'id',
verifyFields: ['id', 'name', 'is_active'],
bodyId: (b) => b && (b.id || (b.virtual_key && b.virtual_key.id)),
bodyFields: (b) => b && (b.virtual_key || b),
},
{
pattern: /\/api\/governance\/routing-rules\/([^/?#]+)/,
table: 'routing_rules', idParam: 1, idColumn: 'id',
verifyFields: ['id', 'name', 'enabled', 'provider', 'scope'],
bodyId: (b) => b && (b.id || (b.rule && b.rule.id)),
bodyFields: (b) => b && (b.rule || b),
},
{
pattern: /\/api\/governance\/model-configs\/([^/?#]+)/,
table: 'governance_model_configs', idParam: 1, idColumn: 'id',
verifyFields: ['id', 'model_name', 'provider'],
bodyId: (b) => b && (b.id || (b.model_config && b.model_config.id)),
bodyFields: (b) => b && (b.model_config || b),
},
{
pattern: /\/api\/governance\/providers\/([^/?#]+)/,
table: 'config_providers', idParam: 1, idColumn: 'name',
verifyFields: ['name'],
bodyId: (b) => b && (b.provider && (b.provider.Provider || b.provider.name)),
bodyFields: (b) => b && b.provider && { name: b.provider.Provider || b.provider.name },
deleteVerifiesExists: true,
},
{
pattern: /\/api\/providers\/([^/?#]+)/,
table: 'config_providers', idParam: 1, idColumn: 'name',
verifyFields: ['name', 'status', 'send_back_raw_request', 'send_back_raw_response'],
bodyId: (b) => b && (b.name || (b.provider && b.provider.name)),
bodyFields: (b) => b && (b.provider || b),
},
{
pattern: /\/api\/mcp\/client\/([^/?#]+)/,
table: 'config_mcp_clients', idParam: 1, idColumn: 'client_id',
verifyFields: ['client_id', 'name', 'connection_type'],
bodyId: (b) => b && (b.client_id || (b.mcp_client && b.mcp_client.client_id)),
bodyFields: (b) => b && (b.mcp_client || b),
},
{
pattern: /\/api\/logs\/([^/?#]+)/,
table: 'logs', idParam: 1, idColumn: 'id', logsDb: true,
verifyFields: ['id'],
bodyId: (b) => b && (b.id || (b.log && b.log.id)),
bodyFields: (b) => b && (b.log || b),
},
{
pattern: /\/api\/mcp-logs\/([^/?#]+)/,
table: 'mcp_tool_logs', idParam: 1, idColumn: 'id', logsDb: true,
verifyFields: ['id'],
bodyId: (b) => b && (b.id || (b.log && b.log.id)),
bodyFields: (b) => b && (b.log || b),
},
{
pattern: /\/api\/plugins\/([^/?#]+)/,
table: 'config_plugins', idParam: 1, idColumn: 'name',
verifyFields: ['name', 'enabled', 'path'],
bodyId: (b) => b && (b.name || (b.plugin && b.plugin.name)),
bodyFields: (b) => b && (b.plugin || b),
},
// ── Collection / aggregate endpoints ──────────────────────────────────────
{
pattern: /\/api\/governance\/customers$/,
table: 'governance_customers', idParam: null, idColumn: 'id',
verifyFields: ['id', 'name'],
bodyId: (b) => b && (b.id || (b.customer && b.customer.id)),
bodyFields: (b) => b && (b.customer || b),
},
{
pattern: /\/api\/governance\/teams$/,
table: 'governance_teams', idParam: null, idColumn: 'id',
verifyFields: ['id', 'name', 'customer_id'],
bodyId: (b) => b && (b.id || (b.team && b.team.id)),
bodyFields: (b) => b && (b.team || b),
},
{
pattern: /\/api\/governance\/virtual-keys$/,
table: 'governance_virtual_keys', idParam: null, idColumn: 'id',
verifyFields: ['id', 'name', 'is_active'],
bodyId: (b) => b && (b.id || (b.virtual_key && b.virtual_key.id)),
bodyFields: (b) => b && (b.virtual_key || b),
},
{
pattern: /\/api\/governance\/routing-rules$/,
table: 'routing_rules', idParam: null, idColumn: 'id',
verifyFields: ['id', 'name', 'enabled', 'provider', 'scope'],
bodyId: (b) => b && (b.id || (b.rule && b.rule.id)),
bodyFields: (b) => b && (b.rule || b),
},
{
pattern: /\/api\/governance\/model-configs$/,
table: 'governance_model_configs', idParam: null, idColumn: 'id',
verifyFields: ['id', 'model_name', 'provider'],
bodyId: (b) => b && (b.id || (b.model_config && b.model_config.id)),
bodyFields: (b) => b && (b.model_config || b),
},
{
pattern: /\/api\/providers$/,
table: 'config_providers', idParam: null, idColumn: 'name',
verifyFields: ['name', 'status', 'send_back_raw_request', 'send_back_raw_response'],
bodyId: (b) => b && (b.name || (b.provider && b.provider.name)),
bodyFields: (b) => b && (b.provider || b),
},
{
pattern: /\/api\/plugins$/,
table: 'config_plugins', idParam: null, idColumn: 'name',
verifyFields: ['name', 'enabled', 'path'],
bodyId: (b) => b && (b.name || (b.plugin && b.plugin.name)),
bodyFields: (b) => b && (b.plugin || b),
},
{
pattern: /\/api\/mcp\/client$/,
table: 'config_mcp_clients', idParam: null, idColumn: 'client_id',
verifyFields: ['client_id', 'name', 'connection_type'],
bodyId: (b) => b && (b.client_id || (b.mcp_client && b.mcp_client.client_id)),
bodyFields: (b) => b && (b.mcp_client || b),
},
{
pattern: /\/api\/mcp\/clients$/,
table: 'config_mcp_clients', idParam: null, idColumn: 'client_id',
verifyFields: ['client_id', 'name', 'connection_type'],
bodyId: (b) => b && (b.client_id || (b.mcp_client && b.mcp_client.client_id)),
bodyFields: (b) => b && (b.mcp_client || b),
},
// Proxy config — stored as a JSON blob in governance_config.value under key "proxy_config"
// PUT response is {"status":"success",...} so we compare the request body against the DB blob.
// GET response is the proxy config object directly, which also works with jsonBlobColumn.
{
pattern: /\/api\/proxy-config$/,
table: 'governance_config', idParam: null, idColumn: 'key',
jsonBlobColumn: 'value',
useRequestBody: true,
verifyFields: ['enabled', 'type', 'url', 'timeout', 'enable_for_inference', 'enable_for_api', 'enable_for_scim'],
bodyId: () => 'proxy_config',
bodyFields: (b) => b,
},
// Config — client_config in config_client, framework_config in framework_configs (multi-table)
{
pattern: /\/api\/config$/,
multiTable: [
{
table: 'config_client',
idColumn: 'id',
verifyFields: ['drop_excess_requests', 'log_retention_days', 'mcp_agent_depth', 'mcp_tool_execution_timeout'],
bodyFields: (b) => b && b.client_config,
},
{
table: 'framework_configs',
idColumn: 'id',
verifyFields: ['pricing_url', 'pricing_sync_interval'],
bodyFields: (b) => b && b.framework_config,
},
],
useRequestBody: true,
bodyId: () => null,
bodyFields: () => null,
},
// Version — build-time constant, not stored in DB
{
pattern: /\/api\/version$/,
skipReason: 'version is build-time constant, not stored in DB',
},
// Read-only table-accessible endpoints (COUNT check)
{ pattern: /\/api\/keys$/, table: 'config_keys', idParam: null, idColumn: 'id', verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/models$/, table: 'config_providers', idParam: null, idColumn: 'name', verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/models\/base$/, table: 'config_providers', idParam: null, idColumn: 'name', verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/governance\/budgets$/, table: 'governance_budgets', idParam: null, idColumn: 'id', verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/governance\/rate-limits$/, table: 'governance_rate_limits', idParam: null, idColumn: 'id', verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/governance\/providers$/, table: 'config_providers', idParam: null, idColumn: 'name', verifyFields: [], bodyId: () => null, bodyFields: () => null },
// Logs aggregate endpoints — verify the table is accessible (COUNT check)
{ pattern: /\/api\/logs\/stats$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/histogram$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/histogram\/tokens$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/histogram\/cost$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/histogram\/models$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/histogram\/latency$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/histogram\/cost\/by-provider$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/histogram\/tokens\/by-provider$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/histogram\/latency\/by-provider$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/filterdata$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/dropped$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs\/recalculate-cost$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/logs$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
// MCP logs aggregate endpoints
{ pattern: /\/api\/mcp-logs\/stats$/, table: 'mcp_tool_logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/mcp-logs\/filterdata$/, table: 'mcp_tool_logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
{ pattern: /\/api\/mcp-logs$/, table: 'mcp_tool_logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
];
function matchMapping(urlPath) {
// Sort priority (highest first):
// 1. Literal patterns with no capturing groups – e.g. /api/mcp-logs/stats$
// These are more specific than wildcard captures and must be tried first.
// 2. Wildcard id-bearing patterns – e.g. /api/mcp-logs/([^/?#]+)
// 3. Collection / aggregate patterns – e.g. /api/mcp-logs$
const sorted = [...URL_TABLE_MAP].sort((a, b) => {
const aHasCapture = /\([^)]+\)/.test(a.pattern.source);
const bHasCapture = /\([^)]+\)/.test(b.pattern.source);
if (aHasCapture !== bHasCapture) return aHasCapture ? 1 : -1;
return (b.idParam !== null ? 1 : 0) - (a.idParam !== null ? 1 : 0);
});
for (const mapping of sorted) {
const m = urlPath.match(mapping.pattern);
if (m) return { mapping, urlId: mapping.idParam !== null ? m[mapping.idParam] : null };
}
return null;
}
// ─── Helpers ──────────────────────────────────────────────────────────────────
function parseBody(response) {
try { return JSON.parse(response.stream ? response.stream.toString() : ''); }
catch (_) { return null; }
}
function parseRequestBody(request) {
try {
const raw = request && request.body && request.body.raw;
if (!raw) return null;
const str = typeof raw === 'string' ? raw : (raw && raw.toString ? raw.toString() : '');
return str ? JSON.parse(str) : null;
} catch (_) {
return null;
}
}
/** Normalize DB/JSON values for comparison. SQLite/Postgres return booleans as 0/1; API returns false/true. */
function valuesEqual(dbVal, respVal) {
if (dbVal === respVal) return true;
if (String(dbVal) === String(respVal)) return true;
// Boolean: 0/1 (DB) vs false/true (JSON)
const dbBool = dbVal === 1 || dbVal === true || (typeof dbVal === 'string' && /^true|1$/i.test(dbVal));
const respBool = respVal === 1 || respVal === true || (typeof respVal === 'string' && /^true|1$/i.test(respVal));
const dbIsBoolLike = dbVal === 0 || dbVal === 1 || dbVal === true || dbVal === false || (typeof dbVal === 'string' && /^true|false|0|1$/i.test(dbVal));
const respIsBoolLike = respVal === 0 || respVal === 1 || respVal === true || respVal === false || (typeof respVal === 'string' && /^true|false|0|1$/i.test(respVal));
if (dbIsBoolLike && respIsBoolLike) return dbBool === respBool;
return false;
}
function checkFieldMismatches(dbRow, respFields, verifyFields) {
if (!respFields || typeof respFields !== 'object') return [];
return verifyFields
.filter(f => f in respFields)
.filter(f => !valuesEqual(dbRow[f], respFields[f]))
.map(f => `${f}: db=${dbRow[f]} resp=${respFields[f]}`);
}
/**
* Like checkFieldMismatches but the dbRow has a single JSON blob column.
* Parses dbRow[jsonBlobColumn] as JSON and compares fields against respFields.
*/
function checkJsonBlobMismatches(dbRow, jsonBlobColumn, respFields, verifyFields) {
if (!respFields || typeof respFields !== 'object') return [];
let blob = {};
try { blob = JSON.parse(dbRow[jsonBlobColumn] || '{}'); } catch (_) {}
return verifyFields
.filter(f => f in respFields)
.filter(f => !valuesEqual(blob[f], respFields[f]))
.map(f => `${f}: db=${blob[f]} resp=${respFields[f]}`);
}
function pad(str, len) {
str = String(str || '');
return str.length >= len ? str : str + ' '.repeat(len - str.length);
}
// ─── Verification handlers ────────────────────────────────────────────────────
async function verifyCreated(db, m, id, body, name) {
if (!id) return { name, result: 'SKIP', detail: 'No record ID in response' };
const selectCols = m.jsonBlobColumn ? m.jsonBlobColumn
: (m.verifyFields.length ? m.verifyFields.join(', ') : m.idColumn);
const { rows } = await db.query(
`SELECT ${selectCols} FROM ${m.table} WHERE ${m.idColumn} = $1`, [id]);
if (!rows.length) return { name, result: 'FAIL', detail: `Row NOT found in ${m.table} where ${m.idColumn}=${id}` };
const mm = m.jsonBlobColumn
? checkJsonBlobMismatches(rows[0], m.jsonBlobColumn, m.bodyFields(body), m.verifyFields)
: checkFieldMismatches(rows[0], m.bodyFields(body), m.verifyFields);
if (mm.length) return { name, result: 'FAIL', detail: `Field mismatch: ${mm.join(', ')}` };
return { name, result: 'PASS', detail: `Row created in ${m.table}: ${m.idColumn}=${id}` };
}
async function verifyExists(db, m, id, body, name) {
if (!id) {
const { rows } = await db.query(`SELECT COUNT(*) AS cnt FROM ${m.table}`, []);
const cnt = rows[0].cnt !== undefined ? rows[0].cnt : rows[0].count;
return { name, result: 'PASS', detail: `${m.table} accessible, ${cnt} rows` };
}
const selectCols = m.jsonBlobColumn ? m.jsonBlobColumn
: (m.verifyFields.length ? m.verifyFields.join(', ') : m.idColumn);
const { rows } = await db.query(
`SELECT ${selectCols} FROM ${m.table} WHERE ${m.idColumn} = $1`, [id]);
if (!rows.length) return { name, result: 'FAIL', detail: `Row NOT found in ${m.table} where ${m.idColumn}=${id}` };
const mm = m.jsonBlobColumn
? checkJsonBlobMismatches(rows[0], m.jsonBlobColumn, m.bodyFields(body), m.verifyFields)
: checkFieldMismatches(rows[0], m.bodyFields(body), m.verifyFields);
if (mm.length) return { name, result: 'FAIL', detail: `Field mismatch: ${mm.join(', ')}` };
return { name, result: 'PASS', detail: `Record verified in ${m.table}: ${m.idColumn}=${id}` };
}
async function verifyUpdated(db, m, id, body, name) {
if (!id) return { name, result: 'SKIP', detail: 'No record ID in response' };
const selectCols = m.jsonBlobColumn ? m.jsonBlobColumn
: (m.verifyFields.length ? m.verifyFields.join(', ') : m.idColumn);
const { rows } = await db.query(
`SELECT ${selectCols} FROM ${m.table} WHERE ${m.idColumn} = $1`, [id]);
if (!rows.length) return { name, result: 'FAIL', detail: `Row NOT found in ${m.table} where ${m.idColumn}=${id}` };
const mm = m.jsonBlobColumn
? checkJsonBlobMismatches(rows[0], m.jsonBlobColumn, m.bodyFields(body), m.verifyFields)
: checkFieldMismatches(rows[0], m.bodyFields(body), m.verifyFields);
if (mm.length) return { name, result: 'FAIL', detail: `Update NOT reflected: ${mm.join(', ')}` };
return { name, result: 'PASS', detail: `Record updated in ${m.table}: ${m.idColumn}=${id}` };
}
/** Verify a single-row table was updated (no id in URL). SELECT LIMIT 1 and compare. */
async function verifyUpdatedSingleRow(db, table, idColumn, verifyFields, bodyFields, body, name) {
const respFields = bodyFields && bodyFields(body);
const selectCols = verifyFields.length ? verifyFields.join(', ') : idColumn;
const { rows } = await db.query(
`SELECT ${selectCols} FROM ${table} LIMIT 1`, []);
if (!rows.length) return { name, result: 'FAIL', detail: `No row in ${table}` };
const mm = checkFieldMismatches(rows[0], respFields, verifyFields);
if (mm.length) return { name, result: 'FAIL', detail: `Update NOT reflected in ${table}: ${mm.join(', ')}` };
return { name, result: 'PASS', detail: `Record updated in ${table}` };
}
async function verifyDeleted(db, m, id, name) {
if (!id) return { name, result: 'SKIP', detail: 'No record ID extractable from DELETE URL' };
const { rows } = await db.query(
`SELECT COUNT(*) AS cnt FROM ${m.table} WHERE ${m.idColumn} = $1`, [id]);
const cnt = parseInt(rows[0].cnt !== undefined ? rows[0].cnt : rows[0].count, 10);
if (cnt > 0) return { name, result: 'FAIL', detail: `Row still exists in ${m.table}: ${m.idColumn}=${id}` };
return { name, result: 'PASS', detail: `Row removed from ${m.table}: ${m.idColumn}=${id}` };
}
async function runVerification(db, method, mapping, id, body, name) {
switch (method) {
case 'POST': return verifyCreated(db, mapping, id, body, name);
case 'GET': return verifyExists(db, mapping, id, body, name);
case 'PUT':
case 'PATCH': return verifyUpdated(db, mapping, id, body, name);
case 'DELETE':
if (mapping.deleteVerifiesExists) return verifyExists(db, mapping, id, body, name);
return verifyDeleted(db, mapping, id, name);
default: return { name, result: 'SKIP', detail: `Method ${method} not verified` };
}
}
/** Run verification for multi-table mappings (e.g. /api/config). */
async function runMultiTableVerification(db, method, mapping, body, name) {
const tables = mapping.multiTable;
const results = [];
for (const t of tables) {
if (method === 'GET') {
const syntheticMapping = { table: t.table, idParam: null, idColumn: t.idColumn, verifyFields: [], bodyId: () => null, bodyFields: () => null };
const r = await verifyExists(db, syntheticMapping, null, null, name);
results.push(r);
} else if (method === 'PUT' || method === 'PATCH') {
const r = await verifyUpdatedSingleRow(db, t.table, t.idColumn, t.verifyFields || [], t.bodyFields, body, name);
results.push(r);
} else {
results.push({ name, result: 'SKIP', detail: `Method ${method} not verified for multi-table` });
}
}
const failed = results.filter((r) => r.result === 'FAIL');
const passed = results.filter((r) => r.result === 'PASS');
if (failed.length > 0) return { name, result: 'FAIL', detail: failed.map((f) => f.detail).join('; ') };
if (passed.length === 0) return results[0] || { name, result: 'SKIP', detail: 'No verifications run' };
const tableNames = tables.map((t) => t.table).join(', ');
return { name, result: 'PASS', detail: `${tableNames} verified` };
}
/**
* Process a single request's DB verification (immediate or from queue).
* Handles bulk DELETE, tracks promises, pushes results.
*/
function processRequestVerification(opts) {
const {
activeDb, method, mapping, urlId, responseBody, name, request,
pendingVerifications, results, silent,
} = opts;
// When useRequestBody is set, prefer the parsed request body for field comparison
// (e.g. PUT endpoints that return a generic success response rather than the resource).
// For GET requests the body is null so it naturally falls back to responseBody.
const verifyBody = (mapping.useRequestBody && parseRequestBody(request)) || responseBody;
// Multi-table verification (e.g. /api/config → config_client + framework_configs)
if (mapping.multiTable) {
const p = runMultiTableVerification(activeDb, method, mapping, verifyBody, name);
pendingVerifications.push(p);
p.then((r) => {
results.push(r);
if (!silent) {
const icon = r.result === 'PASS' ? '✓' : r.result === 'SKIP' ? '~' : '✗';
console.log(`[dbverify] ${icon} ${r.name}: ${r.detail}`);
}
}).catch((e) => {
const r = { name, result: 'FAIL', detail: `Query error: ${e.message}` };
results.push(r);
if (!silent) console.log(`[dbverify] ✗ ${name}: ${r.detail}`);
});
return;
}
let recordId = urlId || (verifyBody && mapping.bodyId(verifyBody));
// Bulk DELETE: extract ids from request body
if (method === 'DELETE' && !recordId) {
const reqBody = parseRequestBody(request);
const ids = (reqBody && Array.isArray(reqBody.ids) && reqBody.ids.length > 0) ? reqBody.ids : null;
if (ids) {
ids.forEach((id, i) => {
const p = runVerification(activeDb, method, mapping, id, verifyBody, ids.length > 1 ? `${name} [id=${id}]` : name);
pendingVerifications.push(p);
p.then((r) => {
results.push(r);
if (!silent) {
const icon = r.result === 'PASS' ? '✓' : r.result === 'SKIP' ? '~' : '✗';
console.log(`[dbverify] ${icon} ${r.name}: ${r.detail}`);
}
}).catch((e) => {
const r = { name, result: 'FAIL', detail: `Query error: ${e.message}` };
results.push(r);
if (!silent) console.log(`[dbverify] ✗ ${name}: ${r.detail}`);
});
});
return;
}
}
const p = runVerification(activeDb, method, mapping, recordId, verifyBody, name);
pendingVerifications.push(p);
p.then((r) => {
results.push(r);
if (!silent) {
const icon = r.result === 'PASS' ? '✓' : r.result === 'SKIP' ? '~' : '✗';
console.log(`[dbverify] ${icon} ${r.name}: ${r.detail}`);
}
}).catch((e) => {
const r = { name, result: 'FAIL', detail: `Query error: ${e.message}` };
results.push(r);
if (!silent) console.log(`[dbverify] ✗ ${name}: ${r.detail}`);
});
}
// ─── Summary ──────────────────────────────────────────────────────────────────
function printSummary(results, dbType) {
const passed = results.filter(r => r.result === 'PASS').length;
const failed = results.filter(r => r.result === 'FAIL').length;
const skipped = results.filter(r => r.result === 'SKIP').length;
const nameW = Math.max(20, ...results.map(r => (r.name || '').length));
const resultW = 6;
const detailW = Math.max(52, ...results.map(r => (r.detail || '').length));
const totalW = nameW + resultW + detailW + 7;
const hline = '─'.repeat(totalW);
const dline = '═'.repeat(totalW);
console.log('');
console.log('╔' + dline + '╗');
console.log('║' + pad(` DB Verification Results (${dbType})`, totalW) + '║');
console.log('╠' + hline + '╣');
console.log('║ ' + pad('Request', nameW) + ' │ ' + pad('Result', resultW) + ' │ ' + pad('Detail', detailW) + ' ║');
console.log('╠' + hline + '╣');
for (const r of results) {
console.log(
'║ ' + pad(r.name || '', nameW) +
' │ ' + pad(r.result, resultW) +
' │ ' + pad(r.detail || '', detailW) + ' ║'
);
}
console.log('╚' + dline + '╝');
console.log(`DB Checks: ${passed} passed, ${failed} failed, ${skipped} skipped (non-2xx or unmapped)`);
console.log('');
if (failed > 0) console.warn(`[dbverify] WARNING: ${failed} DB verification(s) FAILED`);
}
// ─── Reporter entry point ─────────────────────────────────────────────────────
module.exports = function (newman, options) {
const silent = !!(options && options['silent']);
const configPath = (options && options['config'])
|| process.env.BIFROST_CONFIG_PATH
|| path.resolve(process.cwd(), 'config.json');
// Main DB (config_store)
let dbUrl = (options && options['db-url']) || process.env.BIFROST_DB_URL || null;
if (!dbUrl) {
dbUrl = dbUrlFromBifrostConfig(configPath);
if (dbUrl && !silent) console.log(`[dbverify] Auto-detected main DB from config: ${configPath}`);
}
if (!dbUrl) {
console.warn('[dbverify] No main DB URL found. Provide --reporter-dbverify-db-url, BIFROST_DB_URL, or --reporter-dbverify-config. Skipping DB checks.');
}
// Logs DB (logs_store)
let logsDbUrl = (options && options['logs-db-url']) || process.env.BIFROST_LOGS_DB_URL || null;
if (!logsDbUrl) {
logsDbUrl = logsDbUrlFromBifrostConfig(configPath);
if (logsDbUrl && !silent) console.log(`[dbverify] Auto-detected logs DB from config: ${configPath}`);
}
const dbType = dbUrl ? detectDbType(dbUrl) : 'unknown';
const results = [];
const pendingVerifications = [];
const earlyMainDbQueue = [];
const earlyLogsDbQueue = [];
let db = null;
let logsDb = null;
let dbReady = false;
let logsDbReady = false;
function drainQueue(queue, activeDb) {
while (queue.length > 0) {
const item = queue.shift();
processRequestVerification({
activeDb, method: item.method, mapping: item.mapping, urlId: item.urlId,
responseBody: item.responseBody, name: item.name, request: item.request,
pendingVerifications, results, silent,
});
}
}
newman.on('start', function (err) {
if (err) return;
if (dbUrl) {
const safeUrl = dbUrl.replace(/:([^:@]+)@/, ':***@');
createDbClient(dbUrl)
.then((client) => {
db = client;
dbReady = !!client;
if (dbReady && !silent) console.log(`[dbverify] Connected to ${dbType} DB: ${safeUrl}`);
if (dbReady && db) drainQueue(earlyMainDbQueue, db);
})
.catch((e) => {
dbReady = false;
console.warn(`[dbverify] Main DB not reachable, skipping DB checks: ${e.message}`);
earlyMainDbQueue.forEach((item) => results.push({ name: item.name, result: 'SKIP', detail: 'Main DB not connected' }));
earlyMainDbQueue.length = 0;
});
}
if (logsDbUrl) {
const safeLogsUrl = logsDbUrl.replace(/:([^:@]+)@/, ':***@');
createDbClient(logsDbUrl)
.then((client) => {
logsDb = client;
logsDbReady = !!client;
if (logsDbReady && !silent) console.log(`[dbverify] Connected to logs DB (${detectDbType(logsDbUrl)}): ${safeLogsUrl}`);
if (logsDbReady && logsDb) drainQueue(earlyLogsDbQueue, logsDb);
})
.catch((e) => {
logsDbReady = false;
console.warn(`[dbverify] Logs DB not reachable, skipping logs DB checks: ${e.message}`);
earlyLogsDbQueue.forEach((item) => results.push({ name: item.name, result: 'SKIP', detail: 'Logs DB not connected' }));
earlyLogsDbQueue.length = 0;
});
}
});
newman.on('request', function (err, args) {
if (err) return;
const response = args.response;
const request = args.request;
const name = (args.item && args.item.name) || 'Unknown Request';
const statusCode = response && response.code;
if (!statusCode || statusCode < 200 || statusCode > 299) {
results.push({ name, result: 'SKIP', detail: `HTTP ${statusCode || '?'} (non-2xx)` });
return;
}
const method = request.method.toUpperCase();
const urlPath = request.url.toString()
.replace(/\?.*$/, '')
.replace(/^https?:\/\/[^/]+/, '');
const match = matchMapping(urlPath);
if (!match) {
results.push({ name, result: 'SKIP', detail: 'URL not mapped to DB table' });
return;
}
const { mapping, urlId } = match;
if (mapping.skipReason) {
results.push({ name, result: 'SKIP', detail: mapping.skipReason });
return;
}
// Pick the right DB client
const isLogsTable = !!mapping.logsDb;
const activeDb = isLogsTable ? logsDb : db;
const activeReady = isLogsTable ? logsDbReady : dbReady;
const responseBody = parseBody(response);
if (!activeReady || !activeDb) {
const queue = isLogsTable ? earlyLogsDbQueue : earlyMainDbQueue;
queue.push({
method, mapping, urlId, responseBody, name, request,
});
return;
}
processRequestVerification({
activeDb, method, mapping, urlId, responseBody, name, request,
pendingVerifications, results, silent,
});
});
newman.on('done', function () {
Promise.allSettled(pendingVerifications).then(() => {
if (db) db.close();
if (logsDb) logsDb.close();
if (results.length > 0) printSummary(results, dbType);
});
});
};

6
tests/e2e/api/newman-reporter-dbverify/package.json Normal file
View File

@@ -0,0 +1,6 @@
{
"name": "newman-reporter-dbverify",
"version": "1.0.0",
"description": "Newman reporter that verifies CRUD operations against PostgreSQL",
"main": "index.js"
}