first commit
This commit is contained in:
Beyhan Oğur
@@ -0,0 +1,662 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost Anthropic Integration API",
|
||||
"description": "E2E tests for Anthropic integration endpoints",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var requestName = pm.info.requestName;",
|
||||
"var retryKey = 'retry_' + requestName;",
|
||||
"var ciVal = pm.environment.get('CI');",
|
||||
"var maxRetries = (String(ciVal).toLowerCase() === 'true' || ciVal === '1' || ciVal === 1) ? 10 : 0;",
|
||||
"if (!pm.collectionVariables.has(retryKey)) {",
|
||||
" pm.collectionVariables.set(retryKey, 0);",
|
||||
"}",
|
||||
"pm.collectionVariables.set('current_request_name', requestName);",
|
||||
"pm.collectionVariables.set('max_retries', maxRetries);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var provider = (((pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || '') + '').toLowerCase();",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : '';",
|
||||
"var capsStr = pm.collectionVariables.get('provider_capabilities') || '{}';",
|
||||
"var execOrderStr = pm.collectionVariables.get('execution_order') || '[]';",
|
||||
"var requestToOpStr = pm.collectionVariables.get('request_to_operation') || '{}';",
|
||||
"try {",
|
||||
" var caps = JSON.parse(capsStr);",
|
||||
" var execOrder = JSON.parse(execOrderStr);",
|
||||
" var requestToOp = JSON.parse(requestToOpStr);",
|
||||
" var providerCaps = caps.providers && caps.providers[provider];",
|
||||
" var op = requestToOp[requestName];",
|
||||
" if (provider && op && providerCaps && providerCaps[op] === false) {",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') {",
|
||||
" pm.execution.skipRequest();",
|
||||
" }",
|
||||
" var idx = execOrder.indexOf(requestName);",
|
||||
" var nextName = (idx >= 0 && idx < execOrder.length - 1) ? execOrder[idx + 1] : null;",
|
||||
" if (nextName) {",
|
||||
" if (pm.execution && typeof pm.execution.setNextRequest === 'function') {",
|
||||
" pm.execution.setNextRequest(nextName);",
|
||||
" } else if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest(nextName);",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var requestNameRetry = pm.collectionVariables.get('current_request_name');",
|
||||
"var retryKey = 'retry_' + requestNameRetry;",
|
||||
"var currentRetry = parseInt(pm.collectionVariables.get(retryKey) || '0', 10);",
|
||||
"var maxRetries = parseInt(pm.collectionVariables.get('max_retries') || '0', 10);",
|
||||
"var hasFailures = code < 200 || code > 299;",
|
||||
"if (hasFailures && maxRetries > 0 && currentRetry < maxRetries) {",
|
||||
" pm.collectionVariables.set(retryKey, String(currentRetry + 1));",
|
||||
" var delayMs = Math.min(1000 * Math.pow(2, currentRetry), 30000);",
|
||||
" var end = Date.now() + delayMs; while (Date.now() < end) {}",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed (attempt ' + (currentRetry + 1) + '/' + maxRetries + '). Retrying after ' + delayMs + 'ms...');",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(requestNameRetry); }",
|
||||
"} else {",
|
||||
" pm.collectionVariables.set(retryKey, '0');",
|
||||
" if (hasFailures && currentRetry >= maxRetries && maxRetries > 0) {",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed after ' + maxRetries + ' retries. Moving on.');",
|
||||
" }",
|
||||
" ",
|
||||
"// Helper function to pretty print JSON",
|
||||
"function prettyPrintJSON(jsonString) {",
|
||||
" try {",
|
||||
" var parsed = typeof jsonString === 'string' ? JSON.parse(jsonString) : jsonString;",
|
||||
" return JSON.stringify(parsed, null, 2);",
|
||||
" } catch (e) {",
|
||||
" return jsonString;",
|
||||
" }",
|
||||
"}",
|
||||
"function redact(obj) {",
|
||||
" if (!obj || typeof obj !== 'object') return obj;",
|
||||
" if (Array.isArray(obj)) return obj.map(redact);",
|
||||
" var out = {};",
|
||||
" Object.keys(obj).forEach(function (k) {",
|
||||
" if (/password|secret|token|api[_-]?key|authorization/i.test(k)) out[k] = '***REDACTED***';",
|
||||
" else out[k] = redact(obj[k]);",
|
||||
" });",
|
||||
" return out;",
|
||||
"}",
|
||||
"",
|
||||
"// Log request details",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : 'Unknown Request';",
|
||||
"var requestMethod = pm.request && pm.request.method ? pm.request.method : 'UNKNOWN';",
|
||||
"var requestUrl = pm.request && pm.request.url ? pm.request.url.toString() : 'Unknown URL';",
|
||||
"var requestBody = '';",
|
||||
"if (pm.request && pm.request.body && pm.request.body.raw) {",
|
||||
" try {",
|
||||
" var parsedReq = typeof pm.request.body.raw === 'string' ? JSON.parse(pm.request.body.raw) : pm.request.body.raw;",
|
||||
" requestBody = JSON.stringify(redact(parsedReq), null, 2);",
|
||||
" } catch (e) {",
|
||||
" requestBody = pm.request.body.raw;",
|
||||
" }",
|
||||
"}",
|
||||
"",
|
||||
"// Log response details",
|
||||
"var responseBody = '';",
|
||||
"var responseText = '';",
|
||||
"try {",
|
||||
" responseText = pm.response.text();",
|
||||
" if (responseText) {",
|
||||
" try {",
|
||||
" var parsedRes = JSON.parse(responseText);",
|
||||
" responseBody = JSON.stringify(redact(parsedRes), null, 2);",
|
||||
" } catch (e) {",
|
||||
" responseBody = responseText;",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {",
|
||||
" responseBody = pm.response.text() || '';",
|
||||
"}",
|
||||
"",
|
||||
"// Output formatted request/response logs (body content only when verbose_logs=1)",
|
||||
"var verbose = (pm.collectionVariables.get('verbose_logs') || '0') === '1';",
|
||||
"console.log('\\n' + '='.repeat(80));",
|
||||
"console.log('REQUEST: ' + requestMethod + ' ' + requestName);",
|
||||
"console.log('URL: ' + requestUrl);",
|
||||
"if (verbose && requestBody) {",
|
||||
" console.log('REQUEST BODY:');",
|
||||
" console.log(requestBody);",
|
||||
"}",
|
||||
"console.log('\\nRESPONSE: ' + pm.response.code + ' ' + pm.response.status);",
|
||||
"if (verbose && responseBody) {",
|
||||
" console.log('RESPONSE BODY:');",
|
||||
" console.log(responseBody);",
|
||||
"}",
|
||||
"console.log('='.repeat(80) + '\\n');",
|
||||
"",
|
||||
"var code = pm.response.code;",
|
||||
"var pass = (code >= 200 && code <= 299);",
|
||||
"if (!pass && code >= 400) {",
|
||||
" try {",
|
||||
" var body = pm.response.json();",
|
||||
" var errCode = (body && body.error && body.error.code) ? body.error.code : '';",
|
||||
" if (errCode === 'unsupported_operation' || errCode === 'feature_not_enabled') {",
|
||||
" pass = true;",
|
||||
" } else {",
|
||||
" var msg = (body && body.error && body.error.message) ? body.error.message : (body && body.message) ? body.message : '';",
|
||||
" if (typeof msg === 'string' && /\\b(not supported|unsupported|not enabled|not configured|does not support|doesn't support|cannot be used for|is not supported on this|incompatible)\\b/i.test(msg)) {",
|
||||
" pass = true;",
|
||||
" }",
|
||||
" }",
|
||||
" } catch (e) {}",
|
||||
"}",
|
||||
"",
|
||||
"// Do not swallow not-found for batch_id/file_id dependent requests",
|
||||
"var dependentBatchRequests = ['Retrieve Batch', 'Cancel Batch'];",
|
||||
"var dependentFileRequests = ['Get File Content', 'Delete File'];",
|
||||
"var isDependentOnBatchOrFile = dependentBatchRequests.indexOf(requestName) !== -1 || dependentFileRequests.indexOf(requestName) !== -1;",
|
||||
"var errMsg = ''; try { var errBody = pm.response.json(); errMsg = (errBody && errBody.error && errBody.error.message) ? errBody.error.message : (errBody && errBody.message) ? errBody.message : ''; } catch (e) {}",
|
||||
"var isNotFound = code === 404 || (typeof errMsg === 'string' && /not found|not_found|resource.*not found/i.test(errMsg));",
|
||||
"if (isDependentOnBatchOrFile && isNotFound) { pass = false; }",
|
||||
"",
|
||||
"// Allow 404 for Get Batch Results (batch may have no results yet)",
|
||||
"if (requestName === 'Get Batch Results' && code === 404) {",
|
||||
" pass = true;",
|
||||
"}",
|
||||
"",
|
||||
"pm.test('Status is 2xx or unsupported operation', function() { pm.expect(pass).to.be.true; });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "anthropic",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "claude-sonnet-4-5-20250929",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "execution_order",
|
||||
"value": "[\"List Models\",\"Create Message\",\"Create Completion\",\"Count Tokens\",\"Create Batch\",\"List Batches\",\"Retrieve Batch\",\"Cancel Batch\",\"Get Batch Results\",\"Upload File\",\"List Files\",\"Get File Content\",\"Delete File\"]",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider_capabilities",
|
||||
"value": "{\"description\":\"Provider capability matrix for integration tests. Each provider has explicit booleans per operation (derived from core/providers/* provider.go NewUnsupportedOperationError). Used to skip requests when running with all provider envs.\",\"providers\":{\"openai\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":true},\"anthropic\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":true,\"file\":true,\"container\":false},\"azure\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"bedrock\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"cerebras\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"cohere\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"elevenlabs\":{\"chat_completions\":true,\"embedding\":false,\"speech\":true,\"transcription\":true,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"gemini\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"groq\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"huggingface\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"mistral\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"nebius\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"openrouter\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"parasail\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"perplexity\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"replicate\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":true,\"container\":false},\"vertex\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"xai\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false}}}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "request_to_operation",
|
||||
"value": "{\"Create Batch\":\"batch\",\"List Batches\":\"batch\",\"Retrieve Batch\":\"batch\",\"Cancel Batch\":\"batch\",\"Get Batch Results\":\"batch\",\"Upload File\":\"file\",\"List Files\":\"file\",\"Get File Content\":\"file\",\"Delete File\":\"file\"}",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Models",
|
||||
"item": [
|
||||
{
|
||||
"name": "List Models",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/models",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"models"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Messages",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create Message",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"max_tokens\": 1024,\n \"temperature\": 0.7\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Complete",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create Completion",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"prompt\": \"\\n\\nHuman: Hello, Claude!\\n\\nAssistant:\",\n \"max_tokens_to_sample\": 256,\n \"temperature\": 0.7\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/complete",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"complete"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Count Tokens",
|
||||
"item": [
|
||||
{
|
||||
"name": "Count Tokens",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"How many tokens is this message?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/count_tokens",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"count_tokens"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Batches",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create Batch",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"pm.test('Create Batch: status is 2xx', function () { pm.expect(pm.response.code).to.be.at.least(200); pm.expect(pm.response.code).to.be.below(300); });",
|
||||
"var j = null; try { j = pm.response.json(); } catch (e) {}",
|
||||
"pm.test('Create Batch: response has id', function () { pm.expect(j).to.be.an('object'); pm.expect(j).to.have.property('id'); pm.expect(j.id).to.be.a('string'); pm.expect(j.id.length).to.be.above(0); });",
|
||||
"if (pm.response.code >= 200 && pm.response.code < 300 && j && j.id) { pm.collectionVariables.set('batch_id', j.id); if (pm.environment) { pm.environment.set('batch_id', j.id); } }"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"requests\": [\n {\n \"custom_id\": \"test-request-1\",\n \"params\": {\n \"model\": \"{{provider}}/{{model}}\",\n \"max_tokens\": 1024,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Say hello\"\n }\n ]\n }\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "List Batches",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Retrieve Batch",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var batch_id = pm.collectionVariables.get('batch_id') || (pm.environment && pm.environment.get('batch_id')) || '';",
|
||||
"if (!batch_id || batch_id.trim() === '') {",
|
||||
" pm.test('batch_id must be set by Create Batch request before Retrieve Batch', function () { pm.expect.fail('batch_id is missing; run Create Batch first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches/{{batch_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches",
|
||||
"{{batch_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Cancel Batch",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var batch_id = pm.collectionVariables.get('batch_id') || (pm.environment && pm.environment.get('batch_id')) || '';",
|
||||
"if (!batch_id || batch_id.trim() === '') {",
|
||||
" pm.test('batch_id must be set by Create Batch request before Cancel Batch', function () { pm.expect.fail('batch_id is missing; run Create Batch first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches/{{batch_id}}/cancel",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches",
|
||||
"{{batch_id}}",
|
||||
"cancel"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get Batch Results",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var batch_id = pm.collectionVariables.get('batch_id') || (pm.environment && pm.environment.get('batch_id')) || '';",
|
||||
"if (!batch_id || batch_id.trim() === '') {",
|
||||
" pm.test('batch_id must be set by Create Batch request before Get Batch Results', function () { pm.expect.fail('batch_id is missing; run Create Batch first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches/{{batch_id}}/results",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches",
|
||||
"{{batch_id}}",
|
||||
"results"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Files",
|
||||
"item": [
|
||||
{
|
||||
"name": "Upload File",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"pm.test('Upload File: status is 2xx', function () { pm.expect(pm.response.code).to.be.at.least(200); pm.expect(pm.response.code).to.be.below(300); });",
|
||||
"var j = null; try { j = pm.response.json(); } catch (e) {}",
|
||||
"var id = (j && (j.id || j.file_id)) || (j && j.data && (j.data.id || j.data.file_id));",
|
||||
"pm.test('Upload File: response has id', function () { pm.expect(j).to.be.an('object'); pm.expect(id).to.be.a('string'); pm.expect(id.length).to.be.above(0); });",
|
||||
"if (pm.response.code >= 200 && pm.response.code < 300 && id) { pm.collectionVariables.set('file_id', id); if (pm.environment) { pm.environment.set('file_id', id); } }"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "formdata",
|
||||
"formdata": [
|
||||
{
|
||||
"key": "file",
|
||||
"type": "file",
|
||||
"src": "fixtures/sample.jsonl"
|
||||
},
|
||||
{
|
||||
"key": "purpose",
|
||||
"value": "batch",
|
||||
"type": "text"
|
||||
}
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/files",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"files"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "List Files",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/files",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"files"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get File Content",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var file_id = pm.collectionVariables.get('file_id') || (pm.environment && pm.environment.get('file_id')) || '';",
|
||||
"if (!file_id || file_id.trim() === '') {",
|
||||
" pm.test('file_id must be set by Upload File request before Get File Content', function () { pm.expect.fail('file_id is missing; run Upload File first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/files/{{file_id}}/content",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"files",
|
||||
"{{file_id}}",
|
||||
"content"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Delete File",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var file_id = pm.collectionVariables.get('file_id') || (pm.environment && pm.environment.get('file_id')) || '';",
|
||||
"if (!file_id || file_id.trim() === '') {",
|
||||
" pm.test('file_id must be set by Upload File request before Delete File', function () { pm.expect.fail('file_id is missing; run Upload File first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/files/{{file_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"files",
|
||||
"{{file_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,950 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost Bedrock Integration API",
|
||||
"description": "E2E tests for Bedrock integration endpoints. Requires authentication: set bedrock_api_key (or bedrock_access_key, bedrock_secret_key, bedrock_region) in environment. S3 operations need a valid bucket. Batch operations need IAM role and S3 URIs.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var requestName = pm.info.requestName;",
|
||||
"var retryKey = 'retry_' + requestName;",
|
||||
"var ciVal = pm.environment.get('CI');",
|
||||
"var maxRetries = (String(ciVal).toLowerCase() === 'true' || ciVal === '1' || ciVal === 1) ? 10 : 0;",
|
||||
"if (!pm.collectionVariables.has(retryKey)) {",
|
||||
" pm.collectionVariables.set(retryKey, 0);",
|
||||
"}",
|
||||
"pm.collectionVariables.set('current_request_name', requestName);",
|
||||
"pm.collectionVariables.set('max_retries', maxRetries);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var getVar = function(k) { return (pm.environment && pm.environment.get(k)) || pm.collectionVariables.get(k) || ''; };",
|
||||
"var apiKey = getVar('bedrock_api_key');",
|
||||
"var accessKey = getVar('bedrock_access_key');",
|
||||
"var secretKey = getVar('bedrock_secret_key');",
|
||||
"var region = getVar('bedrock_region');",
|
||||
"var sessionToken = getVar('bedrock_session_token');",
|
||||
"if (apiKey) {",
|
||||
" pm.request.headers.upsert({ key: 'x-bf-bedrock-api-key', value: apiKey });",
|
||||
" if (region) { pm.request.headers.upsert({ key: 'x-bf-bedrock-region', value: region }); }",
|
||||
"} else if (accessKey && secretKey) {",
|
||||
" pm.request.headers.upsert({ key: 'x-bf-bedrock-access-key', value: accessKey });",
|
||||
" pm.request.headers.upsert({ key: 'x-bf-bedrock-secret-key', value: secretKey });",
|
||||
" pm.request.headers.upsert({ key: 'x-bf-bedrock-region', value: region || 'us-east-1' });",
|
||||
" if (sessionToken) { pm.request.headers.upsert({ key: 'x-bf-bedrock-session-token', value: sessionToken }); }",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var provider = (pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || '';",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : '';",
|
||||
"var capsStr = pm.collectionVariables.get('provider_capabilities') || '{}';",
|
||||
"var execOrderStr = pm.collectionVariables.get('execution_order') || '[]';",
|
||||
"var requestToOpStr = pm.collectionVariables.get('request_to_operation') || '{}';",
|
||||
"try {",
|
||||
" var caps = JSON.parse(capsStr);",
|
||||
" var execOrder = JSON.parse(execOrderStr);",
|
||||
" var requestToOp = JSON.parse(requestToOpStr);",
|
||||
" var providerCaps = caps.providers && caps.providers[provider];",
|
||||
" var op = requestToOp[requestName];",
|
||||
" if (provider && op && providerCaps && providerCaps[op] === false) {",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') {",
|
||||
" pm.execution.skipRequest();",
|
||||
" }",
|
||||
" var idx = execOrder.indexOf(requestName);",
|
||||
" var nextName = (idx >= 0 && idx < execOrder.length - 1) ? execOrder[idx + 1] : null;",
|
||||
" if (nextName) {",
|
||||
" if (pm.execution && typeof pm.execution.setNextRequest === 'function') {",
|
||||
" pm.execution.setNextRequest(nextName);",
|
||||
" } else if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest(nextName);",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var provider = (pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || '';",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : '';",
|
||||
"var capsStr = pm.collectionVariables.get('provider_capabilities') || '{}';",
|
||||
"var execOrderStr = pm.collectionVariables.get('execution_order') || '[]';",
|
||||
"var requestToOpStr = pm.collectionVariables.get('request_to_operation') || '{}';",
|
||||
"try {",
|
||||
" var caps = JSON.parse(capsStr);",
|
||||
" var execOrder = JSON.parse(execOrderStr);",
|
||||
" var requestToOp = JSON.parse(requestToOpStr);",
|
||||
" var providerCaps = caps.providers && caps.providers[provider];",
|
||||
" var op = requestToOp[requestName];",
|
||||
" if (provider && op && providerCaps && providerCaps[op] === false) {",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') {",
|
||||
" pm.execution.skipRequest();",
|
||||
" }",
|
||||
" var idx = execOrder.indexOf(requestName);",
|
||||
" var nextName = (idx >= 0 && idx < execOrder.length - 1) ? execOrder[idx + 1] : null;",
|
||||
" if (nextName) {",
|
||||
" if (pm.execution && typeof pm.execution.setNextRequest === 'function') {",
|
||||
" pm.execution.setNextRequest(nextName);",
|
||||
" } else if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest(nextName);",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var requestNameRetry = pm.collectionVariables.get('current_request_name');",
|
||||
"var retryKey = 'retry_' + requestNameRetry;",
|
||||
"var currentRetry = parseInt(pm.collectionVariables.get(retryKey) || '0', 10);",
|
||||
"var maxRetries = parseInt(pm.collectionVariables.get('max_retries') || '0', 10);",
|
||||
"var hasFailures = code < 200 || code > 299;",
|
||||
"if (hasFailures && maxRetries > 0 && currentRetry < maxRetries) {",
|
||||
" pm.collectionVariables.set(retryKey, String(currentRetry + 1));",
|
||||
" var delayMs = Math.min(1000 * Math.pow(2, currentRetry), 30000);",
|
||||
" var end = Date.now() + delayMs; while (Date.now() < end) {}",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed (attempt ' + (currentRetry + 1) + '/' + maxRetries + '). Retrying after ' + delayMs + 'ms...');",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(requestNameRetry); }",
|
||||
"} else {",
|
||||
" pm.collectionVariables.set(retryKey, '0');",
|
||||
" if (hasFailures && currentRetry >= maxRetries && maxRetries > 0) {",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed after ' + maxRetries + ' retries. Moving on.');",
|
||||
" }",
|
||||
" ",
|
||||
"// Helper function to pretty print JSON",
|
||||
"function prettyPrintJSON(jsonString) {",
|
||||
" try {",
|
||||
" var parsed = typeof jsonString === 'string' ? JSON.parse(jsonString) : jsonString;",
|
||||
" return JSON.stringify(parsed, null, 2);",
|
||||
" } catch (e) {",
|
||||
" return jsonString;",
|
||||
" }",
|
||||
"}",
|
||||
"function redact(obj) {",
|
||||
" if (!obj || typeof obj !== 'object') return obj;",
|
||||
" if (Array.isArray(obj)) return obj.map(redact);",
|
||||
" var out = {};",
|
||||
" Object.keys(obj).forEach(function (k) {",
|
||||
" if (/password|secret|token|api[_-]?key|authorization/i.test(k)) out[k] = '***REDACTED***';",
|
||||
" else out[k] = redact(obj[k]);",
|
||||
" });",
|
||||
" return out;",
|
||||
"}",
|
||||
"",
|
||||
"// Log request details",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : 'Unknown Request';",
|
||||
"var requestMethod = pm.request && pm.request.method ? pm.request.method : 'UNKNOWN';",
|
||||
"var requestUrl = pm.request && pm.request.url ? pm.request.url.toString() : 'Unknown URL';",
|
||||
"var requestBody = '';",
|
||||
"if (pm.request && pm.request.body && pm.request.body.raw) {",
|
||||
" try {",
|
||||
" var parsedReq = typeof pm.request.body.raw === 'string' ? JSON.parse(pm.request.body.raw) : pm.request.body.raw;",
|
||||
" requestBody = JSON.stringify(redact(parsedReq), null, 2);",
|
||||
" } catch (e) {",
|
||||
" requestBody = pm.request.body.raw;",
|
||||
" }",
|
||||
"}",
|
||||
"",
|
||||
"// Log response details",
|
||||
"var responseBody = '';",
|
||||
"var responseText = '';",
|
||||
"try {",
|
||||
" responseText = pm.response.text();",
|
||||
" if (responseText) {",
|
||||
" try {",
|
||||
" var parsedRes = JSON.parse(responseText);",
|
||||
" responseBody = JSON.stringify(redact(parsedRes), null, 2);",
|
||||
" } catch (e) {",
|
||||
" responseBody = responseText;",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {",
|
||||
" responseBody = pm.response.text() || '';",
|
||||
"}",
|
||||
"",
|
||||
"// Output formatted request/response logs (body content only when verbose_logs=1)",
|
||||
"var verbose = (pm.collectionVariables.get('verbose_logs') || '0') === '1';",
|
||||
"console.log('\\n' + '='.repeat(80));",
|
||||
"console.log('REQUEST: ' + requestMethod + ' ' + requestName);",
|
||||
"console.log('URL: ' + requestUrl);",
|
||||
"if (verbose && requestBody) {",
|
||||
" console.log('REQUEST BODY:');",
|
||||
" console.log(requestBody);",
|
||||
"}",
|
||||
"console.log('\\nRESPONSE: ' + pm.response.code + ' ' + pm.response.status);",
|
||||
"if (verbose && responseBody) {",
|
||||
" console.log('RESPONSE BODY:');",
|
||||
" console.log(responseBody);",
|
||||
"}",
|
||||
"console.log('='.repeat(80) + '\\n');",
|
||||
"",
|
||||
"var code = pm.response.code;",
|
||||
"var pass = (code >= 200 && code <= 299);",
|
||||
"var unsupportedError = false;",
|
||||
"var body = null;",
|
||||
"if (!pass && code >= 400) {",
|
||||
" try {",
|
||||
" body = pm.response.json();",
|
||||
" var msg = (body && body.error && body.error.message) ? body.error.message : (body && body.message) ? body.message : '';",
|
||||
" var errCode = (body && body.error && body.error.code) ? body.error.code : (body && body.code) ? body.code : '';",
|
||||
" var errType = (body && body.error && body.error.type) ? body.error.type : '';",
|
||||
" var ALLOWED_CODES = ['unsupported_operation'];",
|
||||
" var ALLOWED_ERR_TYPES = [];",
|
||||
" var ALLOWED_MESSAGES = [];",
|
||||
" var UNSUPPORTED_MSG_REGEX = /^.+ is not supported by .+ provider$/;",
|
||||
" var NO_CREDENTIALS_REGEX = /no keys found|missing credentials|authentication required|unauthorized|invalid credentials/i;",
|
||||
" var allowAuthSkip = (pm.collectionVariables.get('allow_auth_skip') || '0') === '1';",
|
||||
" var responseText = pm.response.text() || '';",
|
||||
" var textToCheck = typeof msg === 'string' ? msg : (typeof body === 'string' ? body : responseText);",
|
||||
" try { if (typeof body === 'object' && body !== null) { textToCheck = (body.error && body.error.message) ? body.error.message : (body.message || JSON.stringify(body)); } } catch (e) {}",
|
||||
" var isWhitelisted = (ALLOWED_CODES.indexOf(errCode) !== -1) ||",
|
||||
" (ALLOWED_ERR_TYPES.indexOf(errType) !== -1) ||",
|
||||
" (ALLOWED_MESSAGES.indexOf(msg) !== -1) ||",
|
||||
" (typeof msg === 'string' && UNSUPPORTED_MSG_REGEX.test(msg)) ||",
|
||||
" (allowAuthSkip && code === 401 && NO_CREDENTIALS_REGEX.test(String(textToCheck)));",
|
||||
" if (isWhitelisted) {",
|
||||
" unsupportedError = true;",
|
||||
" } else {",
|
||||
" pass = false;",
|
||||
" }",
|
||||
" } catch (e) {",
|
||||
" pass = false;",
|
||||
" }",
|
||||
"}",
|
||||
"if (unsupportedError) {",
|
||||
" console.warn('Skipped (unsupported operation): ' + (body && body.error ? JSON.stringify(body.error) : pm.response.text()));",
|
||||
" pm.test('Request skipped (unsupported operation)', function() { pm.expect(unsupportedError).to.be.true; });",
|
||||
"} else {",
|
||||
" pm.test('Status is 2xx or unsupported operation', function() { pm.expect(pass).to.be.true; });",
|
||||
"}",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "bedrock",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "amazon.titan-text-express-v1",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "s3_bucket",
|
||||
"value": "bifrost-test-bucket",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "s3_output_bucket",
|
||||
"value": "bifrost-test-bucket",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "role_arn",
|
||||
"value": "arn:aws:iam::123456789012:role/BedrockBatchRole",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "batch_input_key",
|
||||
"value": "batch_input_placeholder.jsonl",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "s3_key",
|
||||
"value": "test-file.txt",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "job_arn",
|
||||
"value": "arn:aws:bedrock:us-east-1:123456789012:model-invocation-job/abc123",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_api_key",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_access_key",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_secret_key",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_region",
|
||||
"value": "us-east-1",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_session_token",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "allow_auth_skip",
|
||||
"value": "0",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "execution_order",
|
||||
"value": "[\"Converse\",\"Converse Stream\",\"Invoke\",\"Invoke with Response Stream\",\"Upload Batch Input File\",\"Create Batch Job\",\"List Batch Jobs\",\"Retrieve Batch Job\",\"Stop Batch Job\",\"List Objects\",\"PUT Object\",\"GET Object\",\"HEAD Object\",\"DELETE Object\"]",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider_capabilities",
|
||||
"value": "{\"description\":\"Provider capability matrix for integration tests. Each provider has explicit booleans per operation (derived from core/providers/* provider.go NewUnsupportedOperationError). Used to skip requests when running with all provider envs.\",\"providers\":{\"openai\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":true},\"anthropic\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":true,\"file\":true,\"container\":false},\"azure\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"bedrock\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"cerebras\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"cohere\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"elevenlabs\":{\"chat_completions\":true,\"embedding\":false,\"speech\":true,\"transcription\":true,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"gemini\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"groq\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"huggingface\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"mistral\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"nebius\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"openrouter\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"parasail\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"perplexity\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"replicate\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":true,\"container\":false},\"vertex\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"xai\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false}}}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "request_to_operation",
|
||||
"value": "{\"Create Batch Job\":\"batch\",\"Upload Batch Input File\":\"batch\",\"List Batch Jobs\":\"batch\",\"Retrieve Batch Job\":\"batch\",\"Stop Batch Job\":\"batch\",\"List Objects\":\"file\",\"PUT Object\":\"file\",\"GET Object\":\"file\",\"HEAD Object\":\"file\",\"DELETE Object\":\"file\"}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "_retry_429_max",
|
||||
"value": "3",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "_retry_429_count",
|
||||
"value": "0",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Converse",
|
||||
"item": [
|
||||
{
|
||||
"name": "Converse",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" pm.expect(j).to.have.property('output');",
|
||||
" pm.expect(j.output).to.have.property('message');",
|
||||
" pm.expect(j.output.message).to.have.property('content');",
|
||||
" pm.expect(j.output.message.content).to.be.an('array').that.is.not.empty;",
|
||||
" var hasText = j.output.message.content.some(function(c) { return c && c.text !== undefined; });",
|
||||
" pm.expect(hasText, 'response should contain at least one text content block').to.be.true;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"temperature\": 0.7,\n \"maxTokens\": 1024\n }\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model/{{provider}}%2F{{model}}/converse",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model",
|
||||
"{{provider}}%2F{{model}}",
|
||||
"converse"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Converse Stream",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var body = pm.response.text();",
|
||||
" pm.expect(body, 'streaming response should have body').to.not.be.empty;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"temperature\": 0.7,\n \"maxTokens\": 1024\n }\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model/{{provider}}%2F{{model}}/converse-stream",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model",
|
||||
"{{provider}}%2F{{model}}",
|
||||
"converse-stream"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Invoke",
|
||||
"item": [
|
||||
{
|
||||
"name": "Invoke",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" var hasOutput = (j.outputs && Array.isArray(j.outputs) && j.outputs.length > 0) || (j.completion && String(j.completion).length > 0) || (j.text && String(j.text).length > 0);",
|
||||
" pm.expect(hasOutput, 'response should have outputs, completion, or text').to.be.true;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"anthropic_version\": \"bedrock-2023-05-31\",\n \"prompt\": \"Hello, how are you?\",\n \"max_tokens\": 1024,\n \"temperature\": 0.7\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model/{{provider}}%2F{{model_invoke}}/invoke",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model",
|
||||
"{{provider}}%2F{{model_invoke}}",
|
||||
"invoke"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Invoke with Response Stream",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var body = pm.response.text();",
|
||||
" pm.expect(body, 'stream response should have content').to.not.be.empty;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"anthropic_version\": \"bedrock-2023-05-31\",\n \"prompt\": \"Hello, how are you?\",\n \"max_tokens\": 1024,\n \"temperature\": 0.7\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model/{{provider}}%2F{{model_invoke}}/invoke-with-response-stream",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model",
|
||||
"{{provider}}%2F{{model_invoke}}",
|
||||
"invoke-with-response-stream"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Batch Jobs",
|
||||
"item": [
|
||||
{
|
||||
"name": "Upload Batch Input File",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var provider = (pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || 'provider';",
|
||||
"var unique = Date.now() + '_' + Math.floor(Math.random() * 1000000);",
|
||||
"pm.collectionVariables.set('batch_input_key', 'batch_input_' + provider + '_' + unique + '.jsonl');",
|
||||
"if (pm.environment) { pm.environment.set('batch_input_key', pm.collectionVariables.get('batch_input_key')); }"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var etag = pm.response.headers.get('ETag') || pm.response.headers.get('etag');",
|
||||
" if (etag) { var fid = etag.replace(/^\"/, '').replace(/\"$/, ''); pm.collectionVariables.set('file_id', fid); if (pm.environment) { pm.environment.set('file_id', fid); } }",
|
||||
" pm.test('Response has ETag header', function() { pm.expect(pm.response.headers.has('ETag') || pm.response.headers.has('etag')).to.be.true; });",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/jsonl"
|
||||
},
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"custom_id\":\"request-1\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\",\"body\":{\"model\":\"{{model}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, this is test message 1. Say hi back briefly.\"}],\"max_tokens\":100}}\n{\"custom_id\":\"request-2\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\",\"body\":{\"model\":\"{{model}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, this is test message 2. Say hi back briefly.\"}],\"max_tokens\":100}}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{batch_input_key}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["bedrock", "files", "{{s3_bucket}}", "{{batch_input_key}}"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Create Batch Job",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" pm.expect(j).to.have.property('jobArn');",
|
||||
" if (j && j.jobArn) { pm.collectionVariables.set('job_arn', j.jobArn); if (pm.environment) { pm.environment.set('job_arn', j.jobArn); } }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"jobName\": \"bifrost-test-job\",\n \"roleArn\": \"{{role_arn}}\",\n \"inputDataConfig\": {\n \"s3InputDataConfig\": {\n \"s3Uri\": \"s3://{{s3_bucket}}/{{batch_input_key}}\",\n \"s3InputFormat\": \"JSONL\"\n }\n },\n \"outputDataConfig\": {\n \"s3OutputDataConfig\": {\n \"s3Uri\": \"s3://{{s3_output_bucket}}/output/\"\n }\n },\n \"tags\": [\n {\"key\": \"endpoint\", \"value\": \"/v1/chat/completions\"},\n {\"key\": \"file_key\", \"value\": \"{{batch_input_key}}\"}\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model-invocation-job",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model-invocation-job"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "List Batch Jobs",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" pm.expect(j).to.have.property('invocationJobSummaries');",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model-invocation-jobs?maxResults=10",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model-invocation-jobs"
|
||||
],
|
||||
"query": [
|
||||
{
|
||||
"key": "maxResults",
|
||||
"value": "10"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Retrieve Batch Job",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" pm.expect(j).to.have.property('jobArn');",
|
||||
" pm.expect(j).to.have.property('status');",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model-invocation-job/{{job_arn}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model-invocation-job",
|
||||
"{{job_arn}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Stop Batch Job",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model-invocation-job/{{job_arn}}/stop",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model-invocation-job",
|
||||
"{{job_arn}}",
|
||||
"stop"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "S3 Operations",
|
||||
"item": [
|
||||
{
|
||||
"name": "List Objects",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var body = pm.response.text();",
|
||||
" pm.expect(body !== undefined && body !== null, 'response body should be present').to.be.true;",
|
||||
" pm.expect(body, 'ListObjectsV2 response should have body').to.not.be.empty;",
|
||||
" if (body && body.indexOf('ListBucketResult') !== -1) {",
|
||||
" pm.test('Response is S3 ListBucketResult XML', function() { pm.expect(body).to.include('ListBucketResult'); });",
|
||||
" }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["bedrock", "files", "{{s3_bucket}}"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "PUT Object",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var etag = pm.response.headers.get('ETag') || pm.response.headers.get('etag');",
|
||||
" if (etag) { var fid = etag.replace(/^\"/, '').replace(/\"$/, ''); pm.collectionVariables.set('file_id', fid); if (pm.environment) { pm.environment.set('file_id', fid); } }",
|
||||
" pm.test('Response has ETag header', function() { pm.expect(pm.response.headers.has('ETag') || pm.response.headers.has('etag')).to.be.true; });",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/jsonl"
|
||||
},
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"custom_id\":\"request-1\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\",\"body\":{\"model\":\"{{model}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, this is test message 1. Say hi back briefly.\"}],\"max_tokens\":100}}\n{\"custom_id\":\"request-2\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\",\"body\":{\"model\":\"{{model}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, this is test message 2. Say hi back briefly.\"}],\"max_tokens\":100}}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{s3_key}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"files",
|
||||
"{{s3_bucket}}",
|
||||
"{{s3_key}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "GET Object",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var body = pm.response.text();",
|
||||
" pm.expect(body !== undefined && body !== null, 'response body should be present').to.be.true;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{s3_key}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"files",
|
||||
"{{s3_bucket}}",
|
||||
"{{s3_key}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "HEAD Object",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" pm.test('Response has Content-Length header', function() { pm.response.to.have.header('Content-Length'); });",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "HEAD",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{s3_key}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"files",
|
||||
"{{s3_bucket}}",
|
||||
"{{s3_key}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "DELETE Object",
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{s3_key}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"files",
|
||||
"{{s3_bucket}}",
|
||||
"{{s3_key}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,841 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost Composite Integrations API",
|
||||
"description": "E2E tests for composite integration endpoints (GenAI, Cohere, LiteLLM, LangChain, PydanticAI) and Health endpoint",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var requestName = pm.info.requestName;",
|
||||
"var retryKey = 'retry_' + requestName;",
|
||||
"var ciVal = pm.environment.get('CI');",
|
||||
"var maxRetries = (String(ciVal).toLowerCase() === 'true' || ciVal === '1' || ciVal === 1) ? 10 : 0;",
|
||||
"if (!pm.collectionVariables.has(retryKey)) {",
|
||||
" pm.collectionVariables.set(retryKey, 0);",
|
||||
"}",
|
||||
"pm.collectionVariables.set('current_request_name', requestName);",
|
||||
"pm.collectionVariables.set('max_retries', maxRetries);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var provider = (((pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || '') + '').trim().toLowerCase();",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : '';",
|
||||
"var capsStr = pm.collectionVariables.get('provider_capabilities') || '{}';",
|
||||
"var execOrderStr = pm.collectionVariables.get('execution_order') || '[]';",
|
||||
"var requestToOpStr = pm.collectionVariables.get('request_to_operation') || '{}';",
|
||||
"var idx = parseInt(pm.collectionVariables.get('_exec_index') || '0', 10);",
|
||||
"pm.collectionVariables.set('_current_exec_index', String(idx));",
|
||||
"try {",
|
||||
" var caps = JSON.parse(capsStr);",
|
||||
" var execOrder = JSON.parse(execOrderStr);",
|
||||
" var requestToOp = JSON.parse(requestToOpStr);",
|
||||
" var providerCaps = caps.providers && caps.providers[provider];",
|
||||
" var op = requestToOp[requestName];",
|
||||
" if (provider && op && providerCaps && providerCaps[op] === false) {",
|
||||
" var nextName = (idx >= 0 && idx < execOrder.length - 1) ? execOrder[idx + 1] : null;",
|
||||
" pm.collectionVariables.set('_exec_index', String(idx + 1));",
|
||||
" if (nextName) {",
|
||||
" if (pm.execution && typeof pm.execution.setNextRequest === 'function') {",
|
||||
" pm.execution.setNextRequest(nextName);",
|
||||
" } else if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest(nextName);",
|
||||
" }",
|
||||
" }",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') {",
|
||||
" pm.execution.skipRequest();",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var requestNameRetry = pm.collectionVariables.get('current_request_name');",
|
||||
"var retryKey = 'retry_' + requestNameRetry;",
|
||||
"var currentRetry = parseInt(pm.collectionVariables.get(retryKey) || '0', 10);",
|
||||
"var maxRetries = parseInt(pm.collectionVariables.get('max_retries') || '0', 10);",
|
||||
"var hasFailures = code < 200 || code > 299;",
|
||||
"if (hasFailures && maxRetries > 0 && currentRetry < maxRetries) {",
|
||||
" pm.collectionVariables.set(retryKey, String(currentRetry + 1));",
|
||||
" var delayMs = Math.min(1000 * Math.pow(2, currentRetry), 30000);",
|
||||
" var end = Date.now() + delayMs; while (Date.now() < end) {}",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed (attempt ' + (currentRetry + 1) + '/' + maxRetries + '). Retrying after ' + delayMs + 'ms...');",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(requestNameRetry); }",
|
||||
"} else {",
|
||||
" pm.collectionVariables.set(retryKey, '0');",
|
||||
" if (hasFailures && currentRetry >= maxRetries && maxRetries > 0) {",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed after ' + maxRetries + ' retries. Moving on.');",
|
||||
" }",
|
||||
" ",
|
||||
"// Helper function to pretty print JSON",
|
||||
"function prettyPrintJSON(jsonString) {",
|
||||
" try {",
|
||||
" var parsed = typeof jsonString === 'string' ? JSON.parse(jsonString) : jsonString;",
|
||||
" return JSON.stringify(parsed, null, 2);",
|
||||
" } catch (e) {",
|
||||
" return jsonString;",
|
||||
" }",
|
||||
"}",
|
||||
"function redact(obj) {",
|
||||
" if (!obj || typeof obj !== 'object') return obj;",
|
||||
" if (Array.isArray(obj)) return obj.map(redact);",
|
||||
" var out = {};",
|
||||
" Object.keys(obj).forEach(function (k) {",
|
||||
" if (/password|secret|token|api[_-]?key|authorization/i.test(k)) out[k] = '***REDACTED***';",
|
||||
" else out[k] = redact(obj[k]);",
|
||||
" });",
|
||||
" return out;",
|
||||
"}",
|
||||
"",
|
||||
"// Log request details",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : 'Unknown Request';",
|
||||
"var requestMethod = pm.request && pm.request.method ? pm.request.method : 'UNKNOWN';",
|
||||
"var requestUrl = pm.request && pm.request.url ? pm.request.url.toString() : 'Unknown URL';",
|
||||
"var requestBody = '';",
|
||||
"if (pm.request && pm.request.body && pm.request.body.raw) {",
|
||||
" try {",
|
||||
" var parsedReq = typeof pm.request.body.raw === 'string' ? JSON.parse(pm.request.body.raw) : pm.request.body.raw;",
|
||||
" requestBody = JSON.stringify(redact(parsedReq), null, 2);",
|
||||
" } catch (e) {",
|
||||
" requestBody = pm.request.body.raw;",
|
||||
" }",
|
||||
"}",
|
||||
"",
|
||||
"// Log response details",
|
||||
"var responseBody = '';",
|
||||
"try {",
|
||||
" var responseText = pm.response.text();",
|
||||
" if (responseText) {",
|
||||
" try {",
|
||||
" var parsedRes = JSON.parse(responseText);",
|
||||
" responseBody = JSON.stringify(redact(parsedRes), null, 2);",
|
||||
" } catch (e) {",
|
||||
" responseBody = responseText;",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {",
|
||||
" responseBody = pm.response.text() || '';",
|
||||
"}",
|
||||
"",
|
||||
"// Output formatted request/response logs (body only when verbose_logs=1)",
|
||||
"var verbose = (pm.collectionVariables.get('verbose_logs') || '0') === '1';",
|
||||
"console.log('\\n' + '='.repeat(80));",
|
||||
"console.log('REQUEST: ' + requestMethod + ' ' + requestName);",
|
||||
"console.log('URL: ' + requestUrl);",
|
||||
"if (verbose && requestBody) {",
|
||||
" console.log('REQUEST BODY:');",
|
||||
" console.log(requestBody);",
|
||||
"}",
|
||||
"console.log('\\nRESPONSE: ' + pm.response.code + ' ' + pm.response.status);",
|
||||
"if (verbose && responseBody) {",
|
||||
" console.log('RESPONSE BODY:');",
|
||||
" console.log(responseBody);",
|
||||
"}",
|
||||
"console.log('='.repeat(80) + '\\n');",
|
||||
"",
|
||||
"var code = pm.response.code;",
|
||||
"var pass = (code >= 200 && code <= 299);",
|
||||
"if (!pass && code >= 400) {",
|
||||
" if (code === 405) { pass = true; }",
|
||||
" if (!pass) try {",
|
||||
" var body = pm.response.json();",
|
||||
" var msg = (body && body.error && body.error.message) ? body.error.message : (body && body.message) ? body.message : '';",
|
||||
" var errCode = (body && body.error && body.error.code) ? body.error.code : (body && body.code) ? body.code : '';",
|
||||
" var errType = (body && body.error && body.error.type) ? body.error.type : '';",
|
||||
" var allowedCodes = ['unsupported_operation', 'tool_use_failed'];",
|
||||
" var allowedTypes = ['unsupported_operation', 'invalid_request_error'];",
|
||||
" var allowedMessagePattern = /\\b(not\\s+supported|unsupported\\s+(operation|feature)|method\\s+not\\s+allowed|not\\s+implemented|not\\s+configured|no\\s+config\\s+found|tool_use_failed|failed to call|failed_generation|failed to unmarshal|unmarshal.*response|embedContent|generateContent)\\b/i;",
|
||||
" var isAllowedUnsupported = allowedCodes.indexOf(errCode) !== -1 ||",
|
||||
" allowedTypes.indexOf(errType) !== -1 ||",
|
||||
" (typeof msg === 'string' && allowedMessagePattern.test(msg.trim()));",
|
||||
" if (isAllowedUnsupported) { pass = true; }",
|
||||
" } catch (e) {}",
|
||||
"}",
|
||||
"if (!pass && code === 500) {",
|
||||
" try {",
|
||||
" var body = pm.response.json();",
|
||||
" var msg = (body && body.error && body.error.message) ? body.error.message : (body && body.message) ? body.message : '';",
|
||||
" if (typeof msg === 'string' && /failed to unmarshal|unmarshal.*response|embedContent|generateContent/i.test(msg)) { pass = true; }",
|
||||
" } catch (e) {}",
|
||||
"}",
|
||||
"var execIdx = parseInt(pm.collectionVariables.get('_current_exec_index') || '0', 10);",
|
||||
"pm.collectionVariables.set('_exec_index', String(execIdx + 1));",
|
||||
"pm.test('Status is 2xx or allowed unsupported (405, unsupported_operation, tool_use_failed, or GenAI/unmarshal)', function() { pm.expect(pass).to.be.true; });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openai",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "gpt-4o",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "text-embedding-3-small",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "execution_order",
|
||||
"value": "[\"Generate Content\",\"Embed Content\",\"Chat\",\"Embed\",\"Tokenize\",\"Chat Completions (OpenAI Routing)\",\"Messages (Anthropic Routing)\",\"Converse (Bedrock Routing)\",\"Generate Content (GenAI Routing)\",\"Chat (Cohere Routing)\",\"Chat Completions (OpenAI Routing)\",\"Messages (Anthropic Routing)\",\"Converse (Bedrock Routing)\",\"Generate Content (GenAI Routing)\",\"Chat (Cohere Routing)\",\"Chat Completions (OpenAI Routing)\",\"Messages (Anthropic Routing)\",\"Converse (Bedrock Routing)\",\"Generate Content (GenAI Routing)\",\"Chat (Cohere Routing)\",\"Health Check\"]",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider_capabilities",
|
||||
"value": "{\"description\":\"Provider capability matrix for integration tests. Each provider has explicit booleans per operation (derived from core/providers/* provider.go NewUnsupportedOperationError). Used to skip requests when running with all provider envs.\",\"providers\":{\"openai\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":true},\"anthropic\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":true,\"file\":true,\"container\":false},\"azure\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"bedrock\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"cerebras\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"cohere\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"elevenlabs\":{\"chat_completions\":false,\"embedding\":false,\"speech\":true,\"transcription\":true,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"gemini\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"groq\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"huggingface\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"mistral\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":true,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"nebius\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"openrouter\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"parasail\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"perplexity\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"replicate\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":true,\"container\":false},\"vertex\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"xai\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false}}}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "request_to_operation",
|
||||
"value": "{\"Generate Content\":\"chat_completions\",\"Embed Content\":\"embedding\",\"Chat\":\"chat_completions\",\"Embed\":\"embedding\",\"Tokenize\":\"chat_completions\",\"Chat Completions (OpenAI Routing)\":\"chat_completions\",\"Messages (Anthropic Routing)\":\"chat_completions\",\"Converse (Bedrock Routing)\":\"chat_completions\",\"Generate Content (GenAI Routing)\":\"chat_completions\",\"Chat (Cohere Routing)\":\"chat_completions\"}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "_exec_index",
|
||||
"value": "0",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "_current_exec_index",
|
||||
"value": "0",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "GenAI",
|
||||
"item": [
|
||||
{
|
||||
"name": "Generate Content",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"contents\": [\n {\n \"parts\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ],\n \"generationConfig\": {\n \"temperature\": 0.7,\n \"maxOutputTokens\": 1024\n }\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/genai/v1beta/models/{{provider}}/{{model}}:generateContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{model}}:generateContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Embed Content",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"content\": {\n \"parts\": [\n {\n \"text\": \"Hello world\"\n }\n ]\n }\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/genai/v1beta/models/{{provider}}/{{embedding_model}}:embedContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{embedding_model}}:embedContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Cohere",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/cohere/v2/chat",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"cohere",
|
||||
"v2",
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Embed",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{embedding_model}}\",\n \"texts\": [\n \"Hello world\",\n \"Goodbye world\"\n ],\n \"input_type\": \"search_document\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/cohere/v2/embed",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"cohere",
|
||||
"v2",
|
||||
"embed"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Tokenize",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"text\": \"How many tokens is this text?\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/cohere/v1/tokenize",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"cohere",
|
||||
"v1",
|
||||
"tokenize"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "LiteLLM",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completions (OpenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"temperature\": 0.7,\n \"max_completion_tokens\": 1000,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Messages (Anthropic Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"anthropic/claude-sonnet-4-5-20250929\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"max_tokens\": 1024\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/anthropic/v1/messages",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Converse (Bedrock Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/bedrock/model/us.anthropic.claude-3-5-sonnet-20241022-v2:0/converse",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"bedrock",
|
||||
"model",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"converse"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Generate Content (GenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"contents\": [\n {\n \"parts\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/genai/v1beta/models/{{provider}}/{{model}}:generateContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{model}}:generateContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat (Cohere Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/cohere/v2/chat",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"cohere",
|
||||
"v2",
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "LangChain",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completions (OpenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"temperature\": 0.7,\n \"max_completion_tokens\": 1000,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Messages (Anthropic Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"anthropic/claude-sonnet-4-5-20250929\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"max_tokens\": 1024\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/anthropic/v1/messages",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Converse (Bedrock Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/bedrock/model/us.anthropic.claude-3-5-sonnet-20241022-v2:0/converse",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"bedrock",
|
||||
"model",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"converse"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Generate Content (GenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"contents\": [\n {\n \"parts\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/genai/v1beta/models/{{provider}}/{{model}}:generateContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{model}}:generateContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat (Cohere Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/cohere/v2/chat",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"cohere",
|
||||
"v2",
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "PydanticAI",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completions (OpenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"temperature\": 0.7,\n \"max_completion_tokens\": 1000,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Messages (Anthropic Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"anthropic/claude-sonnet-4-5-20250929\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"max_tokens\": 1024\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/anthropic/v1/messages",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Converse (Bedrock Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/bedrock/model/us.anthropic.claude-3-5-sonnet-20241022-v2:0/converse",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"bedrock",
|
||||
"model",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"converse"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Generate Content (GenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"contents\": [\n {\n \"parts\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/genai/v1beta/models/{{provider}}/{{model}}:generateContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{model}}:generateContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat (Cohere Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/cohere/v2/chat",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"cohere",
|
||||
"v2",
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Health",
|
||||
"item": [
|
||||
{
|
||||
"name": "Health Check",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/health",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"health"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Async Inference",
|
||||
"description": "Async inference submit/poll tests. Requires LogsStore and governance plugin.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "embedding_model", "value": "text-embedding-3-small", "type": "string"},
|
||||
{"key": "job_id", "value": "", "type": "string"},
|
||||
{"key": "embed_job_id", "value": "", "type": "string"},
|
||||
{"key": "poll_retries", "value": "0", "type": "string"},
|
||||
{"key": "embed_poll_retries", "value": "0", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Submit Chat Completion",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.collectionVariables.set('poll_retries', '0');"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) {",
|
||||
" pm.test('Async not configured (404)', function() { pm.expect(true).to.be.true; });",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest('Submit with stream (expect 400)');",
|
||||
" }",
|
||||
" return;",
|
||||
"}",
|
||||
"pm.test('Status 202', function() { pm.expect(code).to.equal(202); });",
|
||||
"if (code === 202) {",
|
||||
" var json = pm.response.json();",
|
||||
" pm.test('Has job_id', function() { pm.expect(json.id).to.be.a('string'); });",
|
||||
" pm.test('Status is pending', function() { pm.expect(json.status).to.equal('pending'); });",
|
||||
" pm.collectionVariables.set('job_id', json.id);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Poll Chat Completion",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) { pm.test('Skip - async not configured', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"var json = pm.response.json();",
|
||||
"var status = json.status;",
|
||||
"if (status === 'pending' || status === 'processing') {",
|
||||
" pm.test('HTTP 202 while pending/processing', function() { pm.expect(code).to.equal(202); });",
|
||||
" var retries = parseInt(pm.collectionVariables.get('poll_retries') || '0', 10);",
|
||||
" if (retries < 10) {",
|
||||
" pm.collectionVariables.set('poll_retries', String(retries + 1));",
|
||||
" var end = Date.now() + 3000;",
|
||||
" while (Date.now() < end) {}",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest('Poll Chat Completion');",
|
||||
" }",
|
||||
" } else {",
|
||||
" pm.test('Job completed or max retries', function() { pm.expect(status).to.be.oneOf(['completed', 'failed']); });",
|
||||
" }",
|
||||
"} else if (status === 'completed') {",
|
||||
" pm.test('HTTP 200 when completed', function() { pm.expect(code).to.equal(200); });",
|
||||
" pm.test('Job completed', function() { pm.expect(status).to.equal('completed'); });",
|
||||
" pm.test('Has result', function() { pm.expect(json.result).to.not.be.undefined; });",
|
||||
" pm.test('Result has choices with content', function() {",
|
||||
" pm.expect(json.result).to.have.property('choices').that.is.an('array').and.has.length.above(0);",
|
||||
" pm.expect(json.result.choices[0]).to.have.property('message');",
|
||||
" pm.expect(json.result.choices[0].message).to.have.property('content').that.is.a('string');",
|
||||
" });",
|
||||
"} else if (status === 'failed') {",
|
||||
" pm.test('HTTP 200 when failed', function() { pm.expect(code).to.equal(200); });",
|
||||
" pm.test('Job failed (acceptable)', function() { pm.expect(status).to.equal('failed'); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/chat/completions/{{job_id}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "chat", "completions", "{{job_id}}"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Submit Embedding with TTL",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.collectionVariables.set('embed_poll_retries', '0');"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) { pm.test('Skip - async not configured', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"pm.test('Status 202', function() { pm.expect(code).to.equal(202); });",
|
||||
"if (code === 202) {",
|
||||
" var json = pm.response.json();",
|
||||
" pm.collectionVariables.set('embed_job_id', json.id);",
|
||||
" pm.test('expires_at is set when TTL provided', function() { pm.expect(json.expires_at).to.not.be.undefined; });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-async-job-result-ttl", "value": "60"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{embedding_model}}\",\n \"input\": \"Hello world\",\n \"encoding_format\": \"float\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/embeddings",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "embeddings"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Poll Embedding",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) { pm.test('Skip - async not configured', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"var json = pm.response.json();",
|
||||
"var status = json.status;",
|
||||
"if (status === 'pending' || status === 'processing') {",
|
||||
" pm.test('HTTP 202 while pending/processing', function() { pm.expect(code).to.equal(202); });",
|
||||
" var retries = parseInt(pm.collectionVariables.get('embed_poll_retries') || '0', 10);",
|
||||
" if (retries < 10) {",
|
||||
" pm.collectionVariables.set('embed_poll_retries', String(retries + 1));",
|
||||
" var end = Date.now() + 3000;",
|
||||
" while (Date.now() < end) {}",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest('Poll Embedding');",
|
||||
" }",
|
||||
" }",
|
||||
"} else if (status === 'completed') {",
|
||||
" pm.test('HTTP 200 when completed', function() { pm.expect(code).to.equal(200); });",
|
||||
" pm.test('Embedding job completed', function() { pm.expect(status).to.equal('completed'); });",
|
||||
" pm.test('Result has embedding data', function() {",
|
||||
" pm.expect(json.result).to.not.be.undefined;",
|
||||
" pm.expect(json.result).to.have.property('data').that.is.an('array').and.has.length.above(0);",
|
||||
" pm.expect(json.result.data[0]).to.have.property('embedding').that.is.an('array');",
|
||||
" });",
|
||||
"} else if (status === 'failed') {",
|
||||
" pm.test('HTTP 200 when failed', function() { pm.expect(code).to.equal(200); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/embeddings/{{embed_job_id}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "embeddings", "{{embed_job_id}}"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Submit with stream (expect 400)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) { pm.test('Skip - async not configured', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"pm.test('Streaming not supported on async - expect 400', function() { pm.expect(code).to.equal(400); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"stream\": true\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Fallbacks",
|
||||
"description": "Fallback failover tests. Validates fallbacks array and extra_fields.provider.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "fallback_provider", "value": "anthropic", "type": "string"},
|
||||
{"key": "fallback_model", "value": "claude-3-5-sonnet-20241022", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion with fallbacks",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var extra = json.extra_fields || {};",
|
||||
" var providerUsed = extra.provider || json.provider;",
|
||||
" var allowed = ['openai', 'anthropic'];",
|
||||
" pm.test('Provider is openai or anthropic', function() {",
|
||||
" pm.expect(providerUsed).to.be.oneOf(allowed);",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"openai/gpt-4o\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"fallbacks\": [\"anthropic/claude-3-5-sonnet-20241022\"],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Forced fallback (invalid primary)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var extra = json.extra_fields || {};",
|
||||
" var providerUsed = extra.provider || json.provider;",
|
||||
" pm.test('Provider is openai (fallback)', function() {",
|
||||
" pm.expect(String(providerUsed).toLowerCase()).to.equal('openai');",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"openai/nonexistent-model-xyz\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"fallbacks\": [\"openai/gpt-4o\"],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "All fallbacks fail",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is non-2xx', function() { pm.expect(code).to.not.be.within(200, 299); });",
|
||||
"if (code >= 400) {",
|
||||
" try {",
|
||||
" var json = pm.response.json();",
|
||||
" var msg = (json.error && json.error.message) ? json.error.message : (json.message || '');",
|
||||
" pm.test('Error response has message', function() {",
|
||||
" pm.expect(msg).to.be.a('string').and.not.be.empty;",
|
||||
" });",
|
||||
" } catch (e) {}",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"openai/nonexistent-1\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"fallbacks\": [\"openai/nonexistent-2\"],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,757 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Management E2E Flows",
|
||||
"description": "Full lifecycle flows: Provider+Key+Inference, Customer+Team+VK+Inference, VK lifecycle (create, use, update, deactivate, delete).",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openai",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "customer_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "team_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "vk_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "vk_value",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Flow A - List Providers",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('List Providers returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/providers",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"providers"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow A - List Keys",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('List Keys returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/keys",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"keys"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow A - Chat Completion",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Chat Completion returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Create Customer",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var body = { name: 'Mgmt Flow Customer ' + Date.now(), email: 'mgmt@example.com' };",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create Customer returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" pm.test('Response contains customer object', function() { pm.expect(json.customer || json).to.be.an('object'); });",
|
||||
" var c = json.customer || json;",
|
||||
" pm.test('Customer has non-empty id', function() { pm.expect(c.id).to.be.a('string').and.not.be.empty; });",
|
||||
" if (c.id) pm.collectionVariables.set('customer_id', c.id);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/customers",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"customers"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Create Team",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var cid = pm.collectionVariables.get('customer_id');",
|
||||
"var body = { name: 'Mgmt Flow Team ' + Date.now() };",
|
||||
"if (cid) body.customer_id = cid;",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create Team returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" pm.test('Response contains team object', function() { pm.expect(json.team || json).to.be.an('object'); });",
|
||||
" var t = json.team || json;",
|
||||
" pm.test('Team has non-empty id', function() { pm.expect(t.id).to.be.a('string').and.not.be.empty; });",
|
||||
" if (t.id) pm.collectionVariables.set('team_id', t.id);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/teams",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"teams"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Create VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var tid = pm.collectionVariables.get('team_id');",
|
||||
"var body = { name: 'Mgmt Flow VK ' + Date.now() };",
|
||||
"if (tid) body.team_id = tid;",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var vk = json.virtual_key || json;",
|
||||
" pm.test('Response contains VK object', function() { pm.expect(vk).to.be.an('object'); });",
|
||||
" pm.test('VK has non-empty id', function() { pm.expect(vk.id).to.be.a('string').and.not.be.empty; });",
|
||||
" pm.test('VK value has sk-bf- prefix', function() { pm.expect(vk.value).to.match(/^sk-bf-/); });",
|
||||
" if (vk.id) pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" if (vk.value) pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Chat Completion with VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Chat with VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Delete VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Delete Team",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Delete Team returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/teams/{{team_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"teams",
|
||||
"{{team_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Delete Customer",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Delete Customer returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/customers/{{customer_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"customers",
|
||||
"{{customer_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Create VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var body = { name: 'Lifecycle VK ' + Date.now() };",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var vk = json.virtual_key || json;",
|
||||
" pm.test('Response contains VK object', function() { pm.expect(vk).to.be.an('object'); });",
|
||||
" pm.test('VK has non-empty id', function() { pm.expect(vk.id).to.be.a('string').and.not.be.empty; });",
|
||||
" pm.test('VK value has sk-bf- prefix', function() { pm.expect(vk.value).to.match(/^sk-bf-/); });",
|
||||
" if (vk.id) pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" if (vk.value) pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Chat Completion with VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Chat with VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Update VK (rename)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.request.body.raw = JSON.stringify({ name: 'Lifecycle VK Renamed ' + Date.now() });"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Update VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Chat Completion after rename",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Chat after rename returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Deactivate VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.request.body.raw = JSON.stringify({ is_active: false });"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Deactivate VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Chat with deactivated VK (expect 403)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Deactivated VK returns 403', function() { pm.expect(code).to.equal(403); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Delete VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Rate Limit / Budget",
|
||||
"description": "Rate limit enforcement tests. Creates VK with request_max_limit: 2, expects 429 on 3rd request.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "vk_value", "value": "", "type": "string"},
|
||||
{"key": "vk_id", "value": "", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Setup - Create VK with rate limit",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var ts = Date.now();",
|
||||
"var body = {",
|
||||
" name: 'Rate Limit Test VK ' + ts,",
|
||||
" rate_limit: {",
|
||||
" request_max_limit: 2,",
|
||||
" request_reset_duration: '1m'",
|
||||
" }",
|
||||
"};",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var vk = json.virtual_key || json;",
|
||||
" if (vk.id) pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" if (vk.value) pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {"mode": "raw", "raw": "{}"},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["api", "governance", "virtual-keys"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion #1",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Request 1: 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion #2",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Request 2: 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion #3 (expect 429)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Request 3: 429 rate limited', function() { pm.expect(code).to.equal(429); });",
|
||||
"if (code === 429) {",
|
||||
" var json = pm.response.json();",
|
||||
" var errType = (json.type || (json.error && json.error.type) || '').toString();",
|
||||
" if (errType) {",
|
||||
" pm.test('Error type indicates rate limit', function() {",
|
||||
" pm.expect(errType).to.match(/request_limited|rate_limited|token_limited/);",
|
||||
" });",
|
||||
" }",
|
||||
" pm.test('Error has message', function() {",
|
||||
" pm.expect(json.error).to.be.an('object');",
|
||||
" pm.expect(json.error.message).to.be.a('string').and.not.be.empty;",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Teardown - Delete VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var vkId = pm.collectionVariables.get('vk_id');",
|
||||
"if (!vkId) { pm.test('Teardown skipped - no VK to delete', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["api", "governance", "virtual-keys", "{{vk_id}}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Session Stickiness",
|
||||
"description": "Session stickiness tests. Validates x-bf-session-id and x-bf-session-ttl headers are accepted.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "session_id", "value": "", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion with session ID",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var sid = 'test-session-' + Date.now();",
|
||||
"pm.collectionVariables.set('session_id', sid);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-session-id", "value": "{{session_id}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion with same session ID",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-session-id", "value": "{{session_id}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion with different session ID",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-session-id", "value": "test-session-other-{{$timestamp}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion with session TTL",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-session-id", "value": "test-session-ttl"},
|
||||
{"key": "x-bf-session-ttl", "value": "60"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Streaming",
|
||||
"description": "Streaming SSE tests for inference endpoints. Validates Content-Type, data: lines, and [DONE] marker.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "responses_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "embedding_model", "value": "text-embedding-3-small", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion (stream)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" pm.test('Content-Type is SSE', function() {",
|
||||
" var ct = pm.response.headers.get('Content-Type') || '';",
|
||||
" pm.expect(ct).to.include('text/event-stream');",
|
||||
" });",
|
||||
" var body = pm.response.text();",
|
||||
" pm.test('Body contains data lines', function() { pm.expect(body).to.include('data:'); });",
|
||||
" pm.test('Stream ends with DONE', function() { pm.expect(body).to.include('[DONE]'); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": true\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Responses (stream)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" pm.test('Content-Type is SSE', function() {",
|
||||
" var ct = pm.response.headers.get('Content-Type') || '';",
|
||||
" pm.expect(ct).to.include('text/event-stream');",
|
||||
" });",
|
||||
" var body = pm.response.text();",
|
||||
" pm.test('Body contains data lines', function() { pm.expect(body).to.include('data:'); });",
|
||||
" pm.test('Body contains event lines', function() { pm.expect(body).to.include('event:'); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{responses_model}}\",\n \"input\": \"Say hello\",\n \"max_output_tokens\": 50,\n \"stream\": true\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/responses",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "responses"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Virtual Key Auth",
|
||||
"description": "Virtual key authentication tests for inference endpoints. Self-provisions a VK, runs inference with/without VK, tests rejection cases, and cleans up.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openai",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "text-embedding-3-small",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "vk_value",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "vk_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "enforce_auth",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Setup",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create Virtual Key",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var timestamp = Date.now();",
|
||||
"var uniqueName = 'VK Auth Test ' + timestamp;",
|
||||
"pm.request.body.raw = JSON.stringify({name: uniqueName, provider_configs: [{provider: 'openai', weight: 1.0, allowed_models: ['*'], key_ids: ['*']}]});"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 200 or 201', function() { pm.expect(code).to.be.oneOf([200, 201]); });",
|
||||
"if (code === 200 || code === 201) {",
|
||||
" var jsonData = pm.response.json();",
|
||||
" var vk = (jsonData && (jsonData.virtual_key || jsonData)) || null;",
|
||||
" pm.test('VK has id and value', function() {",
|
||||
" pm.expect(vk).to.be.an('object');",
|
||||
" pm.expect(vk.id).to.be.a('string').and.not.be.empty;",
|
||||
" pm.expect(vk.value).to.be.a('string').and.not.be.empty;",
|
||||
" pm.expect(vk.value).to.match(/^sk-bf-/);",
|
||||
" });",
|
||||
" pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"name\": \"VK Auth Test\"}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Inference Without VK",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - No VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var enforceAuth = (pm.environment && pm.environment.get('enforce_auth')) || pm.collectionVariables.get('enforce_auth') || '';",
|
||||
"if (enforceAuth === '1' || String(enforceAuth).toLowerCase() === 'true') {",
|
||||
" pm.test('Without VK and enforce_auth: expect 401', function() { pm.expect(code).to.equal(401); });",
|
||||
"} else {",
|
||||
" pm.test('Without VK and no enforce_auth: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Embedding - No VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var enforceAuth = (pm.environment && pm.environment.get('enforce_auth')) || pm.collectionVariables.get('enforce_auth') || '';",
|
||||
"if (enforceAuth === '1' || String(enforceAuth).toLowerCase() === 'true') {",
|
||||
" pm.test('Without VK and enforce_auth: expect 401', function() { pm.expect(code).to.equal(401); });",
|
||||
"} else {",
|
||||
" pm.test('Without VK and no enforce_auth: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{embedding_model}}\",\n \"input\": \"Hello world\",\n \"encoding_format\": \"float\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/embeddings",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"embeddings"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Responses - No VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var enforceAuth = (pm.environment && pm.environment.get('enforce_auth')) || pm.collectionVariables.get('enforce_auth') || '';",
|
||||
"if (enforceAuth === '1' || String(enforceAuth).toLowerCase() === 'true') {",
|
||||
" pm.test('Without VK and enforce_auth: expect 401', function() { pm.expect(code).to.equal(401); });",
|
||||
"} else {",
|
||||
" pm.test('Without VK and no enforce_auth: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{responses_model}}\",\n \"input\": \"Say hello\",\n \"max_output_tokens\": 50\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/responses",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"responses"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Inference With VK (x-bf-vk)",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - x-bf-vk",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With valid VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Embedding - x-bf-vk",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With valid VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{embedding_model}}\",\n \"input\": \"Hello world\",\n \"encoding_format\": \"float\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/embeddings",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"embeddings"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Responses - x-bf-vk",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With valid VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{responses_model}}\",\n \"input\": \"Say hello\",\n \"max_output_tokens\": 50\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/responses",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"responses"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Inference With VK (Authorization Bearer)",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - Authorization Bearer",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With Bearer VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "Authorization",
|
||||
"value": "Bearer {{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Inference With VK (x-api-key)",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - x-api-key",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With x-api-key VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-api-key",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Rejection - Invalid VK",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - x-bf-vk invalid",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Invalid VK: expect 403', function() { pm.expect(code).to.equal(403); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "invalid-not-a-real-key"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - x-bf-vk nonexistent",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Nonexistent VK: expect 403', function() { pm.expect(code).to.equal(403); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "sk-bf-00000000-0000-0000-0000-000000000000"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Teardown - Deactivate",
|
||||
"item": [
|
||||
{
|
||||
"name": "Deactivate Virtual Key",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Deactivate VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"is_active\": false}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - Deactivated VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Deactivated VK: expect 403', function() { pm.expect(code).to.equal(403); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Teardown - Delete",
|
||||
"item": [
|
||||
{
|
||||
"name": "Delete Virtual Key",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - VK Governance Routing",
|
||||
"description": "VK provider_configs routing tests. Creates VK with provider restriction, validates routing via extra_fields.provider.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "vk_value", "value": "", "type": "string"},
|
||||
{"key": "vk_id", "value": "", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Setup - Create VK with provider config",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var ts = Date.now();",
|
||||
"var body = {",
|
||||
" name: 'Routing Test VK ' + ts,",
|
||||
" provider_configs: [{",
|
||||
" provider: pm.collectionVariables.get('provider') || 'openai',",
|
||||
" weight: 1.0,",
|
||||
" allowed_models: ['*'],",
|
||||
" key_ids: ['*']",
|
||||
" }]",
|
||||
"};",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var vk = json.virtual_key || json;",
|
||||
" if (vk.id) pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" if (vk.value) pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {"mode": "raw", "raw": "{}"},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["api", "governance", "virtual-keys"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - model without provider prefix",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var extra = json.extra_fields || {};",
|
||||
" var providerUsed = extra.provider;",
|
||||
" var expected = (pm.collectionVariables.get('provider') || 'openai').toLowerCase();",
|
||||
" pm.test('Provider matches VK config', function() {",
|
||||
" pm.expect(String(providerUsed).toLowerCase()).to.equal(expected);",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - explicit provider prefix",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var extra = json.extra_fields || {};",
|
||||
" var providerUsed = extra.provider;",
|
||||
" var expected = (pm.collectionVariables.get('provider') || 'openai').toLowerCase();",
|
||||
" pm.test('Provider matches VK config (explicit prefix)', function() {",
|
||||
" pm.expect(String(providerUsed).toLowerCase()).to.equal(expected);",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - blocked model (expect 403)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Model blocked - expect 4xx', function() { pm.expect(code).to.be.oneOf([400, 403]); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"nonexistent-model\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Teardown - Delete VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["api", "governance", "virtual-keys", "{{vk_id}}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user