first commit
This commit is contained in:
Beyhan Oğur
1
tests/e2e/.npmrc
Normal file
1
tests/e2e/.npmrc
Normal file
@@ -0,0 +1 @@
|
||||
save-exact=true
|
||||
213
tests/e2e/README.md
Normal file
213
tests/e2e/README.md
Normal file
@@ -0,0 +1,213 @@
|
||||
# Bifrost E2E Tests
|
||||
|
||||
End-to-end tests for the Bifrost UI using Playwright.
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
npm install
|
||||
|
||||
# Install Playwright browsers
|
||||
npx playwright install
|
||||
```
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
# Run all E2E tests
|
||||
make run-e2e
|
||||
|
||||
# Run specific feature tests
|
||||
make run-e2e FLOW=providers
|
||||
make run-e2e FLOW=virtual-keys
|
||||
make run-e2e FLOW=dashboard
|
||||
make run-e2e FLOW=logs
|
||||
make run-e2e FLOW=mcp-logs
|
||||
make run-e2e FLOW=mcp-registry
|
||||
make run-e2e FLOW=routing-rules
|
||||
make run-e2e FLOW=observability
|
||||
make run-e2e FLOW=config
|
||||
make run-e2e FLOW=plugins
|
||||
|
||||
# Run tests in headed mode (visible browser)
|
||||
make run-e2e-headed
|
||||
|
||||
# Run tests with Playwright UI
|
||||
make run-e2e-ui
|
||||
|
||||
# Run specific feature tests via npm
|
||||
npm run test:providers
|
||||
npm run test:virtual-keys
|
||||
npm run test:dashboard
|
||||
npm run test:logs
|
||||
npm run test:mcp-logs
|
||||
npm run test:mcp-registry
|
||||
npm run test:routing-rules
|
||||
npm run test:observability
|
||||
npm run test:config
|
||||
npm run test:plugins
|
||||
|
||||
# View test report
|
||||
npm run report
|
||||
```
|
||||
|
||||
### Parallel flows on CI
|
||||
|
||||
The GitHub Actions workflow **E2E Tests** (`.github/workflows/e2e-tests.yml`) runs each flow in a **separate job in parallel**, since flows are independent. It triggers on push/PR when `ui/`, `tests/e2e/`, or the workflow file change. You can also run it manually (Actions → E2E Tests → Run workflow) and optionally pass a comma-separated list of flows (e.g. `providers,config,plugins`) to run only those.
|
||||
|
||||
## Folder Structure
|
||||
|
||||
```text
|
||||
tests/e2e/
|
||||
├── playwright.config.ts # Playwright configuration
|
||||
├── core/ # Shared utilities & fixtures
|
||||
│ ├── fixtures/ # Custom test fixtures
|
||||
│ ├── pages/ # Base page objects
|
||||
│ ├── actions/ # Reusable actions
|
||||
│ └── utils/ # Utilities and helpers
|
||||
└── features/ # Feature-specific tests
|
||||
├── providers/ # Provider tests
|
||||
├── virtual-keys/ # Virtual key tests
|
||||
├── dashboard/ # Dashboard tests
|
||||
├── logs/ # LLM logs tests
|
||||
├── mcp-logs/ # MCP logs tests
|
||||
├── mcp-registry/ # MCP registry tests
|
||||
├── routing-rules/ # Routing rules tests
|
||||
├── plugins/ # Plugins tests
|
||||
├── observability/ # Observability connectors tests
|
||||
└── config/ # Config settings tests
|
||||
```
|
||||
|
||||
## Writing Tests
|
||||
|
||||
### Using Page Objects
|
||||
|
||||
```typescript
|
||||
import { test, expect } from '../../core/fixtures/base.fixture'
|
||||
|
||||
test('should create provider', async ({ providersPage }) => {
|
||||
await providersPage.goto()
|
||||
await providersPage.selectProvider('openai')
|
||||
// ...
|
||||
})
|
||||
```
|
||||
|
||||
### Test Data
|
||||
|
||||
Use factory functions from the `*.data.ts` files for generating test data:
|
||||
|
||||
```typescript
|
||||
import { createProviderKeyData } from './providers.data'
|
||||
|
||||
const keyData = createProviderKeyData({ name: 'My Key' })
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Environment variables:
|
||||
- `BASE_URL` - Base URL of the application (default: http://localhost:3000)
|
||||
- `CI` - Set to true in CI environments
|
||||
|
||||
## Debugging
|
||||
|
||||
```bash
|
||||
# Run with Playwright Inspector
|
||||
npm run test:debug
|
||||
|
||||
# Generate code with Codegen
|
||||
npm run codegen
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Wait Strategies
|
||||
|
||||
Use semantic waits instead of hardcoded timeouts:
|
||||
|
||||
```typescript
|
||||
// ✅ Good: Semantic waits
|
||||
await page.waitForLoadState('networkidle')
|
||||
await element.waitFor({ state: 'visible' })
|
||||
await expect(element).toBeVisible({ timeout: 5000 })
|
||||
|
||||
// ❌ Bad: Hardcoded timeouts (flaky and slow)
|
||||
await page.waitForTimeout(2000)
|
||||
```
|
||||
|
||||
### Selectors
|
||||
|
||||
Use `data-testid` attributes for robust selectors:
|
||||
|
||||
```typescript
|
||||
// ✅ Good: Test IDs are resilient to UI changes
|
||||
page.locator('[data-testid="chart-log-volume"]')
|
||||
page.getByTestId('create-btn')
|
||||
|
||||
// ❌ Bad: Brittle chained parent selectors
|
||||
page.locator('text=Volume').locator('..').locator('..')
|
||||
```
|
||||
|
||||
### Resource Cleanup
|
||||
|
||||
Always clean up resources created during tests:
|
||||
|
||||
```typescript
|
||||
// ✅ Good: Clean up after assertions
|
||||
test('should create item', async ({ page }) => {
|
||||
await page.createItem(data)
|
||||
expect(await page.itemExists(data.name)).toBe(true)
|
||||
// Cleanup
|
||||
await page.deleteItem(data.name)
|
||||
})
|
||||
```
|
||||
|
||||
### Deterministic Assertions
|
||||
|
||||
Avoid conditional logic that always passes:
|
||||
|
||||
```typescript
|
||||
// ❌ Bad: Always passes (count >= 0 is always true)
|
||||
const count = await page.getCount()
|
||||
expect(count >= 0).toBe(true)
|
||||
|
||||
// ✅ Good: Deterministic assertion
|
||||
const count = await page.getCount()
|
||||
if (count === 0) {
|
||||
expect(emptyState).toBeVisible()
|
||||
} else {
|
||||
expect(count).toBeGreaterThan(0)
|
||||
expect(emptyState).not.toBeVisible()
|
||||
}
|
||||
```
|
||||
|
||||
## Anti-Patterns to Avoid
|
||||
|
||||
1. **`waitForTimeout()`** - Always use semantic waits instead
|
||||
2. **`{ force: true }`** - Fix underlying visibility issues instead
|
||||
3. **Chained parent locators** (`.locator('..')`) - Use `data-testid` attributes
|
||||
4. **Conditional assertions that always pass** - Write deterministic tests
|
||||
5. **Static test data names** - Use timestamps for uniqueness
|
||||
6. **Missing cleanup** - Delete created resources to prevent pollution
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Tests Failing Intermittently
|
||||
|
||||
1. Replace `waitForTimeout()` with proper semantic waits
|
||||
2. Ensure toasts are dismissed: `await page.dismissToasts()`
|
||||
3. Add `waitForPageLoad()` after navigation
|
||||
4. Wait for sheets/modals to complete animation: `await page.waitForSheetAnimation()`
|
||||
|
||||
### Tests Pass Individually but Fail Together
|
||||
|
||||
1. Add cleanup for created resources
|
||||
2. Use unique names with `Date.now()` timestamps
|
||||
3. Check for leftover state from previous tests
|
||||
|
||||
### Element Not Clickable
|
||||
|
||||
1. Ensure element is visible: `await element.waitFor({ state: 'visible' })`
|
||||
2. Scroll element into view: `await element.scrollIntoViewIfNeeded()`
|
||||
3. Dismiss overlaying toasts: `await page.dismissToasts()`
|
||||
4. Don't use `{ force: true }` - fix the root cause
|
||||
221
tests/e2e/api/README.md
Normal file
221
tests/e2e/api/README.md
Normal file
@@ -0,0 +1,221 @@
|
||||
# E2E API tests (Newman / Postman)
|
||||
|
||||
End-to-end API tests for the Bifrost API using Postman collections and [Newman](https://www.npmjs.com/package/newman) (CLI).
|
||||
|
||||
## Contents
|
||||
|
||||
### V1 Endpoint Tests
|
||||
|
||||
| Path | Description |
|
||||
|------|-------------|
|
||||
| `bifrost-v1-complete.postman_collection.json` | Postman collection: all `/v1` endpoints (models, chat, completions, responses, embeddings, audio, images, count tokens, batches, files, containers, MCP) |
|
||||
| `bifrost-v1.postman_environment.json` | Optional/legacy Postman environment (OpenAI). `run-newman-inference-tests.sh` uses **BIFROST_*** environment variables as the fallback when no provider-specific env file is passed (see script and `--help`). |
|
||||
| `run-newman-inference-tests.sh` | Script to run the V1 collection with Newman (single provider or all providers). |
|
||||
|
||||
### Integration Endpoint Tests
|
||||
|
||||
| Path | Description |
|
||||
|------|-------------|
|
||||
| `bifrost-openai-integration.postman_collection.json` | OpenAI integration endpoints: `/openai/v1/*`, `/openai/*`, `/openai/deployments/*` (38 requests) |
|
||||
| `bifrost-anthropic-integration.postman_collection.json` | Anthropic integration endpoints: `/anthropic/v1/*` (13 requests) |
|
||||
| `bifrost-bedrock-integration.postman_collection.json` | Bedrock integration endpoints: `/bedrock/model/*`, `/bedrock/files/*`, `/bedrock/model-invocation-*` (13 requests, including List Objects S3 ListObjectsV2). Auth via request headers set from collection/environment variables. |
|
||||
| `bifrost-composite-integrations.postman_collection.json` | Composite integrations: GenAI, Cohere, LiteLLM, LangChain, PydanticAI, Health (21 requests) |
|
||||
| `run-newman-openai-integration.sh` | Script to run OpenAI integration tests |
|
||||
| `run-newman-anthropic-integration.sh` | Script to run Anthropic integration tests |
|
||||
| `run-newman-bedrock-integration.sh` | Script to run Bedrock integration tests |
|
||||
| `run-newman-composite-integration.sh` | Script to run composite integration tests |
|
||||
| `run-all-integration-tests.sh` | Master script to run all integration test suites |
|
||||
|
||||
### Shared Resources
|
||||
|
||||
| Path | Description |
|
||||
|------|-------------|
|
||||
| `provider_config/` | Per-provider Postman env `.json` files (`bifrost-v1-openai.postman_environment.json`, etc.). Reused across all collections. |
|
||||
| `provider-capabilities.json` | Provider capability matrix: per-provider map of booleans (e.g. `chat_completions: true`, `embedding: false`) for batch, file, container, embedding, speech, transcription, image. Derived from `core/providers/*/provider.go` NewUnsupportedOperationError. Used by integration collections to skip unsupported requests when run with all providers. |
|
||||
| `fixtures/` | Sample files for multipart requests: `sample.mp3`, `sample.jsonl`, `sample.txt` |
|
||||
| `setup-plugin.sh` | Builds the hello-world plugin for API Management plugin tests. Run automatically by API Management and all-integration runners. |
|
||||
| `setup-mcp.sh` | Starts the test MCP server (`examples/mcps/http-no-ping-server`) on http://localhost:3001/ so Add/Update/Delete MCP Client tests can pass. Run automatically by API Management and all-integration runners. |
|
||||
| `newman-reports/` | Test reports organized by collection type (e.g., `openai-integration/`, `anthropic-integration/`). HTML/JSON reports when using `--html` / `--json`. |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Newman](https://www.npmjs.com/package/newman): `npm install -g newman`
|
||||
- Bifrost server running (e.g. `http://localhost:8080`) with at least one provider configured (API keys, etc.)
|
||||
|
||||
## Test infrastructure setup
|
||||
|
||||
Before running **API Management** or **all integration** tests, the runners optionally run:
|
||||
|
||||
- **`setup-plugin.sh`** – Builds `examples/plugins/hello-world` into `build/hello-world.so` (native OS/arch). If the plugin fails to build, plugin tests may fail with "plugin not found" / "failed to load"; those failures are treated as expected when the plugin is missing.
|
||||
- **`setup-mcp.sh`** – Builds and starts the test MCP server (`examples/mcps/http-no-ping-server`) on **http://localhost:3001/** so the collection’s test MCP client (connection string `http://localhost:3001/`) can connect. If the server is already listening on 3001 or the script is skipped, MCP client tests accept 404/500 as fallback.
|
||||
|
||||
Both are called automatically by `runners/run-newman-api-tests.sh` and `runners/run-all-integration-tests.sh`.
|
||||
|
||||
To run setup manually (from this directory):
|
||||
|
||||
```bash
|
||||
./setup-plugin.sh
|
||||
./setup-mcp.sh
|
||||
```
|
||||
|
||||
No Weaviate/cache setup is required: tests accept 405 for unimplemented cache endpoints.
|
||||
|
||||
## Run tests
|
||||
|
||||
From this directory (`tests/e2e/api`):
|
||||
|
||||
### V1 Endpoint Tests
|
||||
|
||||
```bash
|
||||
# Run for all providers in parallel (each provider_config/bifrost-v1-*.postman_environment.json except sgl and ollama)
|
||||
./runners/run-newman-inference-tests.sh
|
||||
|
||||
# Run for a single provider (by name or path to .json env)
|
||||
./runners/run-newman-inference-tests.sh --env openai
|
||||
./runners/run-newman-inference-tests.sh --env provider_config/bifrost-v1-openai.postman_environment.json
|
||||
|
||||
# Options
|
||||
./runners/run-newman-inference-tests.sh --help
|
||||
./runners/run-newman-inference-tests.sh --folder "Chat Completions"
|
||||
./runners/run-newman-inference-tests.sh --html --verbose
|
||||
```
|
||||
|
||||
**Retry logic (CI)**
|
||||
When `CI=1` or `CI=true` is set (case-insensitive), each failing request in the V1 collection is retried up to 3 times before moving to the next request. This helps with flaky tests in CI. The runner passes the value through to Newman when the environment variable is set (e.g. `CI=1 ./runners/run-newman-inference-tests.sh --env openai` or `CI=true ./runners/run-newman-inference-tests.sh --env openai`). Retry attempts are logged to the console as `[RETRY] Request "..." failed (attempt n/3). Retrying...`.
|
||||
|
||||
### Integration Endpoint Tests
|
||||
|
||||
```bash
|
||||
# Run all integration test suites for all providers
|
||||
./run-all-integration-tests.sh
|
||||
|
||||
# Run all integration test suites for a single provider
|
||||
./run-all-integration-tests.sh --env openai
|
||||
|
||||
# Run a specific integration test suite
|
||||
./run-newman-openai-integration.sh # OpenAI integration endpoints
|
||||
./run-newman-anthropic-integration.sh # Anthropic integration endpoints
|
||||
./run-newman-bedrock-integration.sh # Bedrock integration endpoints
|
||||
./run-newman-composite-integration.sh # Composite integrations + Health
|
||||
|
||||
# Run with options
|
||||
./run-newman-openai-integration.sh --html --verbose
|
||||
./run-newman-openai-integration.sh --env azure # Test Azure-specific paths
|
||||
```
|
||||
|
||||
### Test Success Criteria
|
||||
|
||||
A request **passes** if either:
|
||||
- The response status is 2xx, or
|
||||
- The response is 4xx/5xx but the error indicates the operation is not supported by the provider (e.g. `error.code === "unsupported_operation"` or message like "operation is not supported" / "not supported by X provider").
|
||||
|
||||
Any other non-2xx (e.g. 401 with a wrong API key) fails the test.
|
||||
|
||||
**V1 collection ("documented unsupported" assertion)**
|
||||
The **"Or documented unsupported (allowed request types)"** test passes only when the request’s operation category is marked as unsupported for the current provider in **`provider-capabilities.json`** (`providers.<name>.<operation> === false`). The request name is mapped to one of: `chat_completions`, `chat_completions_with_tools`, `text_completion`, `responses`, `responses_with_tools`, `count_tokens`, `batch_create`, `batch_create_file`, `batch_list`, `batch_retrieve`, `batch_cancel`, `batch_results`, `file_upload`, `file_batch_input`, `file_list`, `file_retrieve`, `file_delete`, `file_content`, `container_create`, `container_list`, `container_retrieve`, `container_delete`, `container_file_create`, `container_file_create_reference`, `container_file_list`, `container_file_retrieve`, `container_file_content`, `container_file_delete`, `embedding`, `speech`, `transcription`, `list_models`, `image_generation`, `image_variation`, `image_edit`, `video_generation`, `video_retrieve`, `video_download`, `video_delete`, `video_list`, `video_remix`, `rerank`. These match the operation types in `core/schemas/bifrost.go` (e.g. `FileUploadRequest`, `ContainerFileContentRequest`). **`provider-capabilities.json` is the single source of truth:** the V1 run script (`run-newman-tests.sh`) loads it at run time and passes it to Newman as globals; the collection does not define or embed `provider_capabilities`.
|
||||
|
||||
### Expected failures (known limitations)
|
||||
|
||||
Some failures are expected and do not indicate bugs:
|
||||
|
||||
- **Authentication (401)** – Provider envs may use placeholder or invalid API keys; 401 is then expected. Some OpenAI integration endpoints may show 401 even with valid keys if keys are not configured for all endpoint types.
|
||||
|
||||
- **Batch API config (500)** – **"no batch-enabled keys found"** / **"no config found for batch APIs"** when batch endpoints are not configured for that provider.
|
||||
|
||||
**To fix:** In Bifrost's config (or provider config), enable "Use for Batch APIs" on at least one API key for the provider, e.g. in config JSON:
|
||||
```json
|
||||
{
|
||||
"providers": [
|
||||
{
|
||||
"name": "openai",
|
||||
"api_keys": [
|
||||
{
|
||||
"key": "sk-...",
|
||||
"use_for_batch_apis": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- **Model incompatibility** – Some models do not support certain operations (e.g. Azure gpt-4o does not support text completions, OpenAI chat models cannot be used for text completions); these may return 400 errors.
|
||||
|
||||
- **Responses API with tools** – The V1 "Create Response with Tools" test uses only a function tool (no `web_search`). Using `web_search` or other tool types can trigger 500 errors from the provider (OpenAI has had known 500s with web search on the Responses API).
|
||||
|
||||
- **Bedrock** – Model (converse/invoke) or S3 file operations may fail with 403/500 if AWS keys or S3 are not configured. The Bedrock **integration** collection (`bifrost-bedrock-integration.postman_collection.json`) tests `/bedrock/*` and supports auth via **request headers** (set from collection or environment variables by the collection’s pre-request script). Credentials can be provided via env vars (e.g. `BIFROST_BEDROCK_API_KEY`, `BIFROST_BEDROCK_ACCESS_KEY`, `BIFROST_BEDROCK_SECRET_KEY`, `BIFROST_BEDROCK_REGION`) when using the runner with `--env bedrock`; the runner passes these into Postman variables, which the pre-request script forwards as `x-bf-bedrock-*` headers. Set authentication in `provider_config/bifrost-v1-bedrock.postman_environment.json` or collection variables:
|
||||
- **Option A:** `bedrock_api_key` (API key authentication) and optionally `bedrock_region` (default: us-east-1)
|
||||
- **Option B:** `bedrock_access_key`, `bedrock_secret_key`, `bedrock_region` (required), and optionally `bedrock_session_token` (for temporary credentials)
|
||||
- For S3 operations: set `s3_bucket` and `s3_key` to a bucket you have access to; List Objects (GET `/bedrock/files/{bucket}`) is included and supports optional query params (`prefix`, `max-keys`, `continuation-token`)
|
||||
- For batch operations: set `role_arn` to an IAM role with Bedrock batch permissions; ensure `inputDataConfig` and `outputDataConfig` S3 URIs exist
|
||||
- The V1 collection skips file/batch requests when `file_id` / `batch_id` are placeholders
|
||||
|
||||
- **Composite integrations** – Cohere/OpenAI/Nebius etc. can show 401/402/500 due to keys, billing, or provider limits (e.g. tool calling, embeddings).
|
||||
|
||||
- **Plugin tests** – If `setup-plugin.sh` did not build the hello-world plugin, Create/Get/Update Plugin tests may fail with "plugin not found" / "failed to load"; the test suite treats these as acceptable when the plugin is missing.
|
||||
|
||||
## Integration Endpoint Testing Strategy
|
||||
|
||||
### Native Integration Endpoints
|
||||
|
||||
Each major provider has its own integration test collection that tests provider-specific endpoint patterns:
|
||||
|
||||
- **OpenAI Integration** (`/openai/*`): Tests standard paths (`/openai/v1/chat/completions`), no-v1 paths (`/openai/chat/completions`), and Azure deployment paths (`/openai/deployments/{deployment-id}/chat/completions`). Covers 38 endpoints including chat, completions, embeddings, audio, images, batches, files, and containers.
|
||||
|
||||
- **Anthropic Integration** (`/anthropic/*`): Tests Anthropic-specific paths with different batch result endpoint pattern (`/anthropic/v1/messages/batches/{batch_id}/results` vs OpenAI's pattern). Covers 13 endpoints including messages, complete, count tokens, batches, and files.
|
||||
|
||||
- **Bedrock Integration** (`/bedrock/*`): Tests AWS Bedrock patterns with ARN-based batch job identifiers and S3 file operations. Covers 13 endpoints including converse, invoke, batch jobs, S3 operations (PUT/GET/HEAD/DELETE Object and List Objects), and List Batch Jobs with optional query params.
|
||||
|
||||
### Composite Integration Testing (Delegation)
|
||||
|
||||
The composite integrations collection tests **routing** for frameworks that delegate to other integrations:
|
||||
|
||||
- **LiteLLM, LangChain, PydanticAI**: These are pass-through routers that prefix requests with their framework name, then delegate to the underlying integration. For example:
|
||||
- `POST /litellm/v1/chat/completions` → delegates to OpenAI integration logic
|
||||
- `POST /litellm/anthropic/v1/messages` → delegates to Anthropic integration logic
|
||||
- `POST /langchain/bedrock/model/{model}/converse` → delegates to Bedrock integration logic
|
||||
|
||||
Rather than duplicating 100+ tests for each composite integration, we test **representative routes** (5 per composite) to validate routing works correctly. Comprehensive endpoint coverage is provided by the base integration tests.
|
||||
|
||||
- **GenAI**: Tests Google Gemini API format endpoints (2 requests)
|
||||
- **Cohere**: Tests Cohere API format endpoints (3 requests)
|
||||
- **Health**: Tests the `/health` endpoint (1 request)
|
||||
|
||||
### Skipping unsupported operations (integration collections)
|
||||
|
||||
When integration tests are run for **all providers** (e.g. `./run-newman-openai-integration.sh` without `--env`), each collection is executed once per provider environment. Some providers do not support batch, file, container, embedding, audio, or image operations. To avoid failing on those requests, each integration collection has:
|
||||
|
||||
- **Collection-level prerequest**: Runs before every request. Reads the current **provider** from the environment and the **request name**. If the request maps to an operation category for which the provider has `false` in `provider_capabilities` (e.g. `providers.anthropic.embedding === false`), the request is **skipped** via `postman.setNextRequest(nextRequestName)` so the next request in execution order runs instead.
|
||||
- **Embedded variables**: `execution_order` (JSON array of request names in depth-first order) and `request_to_operation` (JSON map of request name → operation category). For the **V1** collection, `provider_capabilities` is **not** embedded: it is loaded from `provider-capabilities.json` at run time by `run-newman-inference-tests.sh` and passed to Newman as globals.
|
||||
|
||||
**Config file**: `provider-capabilities.json` in this directory is a map per provider of capability flags (e.g. `chat_completions`, `embedding`, `batch`) to booleans. It is the single source of truth for which operations each provider supports (aligned with `core/providers/*/provider.go` returning `NewUnsupportedOperationError`).
|
||||
|
||||
**Updating capabilities or request mappings**:
|
||||
|
||||
1. **Change provider support**: Edit `provider-capabilities.json` and set each capability to `true` or `false` under `providers.<name>` (e.g. `providers.anthropic.embedding: false`). It is the only source of truth; the V1 inference run script loads it at run time (no embedded copy in the collection).
|
||||
2. **Change which requests are skippable**: Edit `scripts/update-collection-capabilities.js` (function `getRequestToOperationMap`) to adjust the request-name → operation map for each collection.
|
||||
3. **Re-inject variables into a collection**: From this directory run:
|
||||
```bash
|
||||
node scripts/update-collection-capabilities.js bifrost-openai-integration.postman_collection.json --inject
|
||||
```
|
||||
This re-extracts execution order, re-reads `provider-capabilities.json`, and overwrites the collection variables and the prerequest script. Run for each integration collection you changed.
|
||||
|
||||
### Batches, Files, Containers (mirror core tests)
|
||||
|
||||
Execution order and request shapes match the core Go tests (`core/internal/llmtests/batch.go`, `containers.go`):
|
||||
|
||||
- **Files** run first: Upload File (sets `file_id`), List, Retrieve, Get Content. No delete yet so the file can be used by Batches.
|
||||
- **Batches**: Create Batch (Inline) sets `batch_id`; Create Batch (File-based) uses `file_id`; List (with `limit=10`), Retrieve, Cancel, Results use `batch_id`; then Delete File.
|
||||
- **Containers**: Create Container sets `container_id`; List, Retrieve; Create Container File (Upload) sets `container_file_id`; List/Retrieve/Content/Delete container file use `container_file_id`; Delete Container last.
|
||||
|
||||
Request bodies match core (e.g. batch inline with `custom_id`/`body`/`Say hello`, container create with `name: "bifrost-test-container"`).
|
||||
|
||||
## Syncing from OpenAPI docs
|
||||
|
||||
The collection and supporting files are maintained under `docs/openapi/`. To refresh this e2e copy:
|
||||
|
||||
- `bifrost-v1-complete.postman_collection.json` ← `docs/openapi/bifrost-v1-complete.postman_collection.json`
|
||||
- `bifrost-v1.postman_environment.json` ← `docs/openapi/bifrost-v1.postman_environment.json`
|
||||
- `runners/run-newman-inference-tests.sh` ← `docs/openapi/run-newman-inference-tests.sh`
|
||||
- `provider_config/*.postman_environment.json` and `provider_config/README.md` ← `docs/openapi/provider_config/` (if syncing from docs)
|
||||
- `fixtures/*` ← `docs/openapi/fixtures/`
|
||||
72
tests/e2e/api/bifrost-v1.postman_environment.json
Normal file
72
tests/e2e/api/bifrost-v1.postman_environment.json
Normal file
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "bifrost-v1-env",
|
||||
"name": "Bifrost V1 (default \u2013 OpenAI)",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openai",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "gpt-4o",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "text-embedding-3-small",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "tts-1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "whisper-1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "gpt-image-1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "voice",
|
||||
"value": "alloy",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,662 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost Anthropic Integration API",
|
||||
"description": "E2E tests for Anthropic integration endpoints",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var requestName = pm.info.requestName;",
|
||||
"var retryKey = 'retry_' + requestName;",
|
||||
"var ciVal = pm.environment.get('CI');",
|
||||
"var maxRetries = (String(ciVal).toLowerCase() === 'true' || ciVal === '1' || ciVal === 1) ? 10 : 0;",
|
||||
"if (!pm.collectionVariables.has(retryKey)) {",
|
||||
" pm.collectionVariables.set(retryKey, 0);",
|
||||
"}",
|
||||
"pm.collectionVariables.set('current_request_name', requestName);",
|
||||
"pm.collectionVariables.set('max_retries', maxRetries);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var provider = (((pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || '') + '').toLowerCase();",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : '';",
|
||||
"var capsStr = pm.collectionVariables.get('provider_capabilities') || '{}';",
|
||||
"var execOrderStr = pm.collectionVariables.get('execution_order') || '[]';",
|
||||
"var requestToOpStr = pm.collectionVariables.get('request_to_operation') || '{}';",
|
||||
"try {",
|
||||
" var caps = JSON.parse(capsStr);",
|
||||
" var execOrder = JSON.parse(execOrderStr);",
|
||||
" var requestToOp = JSON.parse(requestToOpStr);",
|
||||
" var providerCaps = caps.providers && caps.providers[provider];",
|
||||
" var op = requestToOp[requestName];",
|
||||
" if (provider && op && providerCaps && providerCaps[op] === false) {",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') {",
|
||||
" pm.execution.skipRequest();",
|
||||
" }",
|
||||
" var idx = execOrder.indexOf(requestName);",
|
||||
" var nextName = (idx >= 0 && idx < execOrder.length - 1) ? execOrder[idx + 1] : null;",
|
||||
" if (nextName) {",
|
||||
" if (pm.execution && typeof pm.execution.setNextRequest === 'function') {",
|
||||
" pm.execution.setNextRequest(nextName);",
|
||||
" } else if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest(nextName);",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var requestNameRetry = pm.collectionVariables.get('current_request_name');",
|
||||
"var retryKey = 'retry_' + requestNameRetry;",
|
||||
"var currentRetry = parseInt(pm.collectionVariables.get(retryKey) || '0', 10);",
|
||||
"var maxRetries = parseInt(pm.collectionVariables.get('max_retries') || '0', 10);",
|
||||
"var hasFailures = code < 200 || code > 299;",
|
||||
"if (hasFailures && maxRetries > 0 && currentRetry < maxRetries) {",
|
||||
" pm.collectionVariables.set(retryKey, String(currentRetry + 1));",
|
||||
" var delayMs = Math.min(1000 * Math.pow(2, currentRetry), 30000);",
|
||||
" var end = Date.now() + delayMs; while (Date.now() < end) {}",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed (attempt ' + (currentRetry + 1) + '/' + maxRetries + '). Retrying after ' + delayMs + 'ms...');",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(requestNameRetry); }",
|
||||
"} else {",
|
||||
" pm.collectionVariables.set(retryKey, '0');",
|
||||
" if (hasFailures && currentRetry >= maxRetries && maxRetries > 0) {",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed after ' + maxRetries + ' retries. Moving on.');",
|
||||
" }",
|
||||
" ",
|
||||
"// Helper function to pretty print JSON",
|
||||
"function prettyPrintJSON(jsonString) {",
|
||||
" try {",
|
||||
" var parsed = typeof jsonString === 'string' ? JSON.parse(jsonString) : jsonString;",
|
||||
" return JSON.stringify(parsed, null, 2);",
|
||||
" } catch (e) {",
|
||||
" return jsonString;",
|
||||
" }",
|
||||
"}",
|
||||
"function redact(obj) {",
|
||||
" if (!obj || typeof obj !== 'object') return obj;",
|
||||
" if (Array.isArray(obj)) return obj.map(redact);",
|
||||
" var out = {};",
|
||||
" Object.keys(obj).forEach(function (k) {",
|
||||
" if (/password|secret|token|api[_-]?key|authorization/i.test(k)) out[k] = '***REDACTED***';",
|
||||
" else out[k] = redact(obj[k]);",
|
||||
" });",
|
||||
" return out;",
|
||||
"}",
|
||||
"",
|
||||
"// Log request details",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : 'Unknown Request';",
|
||||
"var requestMethod = pm.request && pm.request.method ? pm.request.method : 'UNKNOWN';",
|
||||
"var requestUrl = pm.request && pm.request.url ? pm.request.url.toString() : 'Unknown URL';",
|
||||
"var requestBody = '';",
|
||||
"if (pm.request && pm.request.body && pm.request.body.raw) {",
|
||||
" try {",
|
||||
" var parsedReq = typeof pm.request.body.raw === 'string' ? JSON.parse(pm.request.body.raw) : pm.request.body.raw;",
|
||||
" requestBody = JSON.stringify(redact(parsedReq), null, 2);",
|
||||
" } catch (e) {",
|
||||
" requestBody = pm.request.body.raw;",
|
||||
" }",
|
||||
"}",
|
||||
"",
|
||||
"// Log response details",
|
||||
"var responseBody = '';",
|
||||
"var responseText = '';",
|
||||
"try {",
|
||||
" responseText = pm.response.text();",
|
||||
" if (responseText) {",
|
||||
" try {",
|
||||
" var parsedRes = JSON.parse(responseText);",
|
||||
" responseBody = JSON.stringify(redact(parsedRes), null, 2);",
|
||||
" } catch (e) {",
|
||||
" responseBody = responseText;",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {",
|
||||
" responseBody = pm.response.text() || '';",
|
||||
"}",
|
||||
"",
|
||||
"// Output formatted request/response logs (body content only when verbose_logs=1)",
|
||||
"var verbose = (pm.collectionVariables.get('verbose_logs') || '0') === '1';",
|
||||
"console.log('\\n' + '='.repeat(80));",
|
||||
"console.log('REQUEST: ' + requestMethod + ' ' + requestName);",
|
||||
"console.log('URL: ' + requestUrl);",
|
||||
"if (verbose && requestBody) {",
|
||||
" console.log('REQUEST BODY:');",
|
||||
" console.log(requestBody);",
|
||||
"}",
|
||||
"console.log('\\nRESPONSE: ' + pm.response.code + ' ' + pm.response.status);",
|
||||
"if (verbose && responseBody) {",
|
||||
" console.log('RESPONSE BODY:');",
|
||||
" console.log(responseBody);",
|
||||
"}",
|
||||
"console.log('='.repeat(80) + '\\n');",
|
||||
"",
|
||||
"var code = pm.response.code;",
|
||||
"var pass = (code >= 200 && code <= 299);",
|
||||
"if (!pass && code >= 400) {",
|
||||
" try {",
|
||||
" var body = pm.response.json();",
|
||||
" var errCode = (body && body.error && body.error.code) ? body.error.code : '';",
|
||||
" if (errCode === 'unsupported_operation' || errCode === 'feature_not_enabled') {",
|
||||
" pass = true;",
|
||||
" } else {",
|
||||
" var msg = (body && body.error && body.error.message) ? body.error.message : (body && body.message) ? body.message : '';",
|
||||
" if (typeof msg === 'string' && /\\b(not supported|unsupported|not enabled|not configured|does not support|doesn't support|cannot be used for|is not supported on this|incompatible)\\b/i.test(msg)) {",
|
||||
" pass = true;",
|
||||
" }",
|
||||
" }",
|
||||
" } catch (e) {}",
|
||||
"}",
|
||||
"",
|
||||
"// Do not swallow not-found for batch_id/file_id dependent requests",
|
||||
"var dependentBatchRequests = ['Retrieve Batch', 'Cancel Batch'];",
|
||||
"var dependentFileRequests = ['Get File Content', 'Delete File'];",
|
||||
"var isDependentOnBatchOrFile = dependentBatchRequests.indexOf(requestName) !== -1 || dependentFileRequests.indexOf(requestName) !== -1;",
|
||||
"var errMsg = ''; try { var errBody = pm.response.json(); errMsg = (errBody && errBody.error && errBody.error.message) ? errBody.error.message : (errBody && errBody.message) ? errBody.message : ''; } catch (e) {}",
|
||||
"var isNotFound = code === 404 || (typeof errMsg === 'string' && /not found|not_found|resource.*not found/i.test(errMsg));",
|
||||
"if (isDependentOnBatchOrFile && isNotFound) { pass = false; }",
|
||||
"",
|
||||
"// Allow 404 for Get Batch Results (batch may have no results yet)",
|
||||
"if (requestName === 'Get Batch Results' && code === 404) {",
|
||||
" pass = true;",
|
||||
"}",
|
||||
"",
|
||||
"pm.test('Status is 2xx or unsupported operation', function() { pm.expect(pass).to.be.true; });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "anthropic",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "claude-sonnet-4-5-20250929",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "execution_order",
|
||||
"value": "[\"List Models\",\"Create Message\",\"Create Completion\",\"Count Tokens\",\"Create Batch\",\"List Batches\",\"Retrieve Batch\",\"Cancel Batch\",\"Get Batch Results\",\"Upload File\",\"List Files\",\"Get File Content\",\"Delete File\"]",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider_capabilities",
|
||||
"value": "{\"description\":\"Provider capability matrix for integration tests. Each provider has explicit booleans per operation (derived from core/providers/* provider.go NewUnsupportedOperationError). Used to skip requests when running with all provider envs.\",\"providers\":{\"openai\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":true},\"anthropic\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":true,\"file\":true,\"container\":false},\"azure\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"bedrock\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"cerebras\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"cohere\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"elevenlabs\":{\"chat_completions\":true,\"embedding\":false,\"speech\":true,\"transcription\":true,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"gemini\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"groq\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"huggingface\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"mistral\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"nebius\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"openrouter\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"parasail\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"perplexity\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"replicate\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":true,\"container\":false},\"vertex\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"xai\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false}}}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "request_to_operation",
|
||||
"value": "{\"Create Batch\":\"batch\",\"List Batches\":\"batch\",\"Retrieve Batch\":\"batch\",\"Cancel Batch\":\"batch\",\"Get Batch Results\":\"batch\",\"Upload File\":\"file\",\"List Files\":\"file\",\"Get File Content\":\"file\",\"Delete File\":\"file\"}",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Models",
|
||||
"item": [
|
||||
{
|
||||
"name": "List Models",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/models",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"models"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Messages",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create Message",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"max_tokens\": 1024,\n \"temperature\": 0.7\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Complete",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create Completion",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"prompt\": \"\\n\\nHuman: Hello, Claude!\\n\\nAssistant:\",\n \"max_tokens_to_sample\": 256,\n \"temperature\": 0.7\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/complete",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"complete"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Count Tokens",
|
||||
"item": [
|
||||
{
|
||||
"name": "Count Tokens",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"How many tokens is this message?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/count_tokens",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"count_tokens"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Batches",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create Batch",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"pm.test('Create Batch: status is 2xx', function () { pm.expect(pm.response.code).to.be.at.least(200); pm.expect(pm.response.code).to.be.below(300); });",
|
||||
"var j = null; try { j = pm.response.json(); } catch (e) {}",
|
||||
"pm.test('Create Batch: response has id', function () { pm.expect(j).to.be.an('object'); pm.expect(j).to.have.property('id'); pm.expect(j.id).to.be.a('string'); pm.expect(j.id.length).to.be.above(0); });",
|
||||
"if (pm.response.code >= 200 && pm.response.code < 300 && j && j.id) { pm.collectionVariables.set('batch_id', j.id); if (pm.environment) { pm.environment.set('batch_id', j.id); } }"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"requests\": [\n {\n \"custom_id\": \"test-request-1\",\n \"params\": {\n \"model\": \"{{provider}}/{{model}}\",\n \"max_tokens\": 1024,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Say hello\"\n }\n ]\n }\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "List Batches",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Retrieve Batch",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var batch_id = pm.collectionVariables.get('batch_id') || (pm.environment && pm.environment.get('batch_id')) || '';",
|
||||
"if (!batch_id || batch_id.trim() === '') {",
|
||||
" pm.test('batch_id must be set by Create Batch request before Retrieve Batch', function () { pm.expect.fail('batch_id is missing; run Create Batch first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches/{{batch_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches",
|
||||
"{{batch_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Cancel Batch",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var batch_id = pm.collectionVariables.get('batch_id') || (pm.environment && pm.environment.get('batch_id')) || '';",
|
||||
"if (!batch_id || batch_id.trim() === '') {",
|
||||
" pm.test('batch_id must be set by Create Batch request before Cancel Batch', function () { pm.expect.fail('batch_id is missing; run Create Batch first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches/{{batch_id}}/cancel",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches",
|
||||
"{{batch_id}}",
|
||||
"cancel"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get Batch Results",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var batch_id = pm.collectionVariables.get('batch_id') || (pm.environment && pm.environment.get('batch_id')) || '';",
|
||||
"if (!batch_id || batch_id.trim() === '') {",
|
||||
" pm.test('batch_id must be set by Create Batch request before Get Batch Results', function () { pm.expect.fail('batch_id is missing; run Create Batch first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/messages/batches/{{batch_id}}/results",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages",
|
||||
"batches",
|
||||
"{{batch_id}}",
|
||||
"results"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Files",
|
||||
"item": [
|
||||
{
|
||||
"name": "Upload File",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"pm.test('Upload File: status is 2xx', function () { pm.expect(pm.response.code).to.be.at.least(200); pm.expect(pm.response.code).to.be.below(300); });",
|
||||
"var j = null; try { j = pm.response.json(); } catch (e) {}",
|
||||
"var id = (j && (j.id || j.file_id)) || (j && j.data && (j.data.id || j.data.file_id));",
|
||||
"pm.test('Upload File: response has id', function () { pm.expect(j).to.be.an('object'); pm.expect(id).to.be.a('string'); pm.expect(id.length).to.be.above(0); });",
|
||||
"if (pm.response.code >= 200 && pm.response.code < 300 && id) { pm.collectionVariables.set('file_id', id); if (pm.environment) { pm.environment.set('file_id', id); } }"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "formdata",
|
||||
"formdata": [
|
||||
{
|
||||
"key": "file",
|
||||
"type": "file",
|
||||
"src": "fixtures/sample.jsonl"
|
||||
},
|
||||
{
|
||||
"key": "purpose",
|
||||
"value": "batch",
|
||||
"type": "text"
|
||||
}
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/files",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"files"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "List Files",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/files",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"files"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get File Content",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var file_id = pm.collectionVariables.get('file_id') || (pm.environment && pm.environment.get('file_id')) || '';",
|
||||
"if (!file_id || file_id.trim() === '') {",
|
||||
" pm.test('file_id must be set by Upload File request before Get File Content', function () { pm.expect.fail('file_id is missing; run Upload File first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/files/{{file_id}}/content",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"files",
|
||||
"{{file_id}}",
|
||||
"content"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Delete File",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var file_id = pm.collectionVariables.get('file_id') || (pm.environment && pm.environment.get('file_id')) || '';",
|
||||
"if (!file_id || file_id.trim() === '') {",
|
||||
" pm.test('file_id must be set by Upload File request before Delete File', function () { pm.expect.fail('file_id is missing; run Upload File first.'); });",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') { pm.execution.skipRequest(); } else if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(null); }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/anthropic/v1/files/{{file_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"anthropic",
|
||||
"v1",
|
||||
"files",
|
||||
"{{file_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,950 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost Bedrock Integration API",
|
||||
"description": "E2E tests for Bedrock integration endpoints. Requires authentication: set bedrock_api_key (or bedrock_access_key, bedrock_secret_key, bedrock_region) in environment. S3 operations need a valid bucket. Batch operations need IAM role and S3 URIs.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var requestName = pm.info.requestName;",
|
||||
"var retryKey = 'retry_' + requestName;",
|
||||
"var ciVal = pm.environment.get('CI');",
|
||||
"var maxRetries = (String(ciVal).toLowerCase() === 'true' || ciVal === '1' || ciVal === 1) ? 10 : 0;",
|
||||
"if (!pm.collectionVariables.has(retryKey)) {",
|
||||
" pm.collectionVariables.set(retryKey, 0);",
|
||||
"}",
|
||||
"pm.collectionVariables.set('current_request_name', requestName);",
|
||||
"pm.collectionVariables.set('max_retries', maxRetries);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var getVar = function(k) { return (pm.environment && pm.environment.get(k)) || pm.collectionVariables.get(k) || ''; };",
|
||||
"var apiKey = getVar('bedrock_api_key');",
|
||||
"var accessKey = getVar('bedrock_access_key');",
|
||||
"var secretKey = getVar('bedrock_secret_key');",
|
||||
"var region = getVar('bedrock_region');",
|
||||
"var sessionToken = getVar('bedrock_session_token');",
|
||||
"if (apiKey) {",
|
||||
" pm.request.headers.upsert({ key: 'x-bf-bedrock-api-key', value: apiKey });",
|
||||
" if (region) { pm.request.headers.upsert({ key: 'x-bf-bedrock-region', value: region }); }",
|
||||
"} else if (accessKey && secretKey) {",
|
||||
" pm.request.headers.upsert({ key: 'x-bf-bedrock-access-key', value: accessKey });",
|
||||
" pm.request.headers.upsert({ key: 'x-bf-bedrock-secret-key', value: secretKey });",
|
||||
" pm.request.headers.upsert({ key: 'x-bf-bedrock-region', value: region || 'us-east-1' });",
|
||||
" if (sessionToken) { pm.request.headers.upsert({ key: 'x-bf-bedrock-session-token', value: sessionToken }); }",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var provider = (pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || '';",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : '';",
|
||||
"var capsStr = pm.collectionVariables.get('provider_capabilities') || '{}';",
|
||||
"var execOrderStr = pm.collectionVariables.get('execution_order') || '[]';",
|
||||
"var requestToOpStr = pm.collectionVariables.get('request_to_operation') || '{}';",
|
||||
"try {",
|
||||
" var caps = JSON.parse(capsStr);",
|
||||
" var execOrder = JSON.parse(execOrderStr);",
|
||||
" var requestToOp = JSON.parse(requestToOpStr);",
|
||||
" var providerCaps = caps.providers && caps.providers[provider];",
|
||||
" var op = requestToOp[requestName];",
|
||||
" if (provider && op && providerCaps && providerCaps[op] === false) {",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') {",
|
||||
" pm.execution.skipRequest();",
|
||||
" }",
|
||||
" var idx = execOrder.indexOf(requestName);",
|
||||
" var nextName = (idx >= 0 && idx < execOrder.length - 1) ? execOrder[idx + 1] : null;",
|
||||
" if (nextName) {",
|
||||
" if (pm.execution && typeof pm.execution.setNextRequest === 'function') {",
|
||||
" pm.execution.setNextRequest(nextName);",
|
||||
" } else if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest(nextName);",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var provider = (pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || '';",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : '';",
|
||||
"var capsStr = pm.collectionVariables.get('provider_capabilities') || '{}';",
|
||||
"var execOrderStr = pm.collectionVariables.get('execution_order') || '[]';",
|
||||
"var requestToOpStr = pm.collectionVariables.get('request_to_operation') || '{}';",
|
||||
"try {",
|
||||
" var caps = JSON.parse(capsStr);",
|
||||
" var execOrder = JSON.parse(execOrderStr);",
|
||||
" var requestToOp = JSON.parse(requestToOpStr);",
|
||||
" var providerCaps = caps.providers && caps.providers[provider];",
|
||||
" var op = requestToOp[requestName];",
|
||||
" if (provider && op && providerCaps && providerCaps[op] === false) {",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') {",
|
||||
" pm.execution.skipRequest();",
|
||||
" }",
|
||||
" var idx = execOrder.indexOf(requestName);",
|
||||
" var nextName = (idx >= 0 && idx < execOrder.length - 1) ? execOrder[idx + 1] : null;",
|
||||
" if (nextName) {",
|
||||
" if (pm.execution && typeof pm.execution.setNextRequest === 'function') {",
|
||||
" pm.execution.setNextRequest(nextName);",
|
||||
" } else if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest(nextName);",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var requestNameRetry = pm.collectionVariables.get('current_request_name');",
|
||||
"var retryKey = 'retry_' + requestNameRetry;",
|
||||
"var currentRetry = parseInt(pm.collectionVariables.get(retryKey) || '0', 10);",
|
||||
"var maxRetries = parseInt(pm.collectionVariables.get('max_retries') || '0', 10);",
|
||||
"var hasFailures = code < 200 || code > 299;",
|
||||
"if (hasFailures && maxRetries > 0 && currentRetry < maxRetries) {",
|
||||
" pm.collectionVariables.set(retryKey, String(currentRetry + 1));",
|
||||
" var delayMs = Math.min(1000 * Math.pow(2, currentRetry), 30000);",
|
||||
" var end = Date.now() + delayMs; while (Date.now() < end) {}",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed (attempt ' + (currentRetry + 1) + '/' + maxRetries + '). Retrying after ' + delayMs + 'ms...');",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(requestNameRetry); }",
|
||||
"} else {",
|
||||
" pm.collectionVariables.set(retryKey, '0');",
|
||||
" if (hasFailures && currentRetry >= maxRetries && maxRetries > 0) {",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed after ' + maxRetries + ' retries. Moving on.');",
|
||||
" }",
|
||||
" ",
|
||||
"// Helper function to pretty print JSON",
|
||||
"function prettyPrintJSON(jsonString) {",
|
||||
" try {",
|
||||
" var parsed = typeof jsonString === 'string' ? JSON.parse(jsonString) : jsonString;",
|
||||
" return JSON.stringify(parsed, null, 2);",
|
||||
" } catch (e) {",
|
||||
" return jsonString;",
|
||||
" }",
|
||||
"}",
|
||||
"function redact(obj) {",
|
||||
" if (!obj || typeof obj !== 'object') return obj;",
|
||||
" if (Array.isArray(obj)) return obj.map(redact);",
|
||||
" var out = {};",
|
||||
" Object.keys(obj).forEach(function (k) {",
|
||||
" if (/password|secret|token|api[_-]?key|authorization/i.test(k)) out[k] = '***REDACTED***';",
|
||||
" else out[k] = redact(obj[k]);",
|
||||
" });",
|
||||
" return out;",
|
||||
"}",
|
||||
"",
|
||||
"// Log request details",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : 'Unknown Request';",
|
||||
"var requestMethod = pm.request && pm.request.method ? pm.request.method : 'UNKNOWN';",
|
||||
"var requestUrl = pm.request && pm.request.url ? pm.request.url.toString() : 'Unknown URL';",
|
||||
"var requestBody = '';",
|
||||
"if (pm.request && pm.request.body && pm.request.body.raw) {",
|
||||
" try {",
|
||||
" var parsedReq = typeof pm.request.body.raw === 'string' ? JSON.parse(pm.request.body.raw) : pm.request.body.raw;",
|
||||
" requestBody = JSON.stringify(redact(parsedReq), null, 2);",
|
||||
" } catch (e) {",
|
||||
" requestBody = pm.request.body.raw;",
|
||||
" }",
|
||||
"}",
|
||||
"",
|
||||
"// Log response details",
|
||||
"var responseBody = '';",
|
||||
"var responseText = '';",
|
||||
"try {",
|
||||
" responseText = pm.response.text();",
|
||||
" if (responseText) {",
|
||||
" try {",
|
||||
" var parsedRes = JSON.parse(responseText);",
|
||||
" responseBody = JSON.stringify(redact(parsedRes), null, 2);",
|
||||
" } catch (e) {",
|
||||
" responseBody = responseText;",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {",
|
||||
" responseBody = pm.response.text() || '';",
|
||||
"}",
|
||||
"",
|
||||
"// Output formatted request/response logs (body content only when verbose_logs=1)",
|
||||
"var verbose = (pm.collectionVariables.get('verbose_logs') || '0') === '1';",
|
||||
"console.log('\\n' + '='.repeat(80));",
|
||||
"console.log('REQUEST: ' + requestMethod + ' ' + requestName);",
|
||||
"console.log('URL: ' + requestUrl);",
|
||||
"if (verbose && requestBody) {",
|
||||
" console.log('REQUEST BODY:');",
|
||||
" console.log(requestBody);",
|
||||
"}",
|
||||
"console.log('\\nRESPONSE: ' + pm.response.code + ' ' + pm.response.status);",
|
||||
"if (verbose && responseBody) {",
|
||||
" console.log('RESPONSE BODY:');",
|
||||
" console.log(responseBody);",
|
||||
"}",
|
||||
"console.log('='.repeat(80) + '\\n');",
|
||||
"",
|
||||
"var code = pm.response.code;",
|
||||
"var pass = (code >= 200 && code <= 299);",
|
||||
"var unsupportedError = false;",
|
||||
"var body = null;",
|
||||
"if (!pass && code >= 400) {",
|
||||
" try {",
|
||||
" body = pm.response.json();",
|
||||
" var msg = (body && body.error && body.error.message) ? body.error.message : (body && body.message) ? body.message : '';",
|
||||
" var errCode = (body && body.error && body.error.code) ? body.error.code : (body && body.code) ? body.code : '';",
|
||||
" var errType = (body && body.error && body.error.type) ? body.error.type : '';",
|
||||
" var ALLOWED_CODES = ['unsupported_operation'];",
|
||||
" var ALLOWED_ERR_TYPES = [];",
|
||||
" var ALLOWED_MESSAGES = [];",
|
||||
" var UNSUPPORTED_MSG_REGEX = /^.+ is not supported by .+ provider$/;",
|
||||
" var NO_CREDENTIALS_REGEX = /no keys found|missing credentials|authentication required|unauthorized|invalid credentials/i;",
|
||||
" var allowAuthSkip = (pm.collectionVariables.get('allow_auth_skip') || '0') === '1';",
|
||||
" var responseText = pm.response.text() || '';",
|
||||
" var textToCheck = typeof msg === 'string' ? msg : (typeof body === 'string' ? body : responseText);",
|
||||
" try { if (typeof body === 'object' && body !== null) { textToCheck = (body.error && body.error.message) ? body.error.message : (body.message || JSON.stringify(body)); } } catch (e) {}",
|
||||
" var isWhitelisted = (ALLOWED_CODES.indexOf(errCode) !== -1) ||",
|
||||
" (ALLOWED_ERR_TYPES.indexOf(errType) !== -1) ||",
|
||||
" (ALLOWED_MESSAGES.indexOf(msg) !== -1) ||",
|
||||
" (typeof msg === 'string' && UNSUPPORTED_MSG_REGEX.test(msg)) ||",
|
||||
" (allowAuthSkip && code === 401 && NO_CREDENTIALS_REGEX.test(String(textToCheck)));",
|
||||
" if (isWhitelisted) {",
|
||||
" unsupportedError = true;",
|
||||
" } else {",
|
||||
" pass = false;",
|
||||
" }",
|
||||
" } catch (e) {",
|
||||
" pass = false;",
|
||||
" }",
|
||||
"}",
|
||||
"if (unsupportedError) {",
|
||||
" console.warn('Skipped (unsupported operation): ' + (body && body.error ? JSON.stringify(body.error) : pm.response.text()));",
|
||||
" pm.test('Request skipped (unsupported operation)', function() { pm.expect(unsupportedError).to.be.true; });",
|
||||
"} else {",
|
||||
" pm.test('Status is 2xx or unsupported operation', function() { pm.expect(pass).to.be.true; });",
|
||||
"}",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "bedrock",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "amazon.titan-text-express-v1",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "s3_bucket",
|
||||
"value": "bifrost-test-bucket",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "s3_output_bucket",
|
||||
"value": "bifrost-test-bucket",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "role_arn",
|
||||
"value": "arn:aws:iam::123456789012:role/BedrockBatchRole",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "batch_input_key",
|
||||
"value": "batch_input_placeholder.jsonl",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "s3_key",
|
||||
"value": "test-file.txt",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "job_arn",
|
||||
"value": "arn:aws:bedrock:us-east-1:123456789012:model-invocation-job/abc123",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_api_key",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_access_key",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_secret_key",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_region",
|
||||
"value": "us-east-1",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "bedrock_session_token",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "allow_auth_skip",
|
||||
"value": "0",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "execution_order",
|
||||
"value": "[\"Converse\",\"Converse Stream\",\"Invoke\",\"Invoke with Response Stream\",\"Upload Batch Input File\",\"Create Batch Job\",\"List Batch Jobs\",\"Retrieve Batch Job\",\"Stop Batch Job\",\"List Objects\",\"PUT Object\",\"GET Object\",\"HEAD Object\",\"DELETE Object\"]",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider_capabilities",
|
||||
"value": "{\"description\":\"Provider capability matrix for integration tests. Each provider has explicit booleans per operation (derived from core/providers/* provider.go NewUnsupportedOperationError). Used to skip requests when running with all provider envs.\",\"providers\":{\"openai\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":true},\"anthropic\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":true,\"file\":true,\"container\":false},\"azure\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"bedrock\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"cerebras\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"cohere\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"elevenlabs\":{\"chat_completions\":true,\"embedding\":false,\"speech\":true,\"transcription\":true,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"gemini\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"groq\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"huggingface\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"mistral\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"nebius\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"openrouter\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"parasail\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"perplexity\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"replicate\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":true,\"container\":false},\"vertex\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"xai\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false}}}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "request_to_operation",
|
||||
"value": "{\"Create Batch Job\":\"batch\",\"Upload Batch Input File\":\"batch\",\"List Batch Jobs\":\"batch\",\"Retrieve Batch Job\":\"batch\",\"Stop Batch Job\":\"batch\",\"List Objects\":\"file\",\"PUT Object\":\"file\",\"GET Object\":\"file\",\"HEAD Object\":\"file\",\"DELETE Object\":\"file\"}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "_retry_429_max",
|
||||
"value": "3",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "_retry_429_count",
|
||||
"value": "0",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Converse",
|
||||
"item": [
|
||||
{
|
||||
"name": "Converse",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" pm.expect(j).to.have.property('output');",
|
||||
" pm.expect(j.output).to.have.property('message');",
|
||||
" pm.expect(j.output.message).to.have.property('content');",
|
||||
" pm.expect(j.output.message.content).to.be.an('array').that.is.not.empty;",
|
||||
" var hasText = j.output.message.content.some(function(c) { return c && c.text !== undefined; });",
|
||||
" pm.expect(hasText, 'response should contain at least one text content block').to.be.true;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"temperature\": 0.7,\n \"maxTokens\": 1024\n }\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model/{{provider}}%2F{{model}}/converse",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model",
|
||||
"{{provider}}%2F{{model}}",
|
||||
"converse"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Converse Stream",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var body = pm.response.text();",
|
||||
" pm.expect(body, 'streaming response should have body').to.not.be.empty;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"temperature\": 0.7,\n \"maxTokens\": 1024\n }\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model/{{provider}}%2F{{model}}/converse-stream",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model",
|
||||
"{{provider}}%2F{{model}}",
|
||||
"converse-stream"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Invoke",
|
||||
"item": [
|
||||
{
|
||||
"name": "Invoke",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" var hasOutput = (j.outputs && Array.isArray(j.outputs) && j.outputs.length > 0) || (j.completion && String(j.completion).length > 0) || (j.text && String(j.text).length > 0);",
|
||||
" pm.expect(hasOutput, 'response should have outputs, completion, or text').to.be.true;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"anthropic_version\": \"bedrock-2023-05-31\",\n \"prompt\": \"Hello, how are you?\",\n \"max_tokens\": 1024,\n \"temperature\": 0.7\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model/{{provider}}%2F{{model_invoke}}/invoke",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model",
|
||||
"{{provider}}%2F{{model_invoke}}",
|
||||
"invoke"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Invoke with Response Stream",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var body = pm.response.text();",
|
||||
" pm.expect(body, 'stream response should have content').to.not.be.empty;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"anthropic_version\": \"bedrock-2023-05-31\",\n \"prompt\": \"Hello, how are you?\",\n \"max_tokens\": 1024,\n \"temperature\": 0.7\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model/{{provider}}%2F{{model_invoke}}/invoke-with-response-stream",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model",
|
||||
"{{provider}}%2F{{model_invoke}}",
|
||||
"invoke-with-response-stream"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Batch Jobs",
|
||||
"item": [
|
||||
{
|
||||
"name": "Upload Batch Input File",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var provider = (pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || 'provider';",
|
||||
"var unique = Date.now() + '_' + Math.floor(Math.random() * 1000000);",
|
||||
"pm.collectionVariables.set('batch_input_key', 'batch_input_' + provider + '_' + unique + '.jsonl');",
|
||||
"if (pm.environment) { pm.environment.set('batch_input_key', pm.collectionVariables.get('batch_input_key')); }"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var etag = pm.response.headers.get('ETag') || pm.response.headers.get('etag');",
|
||||
" if (etag) { var fid = etag.replace(/^\"/, '').replace(/\"$/, ''); pm.collectionVariables.set('file_id', fid); if (pm.environment) { pm.environment.set('file_id', fid); } }",
|
||||
" pm.test('Response has ETag header', function() { pm.expect(pm.response.headers.has('ETag') || pm.response.headers.has('etag')).to.be.true; });",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/jsonl"
|
||||
},
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"custom_id\":\"request-1\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\",\"body\":{\"model\":\"{{model}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, this is test message 1. Say hi back briefly.\"}],\"max_tokens\":100}}\n{\"custom_id\":\"request-2\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\",\"body\":{\"model\":\"{{model}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, this is test message 2. Say hi back briefly.\"}],\"max_tokens\":100}}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{batch_input_key}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["bedrock", "files", "{{s3_bucket}}", "{{batch_input_key}}"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Create Batch Job",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" pm.expect(j).to.have.property('jobArn');",
|
||||
" if (j && j.jobArn) { pm.collectionVariables.set('job_arn', j.jobArn); if (pm.environment) { pm.environment.set('job_arn', j.jobArn); } }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"jobName\": \"bifrost-test-job\",\n \"roleArn\": \"{{role_arn}}\",\n \"inputDataConfig\": {\n \"s3InputDataConfig\": {\n \"s3Uri\": \"s3://{{s3_bucket}}/{{batch_input_key}}\",\n \"s3InputFormat\": \"JSONL\"\n }\n },\n \"outputDataConfig\": {\n \"s3OutputDataConfig\": {\n \"s3Uri\": \"s3://{{s3_output_bucket}}/output/\"\n }\n },\n \"tags\": [\n {\"key\": \"endpoint\", \"value\": \"/v1/chat/completions\"},\n {\"key\": \"file_key\", \"value\": \"{{batch_input_key}}\"}\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model-invocation-job",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model-invocation-job"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "List Batch Jobs",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" pm.expect(j).to.have.property('invocationJobSummaries');",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model-invocation-jobs?maxResults=10",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model-invocation-jobs"
|
||||
],
|
||||
"query": [
|
||||
{
|
||||
"key": "maxResults",
|
||||
"value": "10"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Retrieve Batch Job",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var j = pm.response.json();",
|
||||
" pm.expect(j).to.have.property('jobArn');",
|
||||
" pm.expect(j).to.have.property('status');",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model-invocation-job/{{job_arn}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model-invocation-job",
|
||||
"{{job_arn}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Stop Batch Job",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/model-invocation-job/{{job_arn}}/stop",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"model-invocation-job",
|
||||
"{{job_arn}}",
|
||||
"stop"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "S3 Operations",
|
||||
"item": [
|
||||
{
|
||||
"name": "List Objects",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var body = pm.response.text();",
|
||||
" pm.expect(body !== undefined && body !== null, 'response body should be present').to.be.true;",
|
||||
" pm.expect(body, 'ListObjectsV2 response should have body').to.not.be.empty;",
|
||||
" if (body && body.indexOf('ListBucketResult') !== -1) {",
|
||||
" pm.test('Response is S3 ListBucketResult XML', function() { pm.expect(body).to.include('ListBucketResult'); });",
|
||||
" }",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["bedrock", "files", "{{s3_bucket}}"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "PUT Object",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var etag = pm.response.headers.get('ETag') || pm.response.headers.get('etag');",
|
||||
" if (etag) { var fid = etag.replace(/^\"/, '').replace(/\"$/, ''); pm.collectionVariables.set('file_id', fid); if (pm.environment) { pm.environment.set('file_id', fid); } }",
|
||||
" pm.test('Response has ETag header', function() { pm.expect(pm.response.headers.has('ETag') || pm.response.headers.has('etag')).to.be.true; });",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/jsonl"
|
||||
},
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"custom_id\":\"request-1\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\",\"body\":{\"model\":\"{{model}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, this is test message 1. Say hi back briefly.\"}],\"max_tokens\":100}}\n{\"custom_id\":\"request-2\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\",\"body\":{\"model\":\"{{model}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, this is test message 2. Say hi back briefly.\"}],\"max_tokens\":100}}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{s3_key}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"files",
|
||||
"{{s3_bucket}}",
|
||||
"{{s3_key}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "GET Object",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" var body = pm.response.text();",
|
||||
" pm.expect(body !== undefined && body !== null, 'response body should be present').to.be.true;",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{s3_key}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"files",
|
||||
"{{s3_bucket}}",
|
||||
"{{s3_key}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "HEAD Object",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"exec": [
|
||||
"if (pm.response.code >= 200 && pm.response.code <= 299) {",
|
||||
" pm.test('Response has Content-Length header', function() { pm.response.to.have.header('Content-Length'); });",
|
||||
"}"
|
||||
],
|
||||
"type": "text/javascript"
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "HEAD",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{s3_key}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"files",
|
||||
"{{s3_bucket}}",
|
||||
"{{s3_key}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "DELETE Object",
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [
|
||||
{
|
||||
"key": "x-model-provider",
|
||||
"value": "{{provider}}"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/bedrock/files/{{s3_bucket}}/{{s3_key}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"bedrock",
|
||||
"files",
|
||||
"{{s3_bucket}}",
|
||||
"{{s3_key}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,841 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost Composite Integrations API",
|
||||
"description": "E2E tests for composite integration endpoints (GenAI, Cohere, LiteLLM, LangChain, PydanticAI) and Health endpoint",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var requestName = pm.info.requestName;",
|
||||
"var retryKey = 'retry_' + requestName;",
|
||||
"var ciVal = pm.environment.get('CI');",
|
||||
"var maxRetries = (String(ciVal).toLowerCase() === 'true' || ciVal === '1' || ciVal === 1) ? 10 : 0;",
|
||||
"if (!pm.collectionVariables.has(retryKey)) {",
|
||||
" pm.collectionVariables.set(retryKey, 0);",
|
||||
"}",
|
||||
"pm.collectionVariables.set('current_request_name', requestName);",
|
||||
"pm.collectionVariables.set('max_retries', maxRetries);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var provider = (((pm.environment && pm.environment.get('provider')) || pm.collectionVariables.get('provider') || '') + '').trim().toLowerCase();",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : '';",
|
||||
"var capsStr = pm.collectionVariables.get('provider_capabilities') || '{}';",
|
||||
"var execOrderStr = pm.collectionVariables.get('execution_order') || '[]';",
|
||||
"var requestToOpStr = pm.collectionVariables.get('request_to_operation') || '{}';",
|
||||
"var idx = parseInt(pm.collectionVariables.get('_exec_index') || '0', 10);",
|
||||
"pm.collectionVariables.set('_current_exec_index', String(idx));",
|
||||
"try {",
|
||||
" var caps = JSON.parse(capsStr);",
|
||||
" var execOrder = JSON.parse(execOrderStr);",
|
||||
" var requestToOp = JSON.parse(requestToOpStr);",
|
||||
" var providerCaps = caps.providers && caps.providers[provider];",
|
||||
" var op = requestToOp[requestName];",
|
||||
" if (provider && op && providerCaps && providerCaps[op] === false) {",
|
||||
" var nextName = (idx >= 0 && idx < execOrder.length - 1) ? execOrder[idx + 1] : null;",
|
||||
" pm.collectionVariables.set('_exec_index', String(idx + 1));",
|
||||
" if (nextName) {",
|
||||
" if (pm.execution && typeof pm.execution.setNextRequest === 'function') {",
|
||||
" pm.execution.setNextRequest(nextName);",
|
||||
" } else if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest(nextName);",
|
||||
" }",
|
||||
" }",
|
||||
" if (pm.execution && typeof pm.execution.skipRequest === 'function') {",
|
||||
" pm.execution.skipRequest();",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var requestNameRetry = pm.collectionVariables.get('current_request_name');",
|
||||
"var retryKey = 'retry_' + requestNameRetry;",
|
||||
"var currentRetry = parseInt(pm.collectionVariables.get(retryKey) || '0', 10);",
|
||||
"var maxRetries = parseInt(pm.collectionVariables.get('max_retries') || '0', 10);",
|
||||
"var hasFailures = code < 200 || code > 299;",
|
||||
"if (hasFailures && maxRetries > 0 && currentRetry < maxRetries) {",
|
||||
" pm.collectionVariables.set(retryKey, String(currentRetry + 1));",
|
||||
" var delayMs = Math.min(1000 * Math.pow(2, currentRetry), 30000);",
|
||||
" var end = Date.now() + delayMs; while (Date.now() < end) {}",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed (attempt ' + (currentRetry + 1) + '/' + maxRetries + '). Retrying after ' + delayMs + 'ms...');",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) { postman.setNextRequest(requestNameRetry); }",
|
||||
"} else {",
|
||||
" pm.collectionVariables.set(retryKey, '0');",
|
||||
" if (hasFailures && currentRetry >= maxRetries && maxRetries > 0) {",
|
||||
" console.log('[RETRY] Request \"' + requestNameRetry + '\" failed after ' + maxRetries + ' retries. Moving on.');",
|
||||
" }",
|
||||
" ",
|
||||
"// Helper function to pretty print JSON",
|
||||
"function prettyPrintJSON(jsonString) {",
|
||||
" try {",
|
||||
" var parsed = typeof jsonString === 'string' ? JSON.parse(jsonString) : jsonString;",
|
||||
" return JSON.stringify(parsed, null, 2);",
|
||||
" } catch (e) {",
|
||||
" return jsonString;",
|
||||
" }",
|
||||
"}",
|
||||
"function redact(obj) {",
|
||||
" if (!obj || typeof obj !== 'object') return obj;",
|
||||
" if (Array.isArray(obj)) return obj.map(redact);",
|
||||
" var out = {};",
|
||||
" Object.keys(obj).forEach(function (k) {",
|
||||
" if (/password|secret|token|api[_-]?key|authorization/i.test(k)) out[k] = '***REDACTED***';",
|
||||
" else out[k] = redact(obj[k]);",
|
||||
" });",
|
||||
" return out;",
|
||||
"}",
|
||||
"",
|
||||
"// Log request details",
|
||||
"var requestName = pm.info && pm.info.requestName ? pm.info.requestName : 'Unknown Request';",
|
||||
"var requestMethod = pm.request && pm.request.method ? pm.request.method : 'UNKNOWN';",
|
||||
"var requestUrl = pm.request && pm.request.url ? pm.request.url.toString() : 'Unknown URL';",
|
||||
"var requestBody = '';",
|
||||
"if (pm.request && pm.request.body && pm.request.body.raw) {",
|
||||
" try {",
|
||||
" var parsedReq = typeof pm.request.body.raw === 'string' ? JSON.parse(pm.request.body.raw) : pm.request.body.raw;",
|
||||
" requestBody = JSON.stringify(redact(parsedReq), null, 2);",
|
||||
" } catch (e) {",
|
||||
" requestBody = pm.request.body.raw;",
|
||||
" }",
|
||||
"}",
|
||||
"",
|
||||
"// Log response details",
|
||||
"var responseBody = '';",
|
||||
"try {",
|
||||
" var responseText = pm.response.text();",
|
||||
" if (responseText) {",
|
||||
" try {",
|
||||
" var parsedRes = JSON.parse(responseText);",
|
||||
" responseBody = JSON.stringify(redact(parsedRes), null, 2);",
|
||||
" } catch (e) {",
|
||||
" responseBody = responseText;",
|
||||
" }",
|
||||
" }",
|
||||
"} catch (e) {",
|
||||
" responseBody = pm.response.text() || '';",
|
||||
"}",
|
||||
"",
|
||||
"// Output formatted request/response logs (body only when verbose_logs=1)",
|
||||
"var verbose = (pm.collectionVariables.get('verbose_logs') || '0') === '1';",
|
||||
"console.log('\\n' + '='.repeat(80));",
|
||||
"console.log('REQUEST: ' + requestMethod + ' ' + requestName);",
|
||||
"console.log('URL: ' + requestUrl);",
|
||||
"if (verbose && requestBody) {",
|
||||
" console.log('REQUEST BODY:');",
|
||||
" console.log(requestBody);",
|
||||
"}",
|
||||
"console.log('\\nRESPONSE: ' + pm.response.code + ' ' + pm.response.status);",
|
||||
"if (verbose && responseBody) {",
|
||||
" console.log('RESPONSE BODY:');",
|
||||
" console.log(responseBody);",
|
||||
"}",
|
||||
"console.log('='.repeat(80) + '\\n');",
|
||||
"",
|
||||
"var code = pm.response.code;",
|
||||
"var pass = (code >= 200 && code <= 299);",
|
||||
"if (!pass && code >= 400) {",
|
||||
" if (code === 405) { pass = true; }",
|
||||
" if (!pass) try {",
|
||||
" var body = pm.response.json();",
|
||||
" var msg = (body && body.error && body.error.message) ? body.error.message : (body && body.message) ? body.message : '';",
|
||||
" var errCode = (body && body.error && body.error.code) ? body.error.code : (body && body.code) ? body.code : '';",
|
||||
" var errType = (body && body.error && body.error.type) ? body.error.type : '';",
|
||||
" var allowedCodes = ['unsupported_operation', 'tool_use_failed'];",
|
||||
" var allowedTypes = ['unsupported_operation', 'invalid_request_error'];",
|
||||
" var allowedMessagePattern = /\\b(not\\s+supported|unsupported\\s+(operation|feature)|method\\s+not\\s+allowed|not\\s+implemented|not\\s+configured|no\\s+config\\s+found|tool_use_failed|failed to call|failed_generation|failed to unmarshal|unmarshal.*response|embedContent|generateContent)\\b/i;",
|
||||
" var isAllowedUnsupported = allowedCodes.indexOf(errCode) !== -1 ||",
|
||||
" allowedTypes.indexOf(errType) !== -1 ||",
|
||||
" (typeof msg === 'string' && allowedMessagePattern.test(msg.trim()));",
|
||||
" if (isAllowedUnsupported) { pass = true; }",
|
||||
" } catch (e) {}",
|
||||
"}",
|
||||
"if (!pass && code === 500) {",
|
||||
" try {",
|
||||
" var body = pm.response.json();",
|
||||
" var msg = (body && body.error && body.error.message) ? body.error.message : (body && body.message) ? body.message : '';",
|
||||
" if (typeof msg === 'string' && /failed to unmarshal|unmarshal.*response|embedContent|generateContent/i.test(msg)) { pass = true; }",
|
||||
" } catch (e) {}",
|
||||
"}",
|
||||
"var execIdx = parseInt(pm.collectionVariables.get('_current_exec_index') || '0', 10);",
|
||||
"pm.collectionVariables.set('_exec_index', String(execIdx + 1));",
|
||||
"pm.test('Status is 2xx or allowed unsupported (405, unsupported_operation, tool_use_failed, or GenAI/unmarshal)', function() { pm.expect(pass).to.be.true; });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openai",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "gpt-4o",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "text-embedding-3-small",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "execution_order",
|
||||
"value": "[\"Generate Content\",\"Embed Content\",\"Chat\",\"Embed\",\"Tokenize\",\"Chat Completions (OpenAI Routing)\",\"Messages (Anthropic Routing)\",\"Converse (Bedrock Routing)\",\"Generate Content (GenAI Routing)\",\"Chat (Cohere Routing)\",\"Chat Completions (OpenAI Routing)\",\"Messages (Anthropic Routing)\",\"Converse (Bedrock Routing)\",\"Generate Content (GenAI Routing)\",\"Chat (Cohere Routing)\",\"Chat Completions (OpenAI Routing)\",\"Messages (Anthropic Routing)\",\"Converse (Bedrock Routing)\",\"Generate Content (GenAI Routing)\",\"Chat (Cohere Routing)\",\"Health Check\"]",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider_capabilities",
|
||||
"value": "{\"description\":\"Provider capability matrix for integration tests. Each provider has explicit booleans per operation (derived from core/providers/* provider.go NewUnsupportedOperationError). Used to skip requests when running with all provider envs.\",\"providers\":{\"openai\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":true},\"anthropic\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":true,\"file\":true,\"container\":false},\"azure\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"bedrock\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"cerebras\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"cohere\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"elevenlabs\":{\"chat_completions\":false,\"embedding\":false,\"speech\":true,\"transcription\":true,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"gemini\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":true,\"file\":true,\"container\":false},\"groq\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"huggingface\":{\"chat_completions\":true,\"embedding\":true,\"speech\":true,\"transcription\":true,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"mistral\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":true,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"nebius\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"openrouter\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"parasail\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"perplexity\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":false,\"batch\":false,\"file\":false,\"container\":false},\"replicate\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":true,\"container\":false},\"vertex\":{\"chat_completions\":true,\"embedding\":true,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false},\"xai\":{\"chat_completions\":true,\"embedding\":false,\"speech\":false,\"transcription\":false,\"image\":true,\"batch\":false,\"file\":false,\"container\":false}}}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "request_to_operation",
|
||||
"value": "{\"Generate Content\":\"chat_completions\",\"Embed Content\":\"embedding\",\"Chat\":\"chat_completions\",\"Embed\":\"embedding\",\"Tokenize\":\"chat_completions\",\"Chat Completions (OpenAI Routing)\":\"chat_completions\",\"Messages (Anthropic Routing)\":\"chat_completions\",\"Converse (Bedrock Routing)\":\"chat_completions\",\"Generate Content (GenAI Routing)\":\"chat_completions\",\"Chat (Cohere Routing)\":\"chat_completions\"}",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "_exec_index",
|
||||
"value": "0",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "_current_exec_index",
|
||||
"value": "0",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "GenAI",
|
||||
"item": [
|
||||
{
|
||||
"name": "Generate Content",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"contents\": [\n {\n \"parts\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ],\n \"generationConfig\": {\n \"temperature\": 0.7,\n \"maxOutputTokens\": 1024\n }\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/genai/v1beta/models/{{provider}}/{{model}}:generateContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{model}}:generateContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Embed Content",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"content\": {\n \"parts\": [\n {\n \"text\": \"Hello world\"\n }\n ]\n }\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/genai/v1beta/models/{{provider}}/{{embedding_model}}:embedContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{embedding_model}}:embedContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Cohere",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/cohere/v2/chat",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"cohere",
|
||||
"v2",
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Embed",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{embedding_model}}\",\n \"texts\": [\n \"Hello world\",\n \"Goodbye world\"\n ],\n \"input_type\": \"search_document\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/cohere/v2/embed",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"cohere",
|
||||
"v2",
|
||||
"embed"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Tokenize",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"text\": \"How many tokens is this text?\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/cohere/v1/tokenize",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"cohere",
|
||||
"v1",
|
||||
"tokenize"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "LiteLLM",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completions (OpenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"temperature\": 0.7,\n \"max_completion_tokens\": 1000,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Messages (Anthropic Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"anthropic/claude-sonnet-4-5-20250929\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"max_tokens\": 1024\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/anthropic/v1/messages",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Converse (Bedrock Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/bedrock/model/us.anthropic.claude-3-5-sonnet-20241022-v2:0/converse",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"bedrock",
|
||||
"model",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"converse"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Generate Content (GenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"contents\": [\n {\n \"parts\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/genai/v1beta/models/{{provider}}/{{model}}:generateContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{model}}:generateContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat (Cohere Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/litellm/cohere/v2/chat",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"litellm",
|
||||
"cohere",
|
||||
"v2",
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "LangChain",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completions (OpenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"temperature\": 0.7,\n \"max_completion_tokens\": 1000,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Messages (Anthropic Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"anthropic/claude-sonnet-4-5-20250929\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"max_tokens\": 1024\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/anthropic/v1/messages",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Converse (Bedrock Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/bedrock/model/us.anthropic.claude-3-5-sonnet-20241022-v2:0/converse",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"bedrock",
|
||||
"model",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"converse"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Generate Content (GenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"contents\": [\n {\n \"parts\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/genai/v1beta/models/{{provider}}/{{model}}:generateContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{model}}:generateContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat (Cohere Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/langchain/cohere/v2/chat",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"langchain",
|
||||
"cohere",
|
||||
"v2",
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "PydanticAI",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completions (OpenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"temperature\": 0.7,\n \"max_completion_tokens\": 1000,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Messages (Anthropic Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"anthropic/claude-sonnet-4-5-20250929\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ],\n \"max_tokens\": 1024\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/anthropic/v1/messages",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"anthropic",
|
||||
"v1",
|
||||
"messages"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Converse (Bedrock Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/bedrock/model/us.anthropic.claude-3-5-sonnet-20241022-v2:0/converse",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"bedrock",
|
||||
"model",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"converse"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Generate Content (GenAI Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"contents\": [\n {\n \"parts\": [\n {\n \"text\": \"Hello, how are you?\"\n }\n ]\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/genai/v1beta/models/{{provider}}/{{model}}:generateContent",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"genai",
|
||||
"v1beta",
|
||||
"models",
|
||||
"{{provider}}/{{model}}:generateContent"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat (Cohere Routing)",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello, how are you?\"\n }\n ]\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/pydanticai/cohere/v2/chat",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"pydanticai",
|
||||
"cohere",
|
||||
"v2",
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Health",
|
||||
"item": [
|
||||
{
|
||||
"name": "Health Check",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/health",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"health"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Async Inference",
|
||||
"description": "Async inference submit/poll tests. Requires LogsStore and governance plugin.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "embedding_model", "value": "text-embedding-3-small", "type": "string"},
|
||||
{"key": "job_id", "value": "", "type": "string"},
|
||||
{"key": "embed_job_id", "value": "", "type": "string"},
|
||||
{"key": "poll_retries", "value": "0", "type": "string"},
|
||||
{"key": "embed_poll_retries", "value": "0", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Submit Chat Completion",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.collectionVariables.set('poll_retries', '0');"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) {",
|
||||
" pm.test('Async not configured (404)', function() { pm.expect(true).to.be.true; });",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest('Submit with stream (expect 400)');",
|
||||
" }",
|
||||
" return;",
|
||||
"}",
|
||||
"pm.test('Status 202', function() { pm.expect(code).to.equal(202); });",
|
||||
"if (code === 202) {",
|
||||
" var json = pm.response.json();",
|
||||
" pm.test('Has job_id', function() { pm.expect(json.id).to.be.a('string'); });",
|
||||
" pm.test('Status is pending', function() { pm.expect(json.status).to.equal('pending'); });",
|
||||
" pm.collectionVariables.set('job_id', json.id);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Poll Chat Completion",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) { pm.test('Skip - async not configured', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"var json = pm.response.json();",
|
||||
"var status = json.status;",
|
||||
"if (status === 'pending' || status === 'processing') {",
|
||||
" pm.test('HTTP 202 while pending/processing', function() { pm.expect(code).to.equal(202); });",
|
||||
" var retries = parseInt(pm.collectionVariables.get('poll_retries') || '0', 10);",
|
||||
" if (retries < 10) {",
|
||||
" pm.collectionVariables.set('poll_retries', String(retries + 1));",
|
||||
" var end = Date.now() + 3000;",
|
||||
" while (Date.now() < end) {}",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest('Poll Chat Completion');",
|
||||
" }",
|
||||
" } else {",
|
||||
" pm.test('Job completed or max retries', function() { pm.expect(status).to.be.oneOf(['completed', 'failed']); });",
|
||||
" }",
|
||||
"} else if (status === 'completed') {",
|
||||
" pm.test('HTTP 200 when completed', function() { pm.expect(code).to.equal(200); });",
|
||||
" pm.test('Job completed', function() { pm.expect(status).to.equal('completed'); });",
|
||||
" pm.test('Has result', function() { pm.expect(json.result).to.not.be.undefined; });",
|
||||
" pm.test('Result has choices with content', function() {",
|
||||
" pm.expect(json.result).to.have.property('choices').that.is.an('array').and.has.length.above(0);",
|
||||
" pm.expect(json.result.choices[0]).to.have.property('message');",
|
||||
" pm.expect(json.result.choices[0].message).to.have.property('content').that.is.a('string');",
|
||||
" });",
|
||||
"} else if (status === 'failed') {",
|
||||
" pm.test('HTTP 200 when failed', function() { pm.expect(code).to.equal(200); });",
|
||||
" pm.test('Job failed (acceptable)', function() { pm.expect(status).to.equal('failed'); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/chat/completions/{{job_id}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "chat", "completions", "{{job_id}}"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Submit Embedding with TTL",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.collectionVariables.set('embed_poll_retries', '0');"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) { pm.test('Skip - async not configured', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"pm.test('Status 202', function() { pm.expect(code).to.equal(202); });",
|
||||
"if (code === 202) {",
|
||||
" var json = pm.response.json();",
|
||||
" pm.collectionVariables.set('embed_job_id', json.id);",
|
||||
" pm.test('expires_at is set when TTL provided', function() { pm.expect(json.expires_at).to.not.be.undefined; });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-async-job-result-ttl", "value": "60"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{embedding_model}}\",\n \"input\": \"Hello world\",\n \"encoding_format\": \"float\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/embeddings",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "embeddings"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Poll Embedding",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) { pm.test('Skip - async not configured', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"var json = pm.response.json();",
|
||||
"var status = json.status;",
|
||||
"if (status === 'pending' || status === 'processing') {",
|
||||
" pm.test('HTTP 202 while pending/processing', function() { pm.expect(code).to.equal(202); });",
|
||||
" var retries = parseInt(pm.collectionVariables.get('embed_poll_retries') || '0', 10);",
|
||||
" if (retries < 10) {",
|
||||
" pm.collectionVariables.set('embed_poll_retries', String(retries + 1));",
|
||||
" var end = Date.now() + 3000;",
|
||||
" while (Date.now() < end) {}",
|
||||
" if (typeof postman !== 'undefined' && postman.setNextRequest) {",
|
||||
" postman.setNextRequest('Poll Embedding');",
|
||||
" }",
|
||||
" }",
|
||||
"} else if (status === 'completed') {",
|
||||
" pm.test('HTTP 200 when completed', function() { pm.expect(code).to.equal(200); });",
|
||||
" pm.test('Embedding job completed', function() { pm.expect(status).to.equal('completed'); });",
|
||||
" pm.test('Result has embedding data', function() {",
|
||||
" pm.expect(json.result).to.not.be.undefined;",
|
||||
" pm.expect(json.result).to.have.property('data').that.is.an('array').and.has.length.above(0);",
|
||||
" pm.expect(json.result.data[0]).to.have.property('embedding').that.is.an('array');",
|
||||
" });",
|
||||
"} else if (status === 'failed') {",
|
||||
" pm.test('HTTP 200 when failed', function() { pm.expect(code).to.equal(200); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/embeddings/{{embed_job_id}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "embeddings", "{{embed_job_id}}"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Submit with stream (expect 400)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"if (code === 404) { pm.test('Skip - async not configured', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"pm.test('Streaming not supported on async - expect 400', function() { pm.expect(code).to.equal(400); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"stream\": true\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/async/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "async", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Fallbacks",
|
||||
"description": "Fallback failover tests. Validates fallbacks array and extra_fields.provider.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "fallback_provider", "value": "anthropic", "type": "string"},
|
||||
{"key": "fallback_model", "value": "claude-3-5-sonnet-20241022", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion with fallbacks",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var extra = json.extra_fields || {};",
|
||||
" var providerUsed = extra.provider || json.provider;",
|
||||
" var allowed = ['openai', 'anthropic'];",
|
||||
" pm.test('Provider is openai or anthropic', function() {",
|
||||
" pm.expect(providerUsed).to.be.oneOf(allowed);",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"openai/gpt-4o\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"fallbacks\": [\"anthropic/claude-3-5-sonnet-20241022\"],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Forced fallback (invalid primary)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var extra = json.extra_fields || {};",
|
||||
" var providerUsed = extra.provider || json.provider;",
|
||||
" pm.test('Provider is openai (fallback)', function() {",
|
||||
" pm.expect(String(providerUsed).toLowerCase()).to.equal('openai');",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"openai/nonexistent-model-xyz\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"fallbacks\": [\"openai/gpt-4o\"],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "All fallbacks fail",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is non-2xx', function() { pm.expect(code).to.not.be.within(200, 299); });",
|
||||
"if (code >= 400) {",
|
||||
" try {",
|
||||
" var json = pm.response.json();",
|
||||
" var msg = (json.error && json.error.message) ? json.error.message : (json.message || '');",
|
||||
" pm.test('Error response has message', function() {",
|
||||
" pm.expect(msg).to.be.a('string').and.not.be.empty;",
|
||||
" });",
|
||||
" } catch (e) {}",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"openai/nonexistent-1\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"fallbacks\": [\"openai/nonexistent-2\"],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,757 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Management E2E Flows",
|
||||
"description": "Full lifecycle flows: Provider+Key+Inference, Customer+Team+VK+Inference, VK lifecycle (create, use, update, deactivate, delete).",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openai",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "customer_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "team_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "vk_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "vk_value",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Flow A - List Providers",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('List Providers returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/providers",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"providers"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow A - List Keys",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('List Keys returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/keys",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"keys"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow A - Chat Completion",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Chat Completion returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Create Customer",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var body = { name: 'Mgmt Flow Customer ' + Date.now(), email: 'mgmt@example.com' };",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create Customer returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" pm.test('Response contains customer object', function() { pm.expect(json.customer || json).to.be.an('object'); });",
|
||||
" var c = json.customer || json;",
|
||||
" pm.test('Customer has non-empty id', function() { pm.expect(c.id).to.be.a('string').and.not.be.empty; });",
|
||||
" if (c.id) pm.collectionVariables.set('customer_id', c.id);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/customers",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"customers"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Create Team",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var cid = pm.collectionVariables.get('customer_id');",
|
||||
"var body = { name: 'Mgmt Flow Team ' + Date.now() };",
|
||||
"if (cid) body.customer_id = cid;",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create Team returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" pm.test('Response contains team object', function() { pm.expect(json.team || json).to.be.an('object'); });",
|
||||
" var t = json.team || json;",
|
||||
" pm.test('Team has non-empty id', function() { pm.expect(t.id).to.be.a('string').and.not.be.empty; });",
|
||||
" if (t.id) pm.collectionVariables.set('team_id', t.id);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/teams",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"teams"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Create VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var tid = pm.collectionVariables.get('team_id');",
|
||||
"var body = { name: 'Mgmt Flow VK ' + Date.now() };",
|
||||
"if (tid) body.team_id = tid;",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var vk = json.virtual_key || json;",
|
||||
" pm.test('Response contains VK object', function() { pm.expect(vk).to.be.an('object'); });",
|
||||
" pm.test('VK has non-empty id', function() { pm.expect(vk.id).to.be.a('string').and.not.be.empty; });",
|
||||
" pm.test('VK value has sk-bf- prefix', function() { pm.expect(vk.value).to.match(/^sk-bf-/); });",
|
||||
" if (vk.id) pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" if (vk.value) pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Chat Completion with VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Chat with VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Delete VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Delete Team",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Delete Team returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/teams/{{team_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"teams",
|
||||
"{{team_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow B - Delete Customer",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Delete Customer returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/customers/{{customer_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"customers",
|
||||
"{{customer_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Create VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var body = { name: 'Lifecycle VK ' + Date.now() };",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var vk = json.virtual_key || json;",
|
||||
" pm.test('Response contains VK object', function() { pm.expect(vk).to.be.an('object'); });",
|
||||
" pm.test('VK has non-empty id', function() { pm.expect(vk.id).to.be.a('string').and.not.be.empty; });",
|
||||
" pm.test('VK value has sk-bf- prefix', function() { pm.expect(vk.value).to.match(/^sk-bf-/); });",
|
||||
" if (vk.id) pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" if (vk.value) pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Chat Completion with VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Chat with VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Update VK (rename)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.request.body.raw = JSON.stringify({ name: 'Lifecycle VK Renamed ' + Date.now() });"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Update VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Chat Completion after rename",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Chat after rename returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Deactivate VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.request.body.raw = JSON.stringify({ is_active: false });"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Deactivate VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Chat with deactivated VK (expect 403)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Deactivated VK returns 403', function() { pm.expect(code).to.equal(403); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Flow C - Delete VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Rate Limit / Budget",
|
||||
"description": "Rate limit enforcement tests. Creates VK with request_max_limit: 2, expects 429 on 3rd request.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "vk_value", "value": "", "type": "string"},
|
||||
{"key": "vk_id", "value": "", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Setup - Create VK with rate limit",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var ts = Date.now();",
|
||||
"var body = {",
|
||||
" name: 'Rate Limit Test VK ' + ts,",
|
||||
" rate_limit: {",
|
||||
" request_max_limit: 2,",
|
||||
" request_reset_duration: '1m'",
|
||||
" }",
|
||||
"};",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var vk = json.virtual_key || json;",
|
||||
" if (vk.id) pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" if (vk.value) pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {"mode": "raw", "raw": "{}"},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["api", "governance", "virtual-keys"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion #1",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Request 1: 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion #2",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Request 2: 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion #3 (expect 429)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Request 3: 429 rate limited', function() { pm.expect(code).to.equal(429); });",
|
||||
"if (code === 429) {",
|
||||
" var json = pm.response.json();",
|
||||
" var errType = (json.type || (json.error && json.error.type) || '').toString();",
|
||||
" if (errType) {",
|
||||
" pm.test('Error type indicates rate limit', function() {",
|
||||
" pm.expect(errType).to.match(/request_limited|rate_limited|token_limited/);",
|
||||
" });",
|
||||
" }",
|
||||
" pm.test('Error has message', function() {",
|
||||
" pm.expect(json.error).to.be.an('object');",
|
||||
" pm.expect(json.error.message).to.be.a('string').and.not.be.empty;",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}],\n \"max_completion_tokens\": 5,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Teardown - Delete VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var vkId = pm.collectionVariables.get('vk_id');",
|
||||
"if (!vkId) { pm.test('Teardown skipped - no VK to delete', function() { pm.expect(true).to.be.true; }); return; }",
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["api", "governance", "virtual-keys", "{{vk_id}}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Session Stickiness",
|
||||
"description": "Session stickiness tests. Validates x-bf-session-id and x-bf-session-ttl headers are accepted.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "session_id", "value": "", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion with session ID",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var sid = 'test-session-' + Date.now();",
|
||||
"pm.collectionVariables.set('session_id', sid);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-session-id", "value": "{{session_id}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion with same session ID",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-session-id", "value": "{{session_id}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion with different session ID",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-session-id", "value": "test-session-other-{{$timestamp}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion with session TTL",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 2xx', function() { pm.expect(pm.response.code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-session-id", "value": "test-session-ttl"},
|
||||
{"key": "x-bf-session-ttl", "value": "60"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Streaming",
|
||||
"description": "Streaming SSE tests for inference endpoints. Validates Content-Type, data: lines, and [DONE] marker.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "responses_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "embedding_model", "value": "text-embedding-3-small", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion (stream)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" pm.test('Content-Type is SSE', function() {",
|
||||
" var ct = pm.response.headers.get('Content-Type') || '';",
|
||||
" pm.expect(ct).to.include('text/event-stream');",
|
||||
" });",
|
||||
" var body = pm.response.text();",
|
||||
" pm.test('Body contains data lines', function() { pm.expect(body).to.include('data:'); });",
|
||||
" pm.test('Stream ends with DONE', function() { pm.expect(body).to.include('[DONE]'); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": true\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Responses (stream)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" pm.test('Content-Type is SSE', function() {",
|
||||
" var ct = pm.response.headers.get('Content-Type') || '';",
|
||||
" pm.expect(ct).to.include('text/event-stream');",
|
||||
" });",
|
||||
" var body = pm.response.text();",
|
||||
" pm.test('Body contains data lines', function() { pm.expect(body).to.include('data:'); });",
|
||||
" pm.test('Body contains event lines', function() { pm.expect(body).to.include('event:'); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{responses_model}}\",\n \"input\": \"Say hello\",\n \"max_output_tokens\": 50,\n \"stream\": true\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/responses",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "responses"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - Virtual Key Auth",
|
||||
"description": "Virtual key authentication tests for inference endpoints. Self-provisions a VK, runs inference with/without VK, tests rejection cases, and cleans up.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openai",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "text-embedding-3-small",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "vk_value",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "vk_id",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"key": "enforce_auth",
|
||||
"value": "",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Setup",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create Virtual Key",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var timestamp = Date.now();",
|
||||
"var uniqueName = 'VK Auth Test ' + timestamp;",
|
||||
"pm.request.body.raw = JSON.stringify({name: uniqueName, provider_configs: [{provider: 'openai', weight: 1.0, allowed_models: ['*'], key_ids: ['*']}]});"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 200 or 201', function() { pm.expect(code).to.be.oneOf([200, 201]); });",
|
||||
"if (code === 200 || code === 201) {",
|
||||
" var jsonData = pm.response.json();",
|
||||
" var vk = (jsonData && (jsonData.virtual_key || jsonData)) || null;",
|
||||
" pm.test('VK has id and value', function() {",
|
||||
" pm.expect(vk).to.be.an('object');",
|
||||
" pm.expect(vk.id).to.be.a('string').and.not.be.empty;",
|
||||
" pm.expect(vk.value).to.be.a('string').and.not.be.empty;",
|
||||
" pm.expect(vk.value).to.match(/^sk-bf-/);",
|
||||
" });",
|
||||
" pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"name\": \"VK Auth Test\"}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Inference Without VK",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - No VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var enforceAuth = (pm.environment && pm.environment.get('enforce_auth')) || pm.collectionVariables.get('enforce_auth') || '';",
|
||||
"if (enforceAuth === '1' || String(enforceAuth).toLowerCase() === 'true') {",
|
||||
" pm.test('Without VK and enforce_auth: expect 401', function() { pm.expect(code).to.equal(401); });",
|
||||
"} else {",
|
||||
" pm.test('Without VK and no enforce_auth: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Embedding - No VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var enforceAuth = (pm.environment && pm.environment.get('enforce_auth')) || pm.collectionVariables.get('enforce_auth') || '';",
|
||||
"if (enforceAuth === '1' || String(enforceAuth).toLowerCase() === 'true') {",
|
||||
" pm.test('Without VK and enforce_auth: expect 401', function() { pm.expect(code).to.equal(401); });",
|
||||
"} else {",
|
||||
" pm.test('Without VK and no enforce_auth: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{embedding_model}}\",\n \"input\": \"Hello world\",\n \"encoding_format\": \"float\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/embeddings",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"embeddings"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Responses - No VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"var enforceAuth = (pm.environment && pm.environment.get('enforce_auth')) || pm.collectionVariables.get('enforce_auth') || '';",
|
||||
"if (enforceAuth === '1' || String(enforceAuth).toLowerCase() === 'true') {",
|
||||
" pm.test('Without VK and enforce_auth: expect 401', function() { pm.expect(code).to.equal(401); });",
|
||||
"} else {",
|
||||
" pm.test('Without VK and no enforce_auth: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{responses_model}}\",\n \"input\": \"Say hello\",\n \"max_output_tokens\": 50\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/responses",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"responses"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Inference With VK (x-bf-vk)",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - x-bf-vk",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With valid VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Embedding - x-bf-vk",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With valid VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{embedding_model}}\",\n \"input\": \"Hello world\",\n \"encoding_format\": \"float\"\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/embeddings",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"embeddings"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Responses - x-bf-vk",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With valid VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{responses_model}}\",\n \"input\": \"Say hello\",\n \"max_output_tokens\": 50\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/responses",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"responses"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Inference With VK (Authorization Bearer)",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - Authorization Bearer",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With Bearer VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "Authorization",
|
||||
"value": "Bearer {{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Inference With VK (x-api-key)",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - x-api-key",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('With x-api-key VK: expect 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-api-key",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Rejection - Invalid VK",
|
||||
"item": [
|
||||
{
|
||||
"name": "Chat Completion - x-bf-vk invalid",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Invalid VK: expect 403', function() { pm.expect(code).to.equal(403); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "invalid-not-a-real-key"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - x-bf-vk nonexistent",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Nonexistent VK: expect 403', function() { pm.expect(code).to.equal(403); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "sk-bf-00000000-0000-0000-0000-000000000000"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Teardown - Deactivate",
|
||||
"item": [
|
||||
{
|
||||
"name": "Deactivate Virtual Key",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Deactivate VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "PUT",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"is_active\": false}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - Deactivated VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Deactivated VK: expect 403', function() { pm.expect(code).to.equal(403); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"key": "x-bf-vk",
|
||||
"value": "{{vk_value}}"
|
||||
}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"chat",
|
||||
"completions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Teardown - Delete",
|
||||
"item": [
|
||||
{
|
||||
"name": "Delete Virtual Key",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": [
|
||||
"{{base_url}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"governance",
|
||||
"virtual-keys",
|
||||
"{{vk_id}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
{
|
||||
"info": {
|
||||
"name": "Bifrost V1 - VK Governance Routing",
|
||||
"description": "VK provider_configs routing tests. Creates VK with provider restriction, validates routing via extra_fields.provider.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{"key": "base_url", "value": "http://localhost:8080", "type": "string"},
|
||||
{"key": "provider", "value": "openai", "type": "string"},
|
||||
{"key": "chat_model", "value": "gpt-4o", "type": "string"},
|
||||
{"key": "vk_value", "value": "", "type": "string"},
|
||||
{"key": "vk_id", "value": "", "type": "string"}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Setup - Create VK with provider config",
|
||||
"event": [
|
||||
{
|
||||
"listen": "prerequest",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var ts = Date.now();",
|
||||
"var body = {",
|
||||
" name: 'Routing Test VK ' + ts,",
|
||||
" provider_configs: [{",
|
||||
" provider: pm.collectionVariables.get('provider') || 'openai',",
|
||||
" weight: 1.0,",
|
||||
" allowed_models: ['*'],",
|
||||
" key_ids: ['*']",
|
||||
" }]",
|
||||
"};",
|
||||
"pm.request.body.raw = JSON.stringify(body);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Create VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var vk = json.virtual_key || json;",
|
||||
" if (vk.id) pm.collectionVariables.set('vk_id', vk.id);",
|
||||
" if (vk.value) pm.collectionVariables.set('vk_value', vk.value);",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [{"key": "Content-Type", "value": "application/json"}],
|
||||
"body": {"mode": "raw", "raw": "{}"},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["api", "governance", "virtual-keys"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - model without provider prefix",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var extra = json.extra_fields || {};",
|
||||
" var providerUsed = extra.provider;",
|
||||
" var expected = (pm.collectionVariables.get('provider') || 'openai').toLowerCase();",
|
||||
" pm.test('Provider matches VK config', function() {",
|
||||
" pm.expect(String(providerUsed).toLowerCase()).to.equal(expected);",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - explicit provider prefix",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Status is 2xx', function() { pm.expect(code).to.be.within(200, 299); });",
|
||||
"if (code >= 200 && code <= 299) {",
|
||||
" var json = pm.response.json();",
|
||||
" var extra = json.extra_fields || {};",
|
||||
" var providerUsed = extra.provider;",
|
||||
" var expected = (pm.collectionVariables.get('provider') || 'openai').toLowerCase();",
|
||||
" pm.test('Provider matches VK config (explicit prefix)', function() {",
|
||||
" pm.expect(String(providerUsed).toLowerCase()).to.equal(expected);",
|
||||
" });",
|
||||
"}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"{{provider}}/{{chat_model}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Chat Completion - blocked model (expect 403)",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Model blocked - expect 4xx', function() { pm.expect(code).to.be.oneOf([400, 403]); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{"key": "Content-Type", "value": "application/json"},
|
||||
{"key": "x-bf-vk", "value": "{{vk_value}}"}
|
||||
],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"model\": \"nonexistent-model\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],\n \"max_completion_tokens\": 10,\n \"stream\": false\n}"
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{base_url}}/v1/chat/completions",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["v1", "chat", "completions"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Teardown - Delete VK",
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"var code = pm.response.code;",
|
||||
"pm.test('Delete VK returns 2xx', function() { pm.expect(code).to.be.within(200, 299); });"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{base_url}}/api/governance/virtual-keys/{{vk_id}}",
|
||||
"host": ["{{base_url}}"],
|
||||
"path": ["api", "governance", "virtual-keys", "{{vk_id}}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
2
tests/e2e/api/fixtures/sample-gemini.jsonl
Normal file
2
tests/e2e/api/fixtures/sample-gemini.jsonl
Normal file
@@ -0,0 +1,2 @@
|
||||
{"custom_id": "integration-test-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.0-flash", "messages": [{"role": "user", "content": "Say hello"}]}}
|
||||
{"custom_id": "integration-test-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.0-flash", "messages": [{"role": "user", "content": "Say goodbye"}]}}
|
||||
2
tests/e2e/api/fixtures/sample.jsonl
Normal file
2
tests/e2e/api/fixtures/sample.jsonl
Normal file
@@ -0,0 +1,2 @@
|
||||
{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "Say hello"}]}}
|
||||
{"custom_id": "req-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "Say goodbye"}]}}
|
||||
BIN
tests/e2e/api/fixtures/sample.mp3
Normal file
BIN
tests/e2e/api/fixtures/sample.mp3
Normal file
Binary file not shown.
BIN
tests/e2e/api/fixtures/sample.png
Normal file
BIN
tests/e2e/api/fixtures/sample.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 40 KiB |
1
tests/e2e/api/fixtures/sample.txt
Normal file
1
tests/e2e/api/fixtures/sample.txt
Normal file
@@ -0,0 +1 @@
|
||||
Hello world. This is a sample file for Bifrost Postman/Newman tests.
|
||||
837
tests/e2e/api/newman-reporter-dbverify/index.js
Normal file
837
tests/e2e/api/newman-reporter-dbverify/index.js
Normal file
@@ -0,0 +1,837 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Newman DB Verifier Reporter
|
||||
*
|
||||
* After each 2xx API response, fires SQL queries to verify that CRUD operations
|
||||
* are correctly reflected in the database (PostgreSQL or SQLite).
|
||||
*
|
||||
* Main DB connection is resolved in this order:
|
||||
* 1. --reporter-dbverify-db-url (explicit DSN)
|
||||
* 2. BIFROST_DB_URL env var (explicit DSN)
|
||||
* 3. --reporter-dbverify-config (path to Bifrost config.json; auto-detects type + DSN)
|
||||
* 4. ./config.json (auto-discovered in cwd)
|
||||
*
|
||||
* Logs DB connection (for logs/mcp-logs endpoints) is resolved in this order:
|
||||
* 1. --reporter-dbverify-logs-db-url (explicit DSN)
|
||||
* 2. BIFROST_LOGS_DB_URL env var
|
||||
* 3. Same config.json as above (reads logs_store section)
|
||||
*
|
||||
* Supported DSN formats:
|
||||
* postgresql://user:pass@host:port/db[?sslmode=...]
|
||||
* sqlite:///absolute/path/to/file.db
|
||||
* sqlite://relative/path/to/file.db
|
||||
* /absolute/path/to/file.db (bare path → treated as SQLite)
|
||||
*
|
||||
* Other options:
|
||||
* --reporter-dbverify-silent Suppress per-request log lines
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// ─── Bifrost config.json reader ───────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Resolve an EnvVar field from Bifrost config.
|
||||
* Values can be a plain string, an "env.KEY" reference, or an explicit
|
||||
* {"value": "...", "env_var": "..."} object.
|
||||
*/
|
||||
function resolveEnvVar(val) {
|
||||
if (val == null) return '';
|
||||
if (typeof val === 'object') {
|
||||
const v = val.value || '';
|
||||
if (v.startsWith('env.')) return process.env[v.slice(4)] || '';
|
||||
return v;
|
||||
}
|
||||
const s = String(val);
|
||||
if (s.startsWith('env.')) return process.env[s.slice(4)] || '';
|
||||
return s;
|
||||
}
|
||||
|
||||
function sqliteUrlFromPath(filePath, configPath) {
|
||||
const resolved = path.isAbsolute(filePath)
|
||||
? filePath
|
||||
: path.resolve(path.dirname(configPath), filePath);
|
||||
return `sqlite://${resolved}`;
|
||||
}
|
||||
|
||||
function postgresUrlFromConfig(c) {
|
||||
const host = resolveEnvVar(c.host) || 'localhost';
|
||||
const port = resolveEnvVar(c.port) || '5432';
|
||||
const user = resolveEnvVar(c.user) || 'bifrost';
|
||||
const password = resolveEnvVar(c.password) || '';
|
||||
const dbName = resolveEnvVar(c.db_name) || 'bifrost';
|
||||
const sslMode = resolveEnvVar(c.ssl_mode) || 'disable';
|
||||
return `postgresql://${user}:${encodeURIComponent(password)}@${host}:${port}/${dbName}?sslmode=${sslMode}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a Bifrost config.json and return a DB connection URL for the main
|
||||
* config_store, or null if not enabled / unreadable.
|
||||
*/
|
||||
function dbUrlFromBifrostConfig(configPath) {
|
||||
let raw;
|
||||
try { raw = fs.readFileSync(configPath, 'utf8'); }
|
||||
catch (_) { return null; }
|
||||
|
||||
let cfg;
|
||||
try { cfg = JSON.parse(raw); }
|
||||
catch (_) { return null; }
|
||||
|
||||
const cs = cfg.config_store;
|
||||
if (!cs || !cs.enabled) return null;
|
||||
|
||||
if (cs.type === 'sqlite') {
|
||||
const filePath = resolveEnvVar(cs.config && cs.config.path);
|
||||
if (!filePath) return null;
|
||||
return sqliteUrlFromPath(filePath, configPath);
|
||||
}
|
||||
|
||||
if (cs.type === 'postgres') {
|
||||
return postgresUrlFromConfig(cs.config || {});
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a Bifrost config.json and return a DB connection URL for the logs_store,
|
||||
* or null if not enabled / unreadable.
|
||||
*/
|
||||
function logsDbUrlFromBifrostConfig(configPath) {
|
||||
let raw;
|
||||
try { raw = fs.readFileSync(configPath, 'utf8'); }
|
||||
catch (_) { return null; }
|
||||
|
||||
let cfg;
|
||||
try { cfg = JSON.parse(raw); }
|
||||
catch (_) { return null; }
|
||||
|
||||
const ls = cfg.logs_store;
|
||||
if (!ls || !ls.enabled) return null;
|
||||
|
||||
if (ls.type === 'sqlite') {
|
||||
const filePath = resolveEnvVar(ls.config && ls.config.path);
|
||||
if (!filePath) return null;
|
||||
return sqliteUrlFromPath(filePath, configPath);
|
||||
}
|
||||
|
||||
if (ls.type === 'postgres') {
|
||||
return postgresUrlFromConfig(ls.config || {});
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── DB type detection ────────────────────────────────────────────────────────
|
||||
|
||||
function detectDbType(url) {
|
||||
if (/^postgres(ql)?:\/\//i.test(url)) return 'postgres';
|
||||
return 'sqlite';
|
||||
}
|
||||
|
||||
function resolveSqlitePath(url) {
|
||||
return url.replace(/^sqlite:\/\//i, '');
|
||||
}
|
||||
|
||||
// ─── DB backend abstraction ───────────────────────────────────────────────────
|
||||
|
||||
async function createDbClient(url) {
|
||||
const type = detectDbType(url);
|
||||
|
||||
if (type === 'postgres') {
|
||||
let pg;
|
||||
try { pg = require('pg'); }
|
||||
catch (_) {
|
||||
console.warn('[dbverify] pg module not found. Run: npm install in tests/e2e/api/');
|
||||
return null;
|
||||
}
|
||||
const pgClient = new pg.Client({ connectionString: url });
|
||||
await pgClient.connect();
|
||||
return {
|
||||
type: 'postgres',
|
||||
query: async (sql, params) => {
|
||||
const res = await pgClient.query(sql, params);
|
||||
return { rows: res.rows, rowCount: res.rowCount };
|
||||
},
|
||||
close: () => pgClient.end().catch(() => {}),
|
||||
};
|
||||
}
|
||||
|
||||
// SQLite
|
||||
let Database;
|
||||
try { Database = require('better-sqlite3'); }
|
||||
catch (_) {
|
||||
console.warn('[dbverify] better-sqlite3 not found. Run: npm install in tests/e2e/api/');
|
||||
return null;
|
||||
}
|
||||
const filePath = resolveSqlitePath(url);
|
||||
const db = new Database(filePath, { readonly: true });
|
||||
return {
|
||||
type: 'sqlite',
|
||||
query: async (sql, params) => {
|
||||
const rows = db.prepare(sql.replace(/\$\d+/g, '?')).all(...params);
|
||||
return { rows, rowCount: rows.length };
|
||||
},
|
||||
close: () => { try { db.close(); } catch (_) {} },
|
||||
};
|
||||
}
|
||||
|
||||
// ─── URL → Table mapping ──────────────────────────────────────────────────────
|
||||
//
|
||||
// logsDb: true → query is routed to the logs DB (logs_store) instead of the
|
||||
// main config DB (config_store).
|
||||
|
||||
const URL_TABLE_MAP = [
|
||||
// ── Specific (id-bearing) patterns — matched before collection patterns ────
|
||||
|
||||
{
|
||||
pattern: /\/api\/governance\/customers\/([^/?#]+)/,
|
||||
table: 'governance_customers', idParam: 1, idColumn: 'id',
|
||||
verifyFields: ['id', 'name'],
|
||||
bodyId: (b) => b && (b.id || (b.customer && b.customer.id)),
|
||||
bodyFields: (b) => b && (b.customer || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/teams\/([^/?#]+)/,
|
||||
table: 'governance_teams', idParam: 1, idColumn: 'id',
|
||||
verifyFields: ['id', 'name', 'customer_id'],
|
||||
bodyId: (b) => b && (b.id || (b.team && b.team.id)),
|
||||
bodyFields: (b) => b && (b.team || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/virtual-keys\/([^/?#]+)/,
|
||||
table: 'governance_virtual_keys', idParam: 1, idColumn: 'id',
|
||||
verifyFields: ['id', 'name', 'is_active'],
|
||||
bodyId: (b) => b && (b.id || (b.virtual_key && b.virtual_key.id)),
|
||||
bodyFields: (b) => b && (b.virtual_key || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/routing-rules\/([^/?#]+)/,
|
||||
table: 'routing_rules', idParam: 1, idColumn: 'id',
|
||||
verifyFields: ['id', 'name', 'enabled', 'provider', 'scope'],
|
||||
bodyId: (b) => b && (b.id || (b.rule && b.rule.id)),
|
||||
bodyFields: (b) => b && (b.rule || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/model-configs\/([^/?#]+)/,
|
||||
table: 'governance_model_configs', idParam: 1, idColumn: 'id',
|
||||
verifyFields: ['id', 'model_name', 'provider'],
|
||||
bodyId: (b) => b && (b.id || (b.model_config && b.model_config.id)),
|
||||
bodyFields: (b) => b && (b.model_config || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/providers\/([^/?#]+)/,
|
||||
table: 'config_providers', idParam: 1, idColumn: 'name',
|
||||
verifyFields: ['name'],
|
||||
bodyId: (b) => b && (b.provider && (b.provider.Provider || b.provider.name)),
|
||||
bodyFields: (b) => b && b.provider && { name: b.provider.Provider || b.provider.name },
|
||||
deleteVerifiesExists: true,
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/providers\/([^/?#]+)/,
|
||||
table: 'config_providers', idParam: 1, idColumn: 'name',
|
||||
verifyFields: ['name', 'status', 'send_back_raw_request', 'send_back_raw_response'],
|
||||
bodyId: (b) => b && (b.name || (b.provider && b.provider.name)),
|
||||
bodyFields: (b) => b && (b.provider || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/mcp\/client\/([^/?#]+)/,
|
||||
table: 'config_mcp_clients', idParam: 1, idColumn: 'client_id',
|
||||
verifyFields: ['client_id', 'name', 'connection_type'],
|
||||
bodyId: (b) => b && (b.client_id || (b.mcp_client && b.mcp_client.client_id)),
|
||||
bodyFields: (b) => b && (b.mcp_client || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/logs\/([^/?#]+)/,
|
||||
table: 'logs', idParam: 1, idColumn: 'id', logsDb: true,
|
||||
verifyFields: ['id'],
|
||||
bodyId: (b) => b && (b.id || (b.log && b.log.id)),
|
||||
bodyFields: (b) => b && (b.log || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/mcp-logs\/([^/?#]+)/,
|
||||
table: 'mcp_tool_logs', idParam: 1, idColumn: 'id', logsDb: true,
|
||||
verifyFields: ['id'],
|
||||
bodyId: (b) => b && (b.id || (b.log && b.log.id)),
|
||||
bodyFields: (b) => b && (b.log || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/plugins\/([^/?#]+)/,
|
||||
table: 'config_plugins', idParam: 1, idColumn: 'name',
|
||||
verifyFields: ['name', 'enabled', 'path'],
|
||||
bodyId: (b) => b && (b.name || (b.plugin && b.plugin.name)),
|
||||
bodyFields: (b) => b && (b.plugin || b),
|
||||
},
|
||||
|
||||
// ── Collection / aggregate endpoints ──────────────────────────────────────
|
||||
|
||||
{
|
||||
pattern: /\/api\/governance\/customers$/,
|
||||
table: 'governance_customers', idParam: null, idColumn: 'id',
|
||||
verifyFields: ['id', 'name'],
|
||||
bodyId: (b) => b && (b.id || (b.customer && b.customer.id)),
|
||||
bodyFields: (b) => b && (b.customer || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/teams$/,
|
||||
table: 'governance_teams', idParam: null, idColumn: 'id',
|
||||
verifyFields: ['id', 'name', 'customer_id'],
|
||||
bodyId: (b) => b && (b.id || (b.team && b.team.id)),
|
||||
bodyFields: (b) => b && (b.team || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/virtual-keys$/,
|
||||
table: 'governance_virtual_keys', idParam: null, idColumn: 'id',
|
||||
verifyFields: ['id', 'name', 'is_active'],
|
||||
bodyId: (b) => b && (b.id || (b.virtual_key && b.virtual_key.id)),
|
||||
bodyFields: (b) => b && (b.virtual_key || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/routing-rules$/,
|
||||
table: 'routing_rules', idParam: null, idColumn: 'id',
|
||||
verifyFields: ['id', 'name', 'enabled', 'provider', 'scope'],
|
||||
bodyId: (b) => b && (b.id || (b.rule && b.rule.id)),
|
||||
bodyFields: (b) => b && (b.rule || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/governance\/model-configs$/,
|
||||
table: 'governance_model_configs', idParam: null, idColumn: 'id',
|
||||
verifyFields: ['id', 'model_name', 'provider'],
|
||||
bodyId: (b) => b && (b.id || (b.model_config && b.model_config.id)),
|
||||
bodyFields: (b) => b && (b.model_config || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/providers$/,
|
||||
table: 'config_providers', idParam: null, idColumn: 'name',
|
||||
verifyFields: ['name', 'status', 'send_back_raw_request', 'send_back_raw_response'],
|
||||
bodyId: (b) => b && (b.name || (b.provider && b.provider.name)),
|
||||
bodyFields: (b) => b && (b.provider || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/plugins$/,
|
||||
table: 'config_plugins', idParam: null, idColumn: 'name',
|
||||
verifyFields: ['name', 'enabled', 'path'],
|
||||
bodyId: (b) => b && (b.name || (b.plugin && b.plugin.name)),
|
||||
bodyFields: (b) => b && (b.plugin || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/mcp\/client$/,
|
||||
table: 'config_mcp_clients', idParam: null, idColumn: 'client_id',
|
||||
verifyFields: ['client_id', 'name', 'connection_type'],
|
||||
bodyId: (b) => b && (b.client_id || (b.mcp_client && b.mcp_client.client_id)),
|
||||
bodyFields: (b) => b && (b.mcp_client || b),
|
||||
},
|
||||
{
|
||||
pattern: /\/api\/mcp\/clients$/,
|
||||
table: 'config_mcp_clients', idParam: null, idColumn: 'client_id',
|
||||
verifyFields: ['client_id', 'name', 'connection_type'],
|
||||
bodyId: (b) => b && (b.client_id || (b.mcp_client && b.mcp_client.client_id)),
|
||||
bodyFields: (b) => b && (b.mcp_client || b),
|
||||
},
|
||||
// Proxy config — stored as a JSON blob in governance_config.value under key "proxy_config"
|
||||
// PUT response is {"status":"success",...} so we compare the request body against the DB blob.
|
||||
// GET response is the proxy config object directly, which also works with jsonBlobColumn.
|
||||
{
|
||||
pattern: /\/api\/proxy-config$/,
|
||||
table: 'governance_config', idParam: null, idColumn: 'key',
|
||||
jsonBlobColumn: 'value',
|
||||
useRequestBody: true,
|
||||
verifyFields: ['enabled', 'type', 'url', 'timeout', 'enable_for_inference', 'enable_for_api', 'enable_for_scim'],
|
||||
bodyId: () => 'proxy_config',
|
||||
bodyFields: (b) => b,
|
||||
},
|
||||
|
||||
// Config — client_config in config_client, framework_config in framework_configs (multi-table)
|
||||
{
|
||||
pattern: /\/api\/config$/,
|
||||
multiTable: [
|
||||
{
|
||||
table: 'config_client',
|
||||
idColumn: 'id',
|
||||
verifyFields: ['drop_excess_requests', 'log_retention_days', 'mcp_agent_depth', 'mcp_tool_execution_timeout'],
|
||||
bodyFields: (b) => b && b.client_config,
|
||||
},
|
||||
{
|
||||
table: 'framework_configs',
|
||||
idColumn: 'id',
|
||||
verifyFields: ['pricing_url', 'pricing_sync_interval'],
|
||||
bodyFields: (b) => b && b.framework_config,
|
||||
},
|
||||
],
|
||||
useRequestBody: true,
|
||||
bodyId: () => null,
|
||||
bodyFields: () => null,
|
||||
},
|
||||
|
||||
// Version — build-time constant, not stored in DB
|
||||
{
|
||||
pattern: /\/api\/version$/,
|
||||
skipReason: 'version is build-time constant, not stored in DB',
|
||||
},
|
||||
|
||||
// Read-only table-accessible endpoints (COUNT check)
|
||||
{ pattern: /\/api\/keys$/, table: 'config_keys', idParam: null, idColumn: 'id', verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/models$/, table: 'config_providers', idParam: null, idColumn: 'name', verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/models\/base$/, table: 'config_providers', idParam: null, idColumn: 'name', verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/governance\/budgets$/, table: 'governance_budgets', idParam: null, idColumn: 'id', verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/governance\/rate-limits$/, table: 'governance_rate_limits', idParam: null, idColumn: 'id', verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/governance\/providers$/, table: 'config_providers', idParam: null, idColumn: 'name', verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
|
||||
// Logs aggregate endpoints — verify the table is accessible (COUNT check)
|
||||
{ pattern: /\/api\/logs\/stats$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/histogram$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/histogram\/tokens$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/histogram\/cost$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/histogram\/models$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/histogram\/latency$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/histogram\/cost\/by-provider$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/histogram\/tokens\/by-provider$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/histogram\/latency\/by-provider$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/filterdata$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/dropped$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs\/recalculate-cost$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/logs$/, table: 'logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
// MCP logs aggregate endpoints
|
||||
{ pattern: /\/api\/mcp-logs\/stats$/, table: 'mcp_tool_logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/mcp-logs\/filterdata$/, table: 'mcp_tool_logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
{ pattern: /\/api\/mcp-logs$/, table: 'mcp_tool_logs', idParam: null, idColumn: 'id', logsDb: true, verifyFields: [], bodyId: () => null, bodyFields: () => null },
|
||||
];
|
||||
|
||||
function matchMapping(urlPath) {
|
||||
// Sort priority (highest first):
|
||||
// 1. Literal patterns with no capturing groups – e.g. /api/mcp-logs/stats$
|
||||
// These are more specific than wildcard captures and must be tried first.
|
||||
// 2. Wildcard id-bearing patterns – e.g. /api/mcp-logs/([^/?#]+)
|
||||
// 3. Collection / aggregate patterns – e.g. /api/mcp-logs$
|
||||
const sorted = [...URL_TABLE_MAP].sort((a, b) => {
|
||||
const aHasCapture = /\([^)]+\)/.test(a.pattern.source);
|
||||
const bHasCapture = /\([^)]+\)/.test(b.pattern.source);
|
||||
if (aHasCapture !== bHasCapture) return aHasCapture ? 1 : -1;
|
||||
return (b.idParam !== null ? 1 : 0) - (a.idParam !== null ? 1 : 0);
|
||||
});
|
||||
for (const mapping of sorted) {
|
||||
const m = urlPath.match(mapping.pattern);
|
||||
if (m) return { mapping, urlId: mapping.idParam !== null ? m[mapping.idParam] : null };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function parseBody(response) {
|
||||
try { return JSON.parse(response.stream ? response.stream.toString() : ''); }
|
||||
catch (_) { return null; }
|
||||
}
|
||||
|
||||
function parseRequestBody(request) {
|
||||
try {
|
||||
const raw = request && request.body && request.body.raw;
|
||||
if (!raw) return null;
|
||||
const str = typeof raw === 'string' ? raw : (raw && raw.toString ? raw.toString() : '');
|
||||
return str ? JSON.parse(str) : null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Normalize DB/JSON values for comparison. SQLite/Postgres return booleans as 0/1; API returns false/true. */
|
||||
function valuesEqual(dbVal, respVal) {
|
||||
if (dbVal === respVal) return true;
|
||||
if (String(dbVal) === String(respVal)) return true;
|
||||
// Boolean: 0/1 (DB) vs false/true (JSON)
|
||||
const dbBool = dbVal === 1 || dbVal === true || (typeof dbVal === 'string' && /^true|1$/i.test(dbVal));
|
||||
const respBool = respVal === 1 || respVal === true || (typeof respVal === 'string' && /^true|1$/i.test(respVal));
|
||||
const dbIsBoolLike = dbVal === 0 || dbVal === 1 || dbVal === true || dbVal === false || (typeof dbVal === 'string' && /^true|false|0|1$/i.test(dbVal));
|
||||
const respIsBoolLike = respVal === 0 || respVal === 1 || respVal === true || respVal === false || (typeof respVal === 'string' && /^true|false|0|1$/i.test(respVal));
|
||||
if (dbIsBoolLike && respIsBoolLike) return dbBool === respBool;
|
||||
return false;
|
||||
}
|
||||
|
||||
function checkFieldMismatches(dbRow, respFields, verifyFields) {
|
||||
if (!respFields || typeof respFields !== 'object') return [];
|
||||
return verifyFields
|
||||
.filter(f => f in respFields)
|
||||
.filter(f => !valuesEqual(dbRow[f], respFields[f]))
|
||||
.map(f => `${f}: db=${dbRow[f]} resp=${respFields[f]}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Like checkFieldMismatches but the dbRow has a single JSON blob column.
|
||||
* Parses dbRow[jsonBlobColumn] as JSON and compares fields against respFields.
|
||||
*/
|
||||
function checkJsonBlobMismatches(dbRow, jsonBlobColumn, respFields, verifyFields) {
|
||||
if (!respFields || typeof respFields !== 'object') return [];
|
||||
let blob = {};
|
||||
try { blob = JSON.parse(dbRow[jsonBlobColumn] || '{}'); } catch (_) {}
|
||||
return verifyFields
|
||||
.filter(f => f in respFields)
|
||||
.filter(f => !valuesEqual(blob[f], respFields[f]))
|
||||
.map(f => `${f}: db=${blob[f]} resp=${respFields[f]}`);
|
||||
}
|
||||
|
||||
function pad(str, len) {
|
||||
str = String(str || '');
|
||||
return str.length >= len ? str : str + ' '.repeat(len - str.length);
|
||||
}
|
||||
|
||||
// ─── Verification handlers ────────────────────────────────────────────────────
|
||||
|
||||
async function verifyCreated(db, m, id, body, name) {
|
||||
if (!id) return { name, result: 'SKIP', detail: 'No record ID in response' };
|
||||
const selectCols = m.jsonBlobColumn ? m.jsonBlobColumn
|
||||
: (m.verifyFields.length ? m.verifyFields.join(', ') : m.idColumn);
|
||||
const { rows } = await db.query(
|
||||
`SELECT ${selectCols} FROM ${m.table} WHERE ${m.idColumn} = $1`, [id]);
|
||||
if (!rows.length) return { name, result: 'FAIL', detail: `Row NOT found in ${m.table} where ${m.idColumn}=${id}` };
|
||||
const mm = m.jsonBlobColumn
|
||||
? checkJsonBlobMismatches(rows[0], m.jsonBlobColumn, m.bodyFields(body), m.verifyFields)
|
||||
: checkFieldMismatches(rows[0], m.bodyFields(body), m.verifyFields);
|
||||
if (mm.length) return { name, result: 'FAIL', detail: `Field mismatch: ${mm.join(', ')}` };
|
||||
return { name, result: 'PASS', detail: `Row created in ${m.table}: ${m.idColumn}=${id}` };
|
||||
}
|
||||
|
||||
async function verifyExists(db, m, id, body, name) {
|
||||
if (!id) {
|
||||
const { rows } = await db.query(`SELECT COUNT(*) AS cnt FROM ${m.table}`, []);
|
||||
const cnt = rows[0].cnt !== undefined ? rows[0].cnt : rows[0].count;
|
||||
return { name, result: 'PASS', detail: `${m.table} accessible, ${cnt} rows` };
|
||||
}
|
||||
const selectCols = m.jsonBlobColumn ? m.jsonBlobColumn
|
||||
: (m.verifyFields.length ? m.verifyFields.join(', ') : m.idColumn);
|
||||
const { rows } = await db.query(
|
||||
`SELECT ${selectCols} FROM ${m.table} WHERE ${m.idColumn} = $1`, [id]);
|
||||
if (!rows.length) return { name, result: 'FAIL', detail: `Row NOT found in ${m.table} where ${m.idColumn}=${id}` };
|
||||
const mm = m.jsonBlobColumn
|
||||
? checkJsonBlobMismatches(rows[0], m.jsonBlobColumn, m.bodyFields(body), m.verifyFields)
|
||||
: checkFieldMismatches(rows[0], m.bodyFields(body), m.verifyFields);
|
||||
if (mm.length) return { name, result: 'FAIL', detail: `Field mismatch: ${mm.join(', ')}` };
|
||||
return { name, result: 'PASS', detail: `Record verified in ${m.table}: ${m.idColumn}=${id}` };
|
||||
}
|
||||
|
||||
async function verifyUpdated(db, m, id, body, name) {
|
||||
if (!id) return { name, result: 'SKIP', detail: 'No record ID in response' };
|
||||
const selectCols = m.jsonBlobColumn ? m.jsonBlobColumn
|
||||
: (m.verifyFields.length ? m.verifyFields.join(', ') : m.idColumn);
|
||||
const { rows } = await db.query(
|
||||
`SELECT ${selectCols} FROM ${m.table} WHERE ${m.idColumn} = $1`, [id]);
|
||||
if (!rows.length) return { name, result: 'FAIL', detail: `Row NOT found in ${m.table} where ${m.idColumn}=${id}` };
|
||||
const mm = m.jsonBlobColumn
|
||||
? checkJsonBlobMismatches(rows[0], m.jsonBlobColumn, m.bodyFields(body), m.verifyFields)
|
||||
: checkFieldMismatches(rows[0], m.bodyFields(body), m.verifyFields);
|
||||
if (mm.length) return { name, result: 'FAIL', detail: `Update NOT reflected: ${mm.join(', ')}` };
|
||||
return { name, result: 'PASS', detail: `Record updated in ${m.table}: ${m.idColumn}=${id}` };
|
||||
}
|
||||
|
||||
/** Verify a single-row table was updated (no id in URL). SELECT LIMIT 1 and compare. */
|
||||
async function verifyUpdatedSingleRow(db, table, idColumn, verifyFields, bodyFields, body, name) {
|
||||
const respFields = bodyFields && bodyFields(body);
|
||||
const selectCols = verifyFields.length ? verifyFields.join(', ') : idColumn;
|
||||
const { rows } = await db.query(
|
||||
`SELECT ${selectCols} FROM ${table} LIMIT 1`, []);
|
||||
if (!rows.length) return { name, result: 'FAIL', detail: `No row in ${table}` };
|
||||
const mm = checkFieldMismatches(rows[0], respFields, verifyFields);
|
||||
if (mm.length) return { name, result: 'FAIL', detail: `Update NOT reflected in ${table}: ${mm.join(', ')}` };
|
||||
return { name, result: 'PASS', detail: `Record updated in ${table}` };
|
||||
}
|
||||
|
||||
async function verifyDeleted(db, m, id, name) {
|
||||
if (!id) return { name, result: 'SKIP', detail: 'No record ID extractable from DELETE URL' };
|
||||
const { rows } = await db.query(
|
||||
`SELECT COUNT(*) AS cnt FROM ${m.table} WHERE ${m.idColumn} = $1`, [id]);
|
||||
const cnt = parseInt(rows[0].cnt !== undefined ? rows[0].cnt : rows[0].count, 10);
|
||||
if (cnt > 0) return { name, result: 'FAIL', detail: `Row still exists in ${m.table}: ${m.idColumn}=${id}` };
|
||||
return { name, result: 'PASS', detail: `Row removed from ${m.table}: ${m.idColumn}=${id}` };
|
||||
}
|
||||
|
||||
async function runVerification(db, method, mapping, id, body, name) {
|
||||
switch (method) {
|
||||
case 'POST': return verifyCreated(db, mapping, id, body, name);
|
||||
case 'GET': return verifyExists(db, mapping, id, body, name);
|
||||
case 'PUT':
|
||||
case 'PATCH': return verifyUpdated(db, mapping, id, body, name);
|
||||
case 'DELETE':
|
||||
if (mapping.deleteVerifiesExists) return verifyExists(db, mapping, id, body, name);
|
||||
return verifyDeleted(db, mapping, id, name);
|
||||
default: return { name, result: 'SKIP', detail: `Method ${method} not verified` };
|
||||
}
|
||||
}
|
||||
|
||||
/** Run verification for multi-table mappings (e.g. /api/config). */
|
||||
async function runMultiTableVerification(db, method, mapping, body, name) {
|
||||
const tables = mapping.multiTable;
|
||||
const results = [];
|
||||
for (const t of tables) {
|
||||
if (method === 'GET') {
|
||||
const syntheticMapping = { table: t.table, idParam: null, idColumn: t.idColumn, verifyFields: [], bodyId: () => null, bodyFields: () => null };
|
||||
const r = await verifyExists(db, syntheticMapping, null, null, name);
|
||||
results.push(r);
|
||||
} else if (method === 'PUT' || method === 'PATCH') {
|
||||
const r = await verifyUpdatedSingleRow(db, t.table, t.idColumn, t.verifyFields || [], t.bodyFields, body, name);
|
||||
results.push(r);
|
||||
} else {
|
||||
results.push({ name, result: 'SKIP', detail: `Method ${method} not verified for multi-table` });
|
||||
}
|
||||
}
|
||||
const failed = results.filter((r) => r.result === 'FAIL');
|
||||
const passed = results.filter((r) => r.result === 'PASS');
|
||||
if (failed.length > 0) return { name, result: 'FAIL', detail: failed.map((f) => f.detail).join('; ') };
|
||||
if (passed.length === 0) return results[0] || { name, result: 'SKIP', detail: 'No verifications run' };
|
||||
const tableNames = tables.map((t) => t.table).join(', ');
|
||||
return { name, result: 'PASS', detail: `${tableNames} verified` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single request's DB verification (immediate or from queue).
|
||||
* Handles bulk DELETE, tracks promises, pushes results.
|
||||
*/
|
||||
function processRequestVerification(opts) {
|
||||
const {
|
||||
activeDb, method, mapping, urlId, responseBody, name, request,
|
||||
pendingVerifications, results, silent,
|
||||
} = opts;
|
||||
|
||||
// When useRequestBody is set, prefer the parsed request body for field comparison
|
||||
// (e.g. PUT endpoints that return a generic success response rather than the resource).
|
||||
// For GET requests the body is null so it naturally falls back to responseBody.
|
||||
const verifyBody = (mapping.useRequestBody && parseRequestBody(request)) || responseBody;
|
||||
|
||||
// Multi-table verification (e.g. /api/config → config_client + framework_configs)
|
||||
if (mapping.multiTable) {
|
||||
const p = runMultiTableVerification(activeDb, method, mapping, verifyBody, name);
|
||||
pendingVerifications.push(p);
|
||||
p.then((r) => {
|
||||
results.push(r);
|
||||
if (!silent) {
|
||||
const icon = r.result === 'PASS' ? '✓' : r.result === 'SKIP' ? '~' : '✗';
|
||||
console.log(`[dbverify] ${icon} ${r.name}: ${r.detail}`);
|
||||
}
|
||||
}).catch((e) => {
|
||||
const r = { name, result: 'FAIL', detail: `Query error: ${e.message}` };
|
||||
results.push(r);
|
||||
if (!silent) console.log(`[dbverify] ✗ ${name}: ${r.detail}`);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let recordId = urlId || (verifyBody && mapping.bodyId(verifyBody));
|
||||
|
||||
// Bulk DELETE: extract ids from request body
|
||||
if (method === 'DELETE' && !recordId) {
|
||||
const reqBody = parseRequestBody(request);
|
||||
const ids = (reqBody && Array.isArray(reqBody.ids) && reqBody.ids.length > 0) ? reqBody.ids : null;
|
||||
if (ids) {
|
||||
ids.forEach((id, i) => {
|
||||
const p = runVerification(activeDb, method, mapping, id, verifyBody, ids.length > 1 ? `${name} [id=${id}]` : name);
|
||||
pendingVerifications.push(p);
|
||||
p.then((r) => {
|
||||
results.push(r);
|
||||
if (!silent) {
|
||||
const icon = r.result === 'PASS' ? '✓' : r.result === 'SKIP' ? '~' : '✗';
|
||||
console.log(`[dbverify] ${icon} ${r.name}: ${r.detail}`);
|
||||
}
|
||||
}).catch((e) => {
|
||||
const r = { name, result: 'FAIL', detail: `Query error: ${e.message}` };
|
||||
results.push(r);
|
||||
if (!silent) console.log(`[dbverify] ✗ ${name}: ${r.detail}`);
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const p = runVerification(activeDb, method, mapping, recordId, verifyBody, name);
|
||||
pendingVerifications.push(p);
|
||||
p.then((r) => {
|
||||
results.push(r);
|
||||
if (!silent) {
|
||||
const icon = r.result === 'PASS' ? '✓' : r.result === 'SKIP' ? '~' : '✗';
|
||||
console.log(`[dbverify] ${icon} ${r.name}: ${r.detail}`);
|
||||
}
|
||||
}).catch((e) => {
|
||||
const r = { name, result: 'FAIL', detail: `Query error: ${e.message}` };
|
||||
results.push(r);
|
||||
if (!silent) console.log(`[dbverify] ✗ ${name}: ${r.detail}`);
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Summary ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function printSummary(results, dbType) {
|
||||
const passed = results.filter(r => r.result === 'PASS').length;
|
||||
const failed = results.filter(r => r.result === 'FAIL').length;
|
||||
const skipped = results.filter(r => r.result === 'SKIP').length;
|
||||
|
||||
const nameW = Math.max(20, ...results.map(r => (r.name || '').length));
|
||||
const resultW = 6;
|
||||
const detailW = Math.max(52, ...results.map(r => (r.detail || '').length));
|
||||
const totalW = nameW + resultW + detailW + 7;
|
||||
|
||||
const hline = '─'.repeat(totalW);
|
||||
const dline = '═'.repeat(totalW);
|
||||
|
||||
console.log('');
|
||||
console.log('╔' + dline + '╗');
|
||||
console.log('║' + pad(` DB Verification Results (${dbType})`, totalW) + '║');
|
||||
console.log('╠' + hline + '╣');
|
||||
console.log('║ ' + pad('Request', nameW) + ' │ ' + pad('Result', resultW) + ' │ ' + pad('Detail', detailW) + ' ║');
|
||||
console.log('╠' + hline + '╣');
|
||||
for (const r of results) {
|
||||
console.log(
|
||||
'║ ' + pad(r.name || '', nameW) +
|
||||
' │ ' + pad(r.result, resultW) +
|
||||
' │ ' + pad(r.detail || '', detailW) + ' ║'
|
||||
);
|
||||
}
|
||||
console.log('╚' + dline + '╝');
|
||||
console.log(`DB Checks: ${passed} passed, ${failed} failed, ${skipped} skipped (non-2xx or unmapped)`);
|
||||
console.log('');
|
||||
if (failed > 0) console.warn(`[dbverify] WARNING: ${failed} DB verification(s) FAILED`);
|
||||
}
|
||||
|
||||
// ─── Reporter entry point ─────────────────────────────────────────────────────
|
||||
|
||||
module.exports = function (newman, options) {
|
||||
const silent = !!(options && options['silent']);
|
||||
|
||||
const configPath = (options && options['config'])
|
||||
|| process.env.BIFROST_CONFIG_PATH
|
||||
|| path.resolve(process.cwd(), 'config.json');
|
||||
|
||||
// Main DB (config_store)
|
||||
let dbUrl = (options && options['db-url']) || process.env.BIFROST_DB_URL || null;
|
||||
if (!dbUrl) {
|
||||
dbUrl = dbUrlFromBifrostConfig(configPath);
|
||||
if (dbUrl && !silent) console.log(`[dbverify] Auto-detected main DB from config: ${configPath}`);
|
||||
}
|
||||
if (!dbUrl) {
|
||||
console.warn('[dbverify] No main DB URL found. Provide --reporter-dbverify-db-url, BIFROST_DB_URL, or --reporter-dbverify-config. Skipping DB checks.');
|
||||
}
|
||||
|
||||
// Logs DB (logs_store)
|
||||
let logsDbUrl = (options && options['logs-db-url']) || process.env.BIFROST_LOGS_DB_URL || null;
|
||||
if (!logsDbUrl) {
|
||||
logsDbUrl = logsDbUrlFromBifrostConfig(configPath);
|
||||
if (logsDbUrl && !silent) console.log(`[dbverify] Auto-detected logs DB from config: ${configPath}`);
|
||||
}
|
||||
|
||||
const dbType = dbUrl ? detectDbType(dbUrl) : 'unknown';
|
||||
const results = [];
|
||||
const pendingVerifications = [];
|
||||
const earlyMainDbQueue = [];
|
||||
const earlyLogsDbQueue = [];
|
||||
let db = null;
|
||||
let logsDb = null;
|
||||
let dbReady = false;
|
||||
let logsDbReady = false;
|
||||
|
||||
function drainQueue(queue, activeDb) {
|
||||
while (queue.length > 0) {
|
||||
const item = queue.shift();
|
||||
processRequestVerification({
|
||||
activeDb, method: item.method, mapping: item.mapping, urlId: item.urlId,
|
||||
responseBody: item.responseBody, name: item.name, request: item.request,
|
||||
pendingVerifications, results, silent,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
newman.on('start', function (err) {
|
||||
if (err) return;
|
||||
|
||||
if (dbUrl) {
|
||||
const safeUrl = dbUrl.replace(/:([^:@]+)@/, ':***@');
|
||||
createDbClient(dbUrl)
|
||||
.then((client) => {
|
||||
db = client;
|
||||
dbReady = !!client;
|
||||
if (dbReady && !silent) console.log(`[dbverify] Connected to ${dbType} DB: ${safeUrl}`);
|
||||
if (dbReady && db) drainQueue(earlyMainDbQueue, db);
|
||||
})
|
||||
.catch((e) => {
|
||||
dbReady = false;
|
||||
console.warn(`[dbverify] Main DB not reachable, skipping DB checks: ${e.message}`);
|
||||
earlyMainDbQueue.forEach((item) => results.push({ name: item.name, result: 'SKIP', detail: 'Main DB not connected' }));
|
||||
earlyMainDbQueue.length = 0;
|
||||
});
|
||||
}
|
||||
|
||||
if (logsDbUrl) {
|
||||
const safeLogsUrl = logsDbUrl.replace(/:([^:@]+)@/, ':***@');
|
||||
createDbClient(logsDbUrl)
|
||||
.then((client) => {
|
||||
logsDb = client;
|
||||
logsDbReady = !!client;
|
||||
if (logsDbReady && !silent) console.log(`[dbverify] Connected to logs DB (${detectDbType(logsDbUrl)}): ${safeLogsUrl}`);
|
||||
if (logsDbReady && logsDb) drainQueue(earlyLogsDbQueue, logsDb);
|
||||
})
|
||||
.catch((e) => {
|
||||
logsDbReady = false;
|
||||
console.warn(`[dbverify] Logs DB not reachable, skipping logs DB checks: ${e.message}`);
|
||||
earlyLogsDbQueue.forEach((item) => results.push({ name: item.name, result: 'SKIP', detail: 'Logs DB not connected' }));
|
||||
earlyLogsDbQueue.length = 0;
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
newman.on('request', function (err, args) {
|
||||
if (err) return;
|
||||
|
||||
const response = args.response;
|
||||
const request = args.request;
|
||||
const name = (args.item && args.item.name) || 'Unknown Request';
|
||||
const statusCode = response && response.code;
|
||||
|
||||
if (!statusCode || statusCode < 200 || statusCode > 299) {
|
||||
results.push({ name, result: 'SKIP', detail: `HTTP ${statusCode || '?'} (non-2xx)` });
|
||||
return;
|
||||
}
|
||||
|
||||
const method = request.method.toUpperCase();
|
||||
const urlPath = request.url.toString()
|
||||
.replace(/\?.*$/, '')
|
||||
.replace(/^https?:\/\/[^/]+/, '');
|
||||
|
||||
const match = matchMapping(urlPath);
|
||||
if (!match) {
|
||||
results.push({ name, result: 'SKIP', detail: 'URL not mapped to DB table' });
|
||||
return;
|
||||
}
|
||||
|
||||
const { mapping, urlId } = match;
|
||||
|
||||
if (mapping.skipReason) {
|
||||
results.push({ name, result: 'SKIP', detail: mapping.skipReason });
|
||||
return;
|
||||
}
|
||||
|
||||
// Pick the right DB client
|
||||
const isLogsTable = !!mapping.logsDb;
|
||||
const activeDb = isLogsTable ? logsDb : db;
|
||||
const activeReady = isLogsTable ? logsDbReady : dbReady;
|
||||
|
||||
const responseBody = parseBody(response);
|
||||
|
||||
if (!activeReady || !activeDb) {
|
||||
const queue = isLogsTable ? earlyLogsDbQueue : earlyMainDbQueue;
|
||||
queue.push({
|
||||
method, mapping, urlId, responseBody, name, request,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
processRequestVerification({
|
||||
activeDb, method, mapping, urlId, responseBody, name, request,
|
||||
pendingVerifications, results, silent,
|
||||
});
|
||||
});
|
||||
|
||||
newman.on('done', function () {
|
||||
Promise.allSettled(pendingVerifications).then(() => {
|
||||
if (db) db.close();
|
||||
if (logsDb) logsDb.close();
|
||||
if (results.length > 0) printSummary(results, dbType);
|
||||
});
|
||||
});
|
||||
};
|
||||
6
tests/e2e/api/newman-reporter-dbverify/package.json
Normal file
6
tests/e2e/api/newman-reporter-dbverify/package.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"name": "newman-reporter-dbverify",
|
||||
"version": "1.0.0",
|
||||
"description": "Newman reporter that verifies CRUD operations against PostgreSQL",
|
||||
"main": "index.js"
|
||||
}
|
||||
11
tests/e2e/api/package.json
Normal file
11
tests/e2e/api/package.json
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "bifrost-e2e-api-tests",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "E2E API test dependencies",
|
||||
"dependencies": {
|
||||
"pg": "^8.13.0",
|
||||
"better-sqlite3": "^11.0.0",
|
||||
"newman-reporter-dbverify": "file:./newman-reporter-dbverify"
|
||||
}
|
||||
}
|
||||
797
tests/e2e/api/provider-capabilities.json
Normal file
797
tests/e2e/api/provider-capabilities.json
Normal file
@@ -0,0 +1,797 @@
|
||||
{
|
||||
"description": "Provider capability matrix for integration tests. Each provider has explicit booleans per operation (derived from core/providers/* provider.go NewUnsupportedOperationError). Used to skip requests when running with all provider envs.",
|
||||
"providers": {
|
||||
"openai": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": true,
|
||||
"responses": true,
|
||||
"responses_with_tools": true,
|
||||
"count_tokens": true,
|
||||
"embedding": true,
|
||||
"speech": true,
|
||||
"transcription": true,
|
||||
"list_models": true,
|
||||
"image_generation": true,
|
||||
"image_variation": false,
|
||||
"image_edit": true,
|
||||
"batch_create": true,
|
||||
"batch_list": true,
|
||||
"batch_retrieve": true,
|
||||
"batch_cancel": true,
|
||||
"batch_results": true,
|
||||
"file_batch_input": true,
|
||||
"batch_create_file": true,
|
||||
"file_upload": true,
|
||||
"file_list": true,
|
||||
"file_retrieve": true,
|
||||
"file_delete": true,
|
||||
"file_content": true,
|
||||
"container_create": true,
|
||||
"container_list": true,
|
||||
"container_retrieve": true,
|
||||
"container_delete": true,
|
||||
"container_file_create": true,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": true,
|
||||
"container_file_retrieve": true,
|
||||
"container_file_content": true,
|
||||
"container_file_delete": true,
|
||||
"video_generation": true,
|
||||
"video_retrieve": true,
|
||||
"video_download": true,
|
||||
"video_delete": true,
|
||||
"video_list": true,
|
||||
"video_remix": true,
|
||||
"rerank": false
|
||||
},
|
||||
"anthropic": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": true,
|
||||
"embedding": false,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": true,
|
||||
"batch_list": true,
|
||||
"batch_retrieve": true,
|
||||
"batch_cancel": true,
|
||||
"batch_results": true,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": true,
|
||||
"file_list": true,
|
||||
"file_retrieve": true,
|
||||
"file_delete": true,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"azure": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": true,
|
||||
"responses_with_tools": true,
|
||||
"count_tokens": false,
|
||||
"embedding": true,
|
||||
"speech": false,
|
||||
"transcription": true,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": true,
|
||||
"file_list": true,
|
||||
"file_retrieve": true,
|
||||
"file_delete": true,
|
||||
"file_content": true,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"bedrock": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": true,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": true,
|
||||
"image_variation": true,
|
||||
"image_edit": true,
|
||||
"batch_create": true,
|
||||
"batch_list": true,
|
||||
"batch_retrieve": true,
|
||||
"batch_cancel": true,
|
||||
"batch_results": true,
|
||||
"file_batch_input": true,
|
||||
"batch_create_file": true,
|
||||
"file_upload": true,
|
||||
"file_list": true,
|
||||
"file_retrieve": true,
|
||||
"file_delete": true,
|
||||
"file_content": true,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": true
|
||||
},
|
||||
"cerebras": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": true,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": false,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"cohere": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": true,
|
||||
"embedding": true,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": true
|
||||
},
|
||||
"elevenlabs": {
|
||||
"chat_completions": false,
|
||||
"chat_completions_with_tools": false,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": false,
|
||||
"speech": true,
|
||||
"transcription": true,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"gemini": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": true,
|
||||
"embedding": true,
|
||||
"speech": true,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": true,
|
||||
"image_variation": false,
|
||||
"image_edit": true,
|
||||
"batch_create": true,
|
||||
"batch_list": true,
|
||||
"batch_retrieve": true,
|
||||
"batch_cancel": true,
|
||||
"batch_results": true,
|
||||
"file_batch_input": true,
|
||||
"batch_create_file": true,
|
||||
"file_upload": true,
|
||||
"file_list": true,
|
||||
"file_retrieve": true,
|
||||
"file_delete": true,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": true
|
||||
},
|
||||
"groq": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": false,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"huggingface": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": true,
|
||||
"speech": true,
|
||||
"transcription": true,
|
||||
"list_models": true,
|
||||
"image_generation": true,
|
||||
"image_variation": false,
|
||||
"image_edit": true,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"mistral": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": true,
|
||||
"speech": false,
|
||||
"transcription": true,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"nebius": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": true,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": true,
|
||||
"embedding": true,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": true,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"openrouter": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": true,
|
||||
"responses": true,
|
||||
"responses_with_tools": true,
|
||||
"count_tokens": false,
|
||||
"embedding": false,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"parasail": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": false,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"perplexity": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": false,
|
||||
"text_completion": false,
|
||||
"responses": true,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": false,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": false,
|
||||
"image_generation": false,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
},
|
||||
"replicate": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": true,
|
||||
"responses": true,
|
||||
"responses_with_tools": true,
|
||||
"count_tokens": false,
|
||||
"embedding": false,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": true,
|
||||
"image_variation": false,
|
||||
"image_edit": true,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": true,
|
||||
"file_list": true,
|
||||
"file_retrieve": true,
|
||||
"file_delete": true,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": true,
|
||||
"video_retrieve": true,
|
||||
"video_download": true,
|
||||
"video_delete": true,
|
||||
"video_list": true,
|
||||
"video_remix": true,
|
||||
"rerank": false
|
||||
},
|
||||
"vertex": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": false,
|
||||
"responses": true,
|
||||
"responses_with_tools": true,
|
||||
"count_tokens": false,
|
||||
"embedding": true,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": true,
|
||||
"image_variation": false,
|
||||
"image_edit": true,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": true,
|
||||
"video_retrieve": true,
|
||||
"video_download": true,
|
||||
"video_delete": true,
|
||||
"video_list": true,
|
||||
"video_remix": true,
|
||||
"rerank": true
|
||||
},
|
||||
"xai": {
|
||||
"chat_completions": true,
|
||||
"chat_completions_with_tools": true,
|
||||
"text_completion": true,
|
||||
"responses": false,
|
||||
"responses_with_tools": false,
|
||||
"count_tokens": false,
|
||||
"embedding": false,
|
||||
"speech": false,
|
||||
"transcription": false,
|
||||
"list_models": true,
|
||||
"image_generation": true,
|
||||
"image_variation": false,
|
||||
"image_edit": false,
|
||||
"batch_create": false,
|
||||
"batch_list": false,
|
||||
"batch_retrieve": false,
|
||||
"batch_cancel": false,
|
||||
"batch_results": false,
|
||||
"file_batch_input": false,
|
||||
"batch_create_file": false,
|
||||
"file_upload": false,
|
||||
"file_list": false,
|
||||
"file_retrieve": false,
|
||||
"file_delete": false,
|
||||
"file_content": false,
|
||||
"container_create": false,
|
||||
"container_list": false,
|
||||
"container_retrieve": false,
|
||||
"container_delete": false,
|
||||
"container_file_create": false,
|
||||
"container_file_create_reference": false,
|
||||
"container_file_list": false,
|
||||
"container_file_retrieve": false,
|
||||
"container_file_content": false,
|
||||
"container_file_delete": false,
|
||||
"video_generation": false,
|
||||
"video_retrieve": false,
|
||||
"video_download": false,
|
||||
"video_delete": false,
|
||||
"video_list": false,
|
||||
"video_remix": false,
|
||||
"rerank": false
|
||||
}
|
||||
}
|
||||
}
|
||||
64
tests/e2e/api/provider_config/README.md
Normal file
64
tests/e2e/api/provider_config/README.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# Provider config (Postman env files)
|
||||
|
||||
Per-provider Postman environment `.json` files for running the Bifrost V1 API Newman e2e tests. Each file defines `base_url`, `provider`, `model`, and other model-type variables for that provider.
|
||||
|
||||
## Variables
|
||||
|
||||
Each `bifrost-v1-<provider>.postman_environment.json` typically includes:
|
||||
|
||||
| Key | Description |
|
||||
|-----|-------------|
|
||||
| `base_url` | Gateway base URL (default `http://localhost:8080`) |
|
||||
| `provider` | Provider name (e.g. `openai`, `anthropic`, `gemini`) |
|
||||
| `model` | Chat/completions model |
|
||||
| `embedding_model` | Embeddings model |
|
||||
| `speech_model` | TTS model |
|
||||
| `transcription_model` | Transcription model |
|
||||
| `image_model` | Image generation model |
|
||||
| `batch_id`, `file_id`, `container_id` | Placeholders; overwritten at runtime when tests create resources |
|
||||
|
||||
## Usage
|
||||
|
||||
From `tests/e2e/api`:
|
||||
|
||||
```bash
|
||||
# Run for all providers (each bifrost-v1-*.postman_environment.json in this folder, except sgl and ollama)
|
||||
./runners/run-newman-inference-tests.sh
|
||||
|
||||
# Run for a single provider
|
||||
./runners/run-newman-inference-tests.sh --env openai
|
||||
./runners/run-newman-inference-tests.sh --env provider_config/bifrost-v1-openai.postman_environment.json
|
||||
```
|
||||
|
||||
Ensure the Bifrost server is running and the chosen provider(s) are configured (API keys, etc.). Depending on provider capabilities, tests may either succeed (2xx) or return expected unsupported-operation responses.
|
||||
|
||||
## Provider-specific notes
|
||||
|
||||
- **Cohere** – Requires a valid Cohere API key in Bifrost provider config. Key format and auth may differ from other providers; 401 is expected if the key is missing or invalid.
|
||||
- **Vertex** – Requires `region` in the key config for embeddings and other operations. Set this in Bifrost provider config (project, region, credentials). Embeddings typically require a supported region such as `us-central1`.
|
||||
- **Replicate** – Set `replicate_owner` (e.g. via environment or Postman env) when running Replicate tests; otherwise API calls may fail.
|
||||
|
||||
## Files
|
||||
|
||||
All Bifrost providers are included except **sgl** and **ollama** (excluded in `runners/run-newman-inference-tests.sh` when running “all providers”).
|
||||
|
||||
- `bifrost-v1-openai.postman_environment.json`
|
||||
- `bifrost-v1-anthropic.postman_environment.json`
|
||||
- `bifrost-v1-azure.postman_environment.json`
|
||||
- `bifrost-v1-bedrock.postman_environment.json`
|
||||
- `bifrost-v1-cerebras.postman_environment.json`
|
||||
- `bifrost-v1-cohere.postman_environment.json`
|
||||
- `bifrost-v1-elevenlabs.postman_environment.json`
|
||||
- `bifrost-v1-gemini.postman_environment.json`
|
||||
- `bifrost-v1-groq.postman_environment.json`
|
||||
- `bifrost-v1-huggingface.postman_environment.json`
|
||||
- `bifrost-v1-mistral.postman_environment.json`
|
||||
- `bifrost-v1-nebius.postman_environment.json`
|
||||
- `bifrost-v1-openrouter.postman_environment.json`
|
||||
- `bifrost-v1-parasail.postman_environment.json`
|
||||
- `bifrost-v1-perplexity.postman_environment.json`
|
||||
- `bifrost-v1-replicate.postman_environment.json`
|
||||
- `bifrost-v1-vertex.postman_environment.json`
|
||||
- `bifrost-v1-xai.postman_environment.json`
|
||||
|
||||
To add a provider, copy an existing env file, rename it to `bifrost-v1-<provider>.postman_environment.json`, and set the `provider` and model values for that provider.
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-anthropic",
|
||||
"name": "Bifrost V1 – anthropic",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "anthropic",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "claude-sonnet-4-5",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "claude-3-5-sonnet-20241022",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "claude-sonnet-4-5",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "claude-sonnet-4-5",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "claude-sonnet-4-5",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "claude-sonnet-4-5",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "claude-sonnet-4-5",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "claude-3-5-sonnet-20241022",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "claude-sonnet-4-5",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-azure",
|
||||
"name": "Bifrost V1 – azure",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "azure",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "gpt-4o",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "gpt-35-turbo-instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "text-embedding-ada-002",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "gpt-4o-mini-tts",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "whisper",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "gpt-image-1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "gpt-35-turbo-instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-bedrock",
|
||||
"name": "Bifrost V1 – bedrock",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "bedrock",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "global.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "amazon.titan-text-express-v1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "global.cohere.embed-v4:0",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "unsupported_speech_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "unsupported_transcription_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "amazon.nova-canvas-v1:0",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_variation_model",
|
||||
"value": "amazon.nova-canvas-v1:0",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "bedrock_api_key",
|
||||
"value": "",
|
||||
"type": "secret",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "bedrock_access_key",
|
||||
"value": "",
|
||||
"type": "secret",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "bedrock_secret_key",
|
||||
"value": "",
|
||||
"type": "secret",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "bedrock_region",
|
||||
"value": "us-east-1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "bedrock_session_token",
|
||||
"value": "",
|
||||
"type": "secret",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "s3_bucket",
|
||||
"value": "bifrost-batch-api-file-upload-testing",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "s3_key",
|
||||
"value": "test-file.txt",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "job_arn",
|
||||
"value": "arn:aws:bedrock:us-east-1:123456789012:model-invocation-job/abc123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "role_arn",
|
||||
"value": "arn:aws:iam::123456789012:role/BedrockBatchRole",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "output_s3_uri",
|
||||
"value": "s3://bifrost-batch-api-file-upload-testing/batch-output/",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "global.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "amazon.titan-text-express-v1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-cerebras",
|
||||
"name": "Bifrost V1 – cerebras",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "cerebras",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "llama3.1-8b",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-cohere",
|
||||
"name": "Bifrost V1 – cohere",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "cohere",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "command-a-03-2025",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "command-a-03-2025",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "embed-v4.0",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "unsupported_speech_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "unsupported_transcription_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "unsupported_image_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "command-a-03-2025",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "command-a-03-2025",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "command-a-03-2025",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-elevenlabs",
|
||||
"name": "Bifrost V1 – elevenlabs",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "elevenlabs",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "eleven_multilingual_v2",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "eleven_multilingual_v2",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "unsupported_embedding_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "eleven_multilingual_v2",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "scribe_v1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "unsupported_image_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "voice",
|
||||
"value": "21m00Tcm4TlvDq8ikWAM",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "eleven_multilingual_v2",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "eleven_multilingual_v2",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "eleven_multilingual_v2",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-gemini",
|
||||
"name": "Bifrost V1 – gemini",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "gemini",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "gemini-2.0-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "unsupported_model_invoke",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "gemini-2.0-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "gemini-2.0-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "gemini-2.0-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "gemini-embedding-001",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "gemini-2.5-flash-preview-tts",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "gemini-2.5-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "gemini-2.5-flash-image",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "voice",
|
||||
"value": "achernar",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_input",
|
||||
"value": "The quick brown fox jumped over the lazy dog.",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-groq",
|
||||
"name": "Bifrost V1 – groq",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "groq",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "llama-3.3-70b-versatile",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "llama-3.1-8b-instant",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "llama-3.3-70b-versatile",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "llama-3.3-70b-versatile",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "llama-3.3-70b-versatile",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "llama-3.3-70b-versatile",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "llama-3.3-70b-versatile",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "llama-3.1-8b-instant",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "llama-3.3-70b-versatile",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-huggingface",
|
||||
"name": "Bifrost V1 – huggingface",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "huggingface",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "groq/meta-llama/Llama-3.3-70B-Instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "groq/meta-llama/Llama-3.3-70B-Instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "sambanova/intfloat/e5-mistral-7b-instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "fal-ai/hexgrad/Kokoro-82M",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "fal-ai/openai/whisper-large-v3",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "fal-ai/fal-ai/flux/dev",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "groq/meta-llama/Llama-3.3-70B-Instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "groq/meta-llama/Llama-3.3-70B-Instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "groq/meta-llama/Llama-3.3-70B-Instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-mistral",
|
||||
"name": "Bifrost V1 – mistral",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "mistral",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "mistral-medium-2508",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "mistral-medium-2508",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "codestral-embed",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "unsupported_speech_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "voxtral-mini-latest",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "unsupported_image_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "mistral-medium-2508",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "mistral-medium-2508",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "mistral-medium-2508",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-openai",
|
||||
"name": "Bifrost V1 – openai",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openai",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "gpt-4o",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "gpt-3.5-turbo-instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "text-embedding-3-small",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "tts-1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "whisper-1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "gpt-image-1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "s3_bucket",
|
||||
"value": "openai-files",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "s3_output_bucket",
|
||||
"value": "openai-output",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "role_arn",
|
||||
"value": "not-required-for-openai",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "gpt-3.5-turbo-instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "gpt-4o",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-openrouter",
|
||||
"name": "Bifrost V1 – openrouter",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "openrouter",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "google/gemini-3-flash-preview",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "openai/gpt-3.5-turbo-instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "__openrouter_unsupported_embedding__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "__openrouter_unsupported_speech__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "__openrouter_unsupported_transcription__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "__openrouter_unsupported_image__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "completions_prompt",
|
||||
"value": "what is two plus two, answer in one word",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "google/gemini-3-flash-preview",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "openai/gpt-3.5-turbo-instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "google/gemini-3-flash-preview",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-parasail",
|
||||
"name": "Bifrost V1 – parasail",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "parasail",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "parasail-llama-33-70b-fp8",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "parasail-llama-33-70b-fp8",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "unsupported",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "unsupported",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "unsupported",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "unsupported",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "parasail-llama-33-70b-fp8",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "parasail-llama-33-70b-fp8",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "parasail-llama-33-70b-fp8",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-perplexity",
|
||||
"name": "Bifrost V1 – perplexity",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "perplexity",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "sonar-pro",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "sonar-pro",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "unsupported_embedding_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "unsupported_speech_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "unsupported_transcription_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "unsupported_image_model",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "sonar-pro",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "sonar-pro",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "sonar-pro",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-replicate",
|
||||
"name": "Bifrost V1 – replicate",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "replicate",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "openai/gpt-4.1-mini",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "openai/gpt-3.5-turbo-instruct",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "__replicate_unsupported_embedding__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "__replicate_unsupported_speech__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "__replicate_unsupported_transcription__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "black-forest-labs/flux-dev",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "replicate_owner",
|
||||
"value": "",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "replicate_expiry",
|
||||
"value": "1830297599",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "openai/gpt-4.1-mini",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "openai/gpt-4.1-mini",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "openai/gpt-4.1-mini",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-vertex",
|
||||
"name": "Bifrost V1 – vertex",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "vertex",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "gemini-2.5-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "gemini-1.5-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "gemini-embedding-001",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "__vertex_unsupported_speech__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "__vertex_unsupported_transcription__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "imagen-4.0-generate-001",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_edit_model",
|
||||
"value": "imagen-3.0-capability-001",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "region",
|
||||
"value": "us-central1",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "gemini-2.5-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "gemini-1.5-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "gemini-2.5-flash",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"id": "bifrost-v1-env-xai",
|
||||
"name": "Bifrost V1 – xai",
|
||||
"values": [
|
||||
{
|
||||
"key": "base_url",
|
||||
"value": "http://localhost:8080",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "provider",
|
||||
"value": "xai",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model",
|
||||
"value": "grok-3",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "model_invoke",
|
||||
"value": "grok-3",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "embedding_model",
|
||||
"value": "__xai_unsupported_embedding__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "speech_model",
|
||||
"value": "__xai_unsupported_speech__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "transcription_model",
|
||||
"value": "__xai_unsupported_transcription__",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "image_model",
|
||||
"value": "grok-imagine-image",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "batch_id",
|
||||
"value": "batch_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "file_id",
|
||||
"value": "file_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "container_id",
|
||||
"value": "container_123",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "chat_model",
|
||||
"value": "grok-3",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "text_completion_model",
|
||||
"value": "grok-3",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"key": "responses_model",
|
||||
"value": "grok-3",
|
||||
"type": "default",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
497
tests/e2e/api/runners/individual/run-newman-anthropic-integration.sh
Executable file
497
tests/e2e/api/runners/individual/run-newman-anthropic-integration.sh
Executable file
@@ -0,0 +1,497 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost Anthropic Integration API Newman Test Runner
|
||||
# This script runs the Anthropic integration API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-anthropic-integration.postman_collection.json"
|
||||
ENVIRONMENT="bifrost-v1.postman_environment.json"
|
||||
REPORT_DIR="newman-reports/anthropic-integration"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost Anthropic Integration API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost Anthropic Integration API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if environment exists
|
||||
if [ ! -f "$ENVIRONMENT" ]; then
|
||||
echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT${NC}"
|
||||
echo "Using collection variables only"
|
||||
ENV_FLAG=""
|
||||
else
|
||||
ENV_FLAG="-e $ENVIRONMENT"
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE="--verbose" # Enable verbose by default to capture console.log statements
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="--folder \"$2\""
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="cli,html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="cli,json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local -a cmd=(newman run "$COLLECTION")
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd+=(-e "${2}")
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
if [ -n "$ENV_FLAG" ]; then
|
||||
cmd+=(-e "$ENVIRONMENT")
|
||||
fi
|
||||
cmd+=(--env-var "base_url=$base_url" --env-var "provider=$provider" --env-var "model=$model" --env-var "embedding_model=$embedding_model" --env-var "speech_model=$speech_model" --env-var "transcription_model=$transcription_model" --env-var "image_model=$image_model")
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd+=(--folder "$FOLDER")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report_${1:-run}.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report_${1:-run}.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd+=(--env-var "CI=1")
|
||||
fi
|
||||
|
||||
"${cmd[@]}"
|
||||
}
|
||||
|
||||
# Post-process log file to pretty-print JSON blocks using jq
|
||||
post_process_log() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if [ ! -f "$input_file" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
cp "$input_file" "$output_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$input_file" "$output_file" << 'PYTHON_SCRIPT'
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def format_json_with_jq(json_text):
|
||||
"""Format JSON using jq if available, otherwise use Python's json module"""
|
||||
if shutil.which('jq'):
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
['jq', '.'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
stdout, stderr = process.communicate(input=json_text)
|
||||
if process.returncode == 0:
|
||||
return stdout
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to Python's json module
|
||||
try:
|
||||
parsed = json.loads(json_text)
|
||||
return json.dumps(parsed, indent=2)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return json_text
|
||||
|
||||
def process_log_file(input_file, output_file):
|
||||
"""Process log file and format JSON blocks"""
|
||||
with open(input_file, 'r', encoding='utf-8', errors='ignore') as f_in:
|
||||
with open(output_file, 'w', encoding='utf-8') as f_out:
|
||||
in_json_block = False
|
||||
json_lines = []
|
||||
|
||||
for line in f_in:
|
||||
# Check if we're entering a JSON block
|
||||
if 'REQUEST BODY:' in line or 'RESPONSE BODY:' in line:
|
||||
if in_json_block and json_lines:
|
||||
# Format previous JSON block
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = True
|
||||
f_out.write(line)
|
||||
continue
|
||||
|
||||
# Check if we're exiting a JSON block
|
||||
if in_json_block:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('=') or line.startswith('REQUEST:') or line.startswith('RESPONSE:'):
|
||||
# End of JSON block, format and write
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = False
|
||||
else:
|
||||
json_lines.append(line)
|
||||
continue
|
||||
|
||||
f_out.write(line)
|
||||
|
||||
# Handle case where file ends in a JSON block
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Error: Expected 2 arguments, got {len(sys.argv) - 1}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
process_log_file(sys.argv[1], sys.argv[2])
|
||||
except Exception as e:
|
||||
print(f"Error processing log file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYTHON_SCRIPT
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
set +e
|
||||
TEMP_LOG="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log.tmp"
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
set +e
|
||||
TEMP_LOG="$REPORT_DIR/default.log.tmp"
|
||||
run_newman > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/default.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
temp_logfile="${logfile}.tmp"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( set +e; run_newman "$name" "$jsonfile" > "$temp_logfile" 2>&1; ec=$?; set -e; post_process_log "$temp_logfile" "$logfile"; rm -f "$temp_logfile"; exit $ec ) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
161
tests/e2e/api/runners/individual/run-newman-async-tests.sh
Executable file
161
tests/e2e/api/runners/individual/run-newman-async-tests.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Async Inference Newman Test Runner
|
||||
# Runs async submit/poll tests. Requires LogsStore and governance plugin.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-async.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/async"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Prerequisites: LogsStore and governance plugin must be configured."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Async Inference Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All async tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
511
tests/e2e/api/runners/individual/run-newman-bedrock-integration.sh
Executable file
511
tests/e2e/api/runners/individual/run-newman-bedrock-integration.sh
Executable file
@@ -0,0 +1,511 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost Bedrock Integration API Newman Test Runner
|
||||
# This script runs the Bedrock integration API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-bedrock-integration.postman_collection.json"
|
||||
ENVIRONMENT="bifrost-v1.postman_environment.json"
|
||||
REPORT_DIR="newman-reports/bedrock-integration"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost Bedrock Integration API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost Bedrock Integration API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if environment exists
|
||||
USE_DEFAULT_ENV=0
|
||||
if [ ! -f "$ENVIRONMENT" ]; then
|
||||
echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT${NC}"
|
||||
echo "Using collection variables only"
|
||||
else
|
||||
USE_DEFAULT_ENV=1
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE="--verbose" # Enable verbose by default to capture console.log statements
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="cli,html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="cli,json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_MODEL_INVOKE Override model for Bedrock invoke/invoke-stream (default: amazon.titan-text-express-v1)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo " BIFROST_BEDROCK_API_KEY Bedrock API key (when --env bedrock)"
|
||||
echo " BIFROST_BEDROCK_ACCESS_KEY Bedrock AWS access key (when --env bedrock)"
|
||||
echo " BIFROST_BEDROCK_SECRET_KEY Bedrock AWS secret key (when --env bedrock)"
|
||||
echo " BIFROST_BEDROCK_REGION Bedrock region (default: us-east-1)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local -a cmd=(newman run "$COLLECTION")
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd+=(-e "${2}")
|
||||
# Pass Bedrock credentials from env when using bedrock provider
|
||||
if [[ "${1:-}" == "bedrock" ]]; then
|
||||
[ -n "${BIFROST_BEDROCK_API_KEY:-}" ] && cmd+=(--env-var "bedrock_api_key=$BIFROST_BEDROCK_API_KEY")
|
||||
[ -n "${BIFROST_BEDROCK_ACCESS_KEY:-}" ] && cmd+=(--env-var "bedrock_access_key=$BIFROST_BEDROCK_ACCESS_KEY")
|
||||
[ -n "${BIFROST_BEDROCK_SECRET_KEY:-}" ] && cmd+=(--env-var "bedrock_secret_key=$BIFROST_BEDROCK_SECRET_KEY")
|
||||
[ -n "${BIFROST_BEDROCK_REGION:-}" ] && cmd+=(--env-var "bedrock_region=$BIFROST_BEDROCK_REGION")
|
||||
[ -n "${BIFROST_BEDROCK_SESSION_TOKEN:-}" ] && cmd+=(--env-var "bedrock_session_token=$BIFROST_BEDROCK_SESSION_TOKEN")
|
||||
fi
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local model_invoke="${BIFROST_MODEL_INVOKE:-amazon.titan-text-express-v1}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
if [ "${USE_DEFAULT_ENV:-0}" -eq 1 ]; then
|
||||
cmd+=(-e "$ENVIRONMENT")
|
||||
fi
|
||||
cmd+=(--env-var "base_url=$base_url" --env-var "provider=$provider" --env-var "model=$model" --env-var "model_invoke=$model_invoke" --env-var "embedding_model=$embedding_model" --env-var "speech_model=$speech_model" --env-var "transcription_model=$transcription_model" --env-var "image_model=$image_model")
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd+=(--folder "$FOLDER")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report_${1:-run}.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report_${1:-run}.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd+=(--env-var "CI=1")
|
||||
fi
|
||||
|
||||
"${cmd[@]}"
|
||||
}
|
||||
|
||||
# Post-process log file to pretty-print JSON blocks using jq
|
||||
post_process_log() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if [ ! -f "$input_file" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
cp "$input_file" "$output_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$input_file" "$output_file" << 'PYTHON_SCRIPT'
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def format_json_with_jq(json_text):
|
||||
"""Format JSON using jq if available, otherwise use Python's json module"""
|
||||
if shutil.which('jq'):
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
['jq', '.'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
stdout, stderr = process.communicate(input=json_text)
|
||||
if process.returncode == 0:
|
||||
return stdout
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to Python's json module
|
||||
try:
|
||||
parsed = json.loads(json_text)
|
||||
return json.dumps(parsed, indent=2)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return json_text
|
||||
|
||||
def process_log_file(input_file, output_file):
|
||||
"""Process log file and format JSON blocks"""
|
||||
with open(input_file, 'r', encoding='utf-8', errors='ignore') as f_in:
|
||||
with open(output_file, 'w', encoding='utf-8') as f_out:
|
||||
in_json_block = False
|
||||
json_lines = []
|
||||
|
||||
for line in f_in:
|
||||
# Check if we're entering a JSON block
|
||||
if 'REQUEST BODY:' in line or 'RESPONSE BODY:' in line:
|
||||
if in_json_block and json_lines:
|
||||
# Format previous JSON block
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = True
|
||||
f_out.write(line)
|
||||
continue
|
||||
|
||||
# Check if we're exiting a JSON block
|
||||
if in_json_block:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('=') or line.startswith('REQUEST:') or line.startswith('RESPONSE:'):
|
||||
# End of JSON block, format and write
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = False
|
||||
else:
|
||||
json_lines.append(line)
|
||||
continue
|
||||
|
||||
f_out.write(line)
|
||||
|
||||
# Handle case where file ends in a JSON block
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Error: Expected 2 arguments, got {len(sys.argv) - 1}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
process_log_file(sys.argv[1], sys.argv[2])
|
||||
except Exception as e:
|
||||
print(f"Error processing log file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYTHON_SCRIPT
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log.tmp"
|
||||
set +e
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/default.log.tmp"
|
||||
set +e
|
||||
run_newman > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/default.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
temp_logfile="${logfile}.tmp"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( set +e; run_newman "$name" "$jsonfile" > "$temp_logfile" 2>&1; rc=$?; set -e; post_process_log "$temp_logfile" "$logfile" || cp "$temp_logfile" "$logfile"; rm -f "$temp_logfile"; exit $rc ) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
497
tests/e2e/api/runners/individual/run-newman-composite-integration.sh
Executable file
497
tests/e2e/api/runners/individual/run-newman-composite-integration.sh
Executable file
@@ -0,0 +1,497 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost Composite Integrations API Newman Test Runner
|
||||
# This script runs the Composite Integrations API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
# Run from script directory so paths to collection and provider-config work
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-composite-integrations.postman_collection.json"
|
||||
ENVIRONMENT="bifrost-v1.postman_environment.json"
|
||||
REPORT_DIR="newman-reports/composite-integration"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost Composite Integrations API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost Composite Integrations API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if environment exists
|
||||
if [ ! -f "$ENVIRONMENT" ]; then
|
||||
echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT${NC}"
|
||||
echo "Using collection variables only"
|
||||
ENV_FLAG=""
|
||||
else
|
||||
ENV_FLAG="-e $ENVIRONMENT"
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE="--verbose" # Enable verbose by default to capture console.log statements
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="--folder \"$2\""
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="cli,html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="cli,json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local cmd="newman run $COLLECTION"
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd="$cmd -e ${2}"
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
if [ -n "$ENV_FLAG" ]; then
|
||||
cmd="$cmd $ENV_FLAG"
|
||||
fi
|
||||
cmd="$cmd --env-var \"base_url=$base_url\" --env-var \"provider=$provider\" --env-var \"model=$model\" --env-var \"embedding_model=$embedding_model\" --env-var \"speech_model=$speech_model\" --env-var \"transcription_model=$transcription_model\" --env-var \"image_model=$image_model\""
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd="$cmd $FOLDER"
|
||||
cmd="$cmd --timeout-script 120000 --timeout 900000 -r $REPORTERS"
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd="$cmd --reporter-html-export $REPORT_DIR/report_${1:-run}.html"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd="$cmd --reporter-json-export $REPORT_DIR/report_${1:-run}.json"
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd="$cmd $VERBOSE"
|
||||
[ -n "$BAIL" ] && cmd="$cmd $BAIL"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd="$cmd --env-var \"CI=1\""
|
||||
fi
|
||||
|
||||
eval $cmd
|
||||
}
|
||||
|
||||
# Post-process log file to pretty-print JSON blocks using jq
|
||||
post_process_log() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if [ ! -f "$input_file" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
cp "$input_file" "$output_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$input_file" "$output_file" << 'PYTHON_SCRIPT'
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def format_json_with_jq(json_text):
|
||||
"""Format JSON using jq if available, otherwise use Python's json module"""
|
||||
if shutil.which('jq'):
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
['jq', '.'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
stdout, stderr = process.communicate(input=json_text)
|
||||
if process.returncode == 0:
|
||||
return stdout
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to Python's json module
|
||||
try:
|
||||
parsed = json.loads(json_text)
|
||||
return json.dumps(parsed, indent=2)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return json_text
|
||||
|
||||
def process_log_file(input_file, output_file):
|
||||
"""Process log file and format JSON blocks"""
|
||||
with open(input_file, 'r', encoding='utf-8', errors='ignore') as f_in:
|
||||
with open(output_file, 'w', encoding='utf-8') as f_out:
|
||||
in_json_block = False
|
||||
json_lines = []
|
||||
|
||||
for line in f_in:
|
||||
# Check if we're entering a JSON block
|
||||
if 'REQUEST BODY:' in line or 'RESPONSE BODY:' in line:
|
||||
if in_json_block and json_lines:
|
||||
# Format previous JSON block
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = True
|
||||
f_out.write(line)
|
||||
continue
|
||||
|
||||
# Check if we're exiting a JSON block
|
||||
if in_json_block:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('=') or line.startswith('REQUEST:') or line.startswith('RESPONSE:'):
|
||||
# End of JSON block, format and write
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = False
|
||||
else:
|
||||
json_lines.append(line)
|
||||
continue
|
||||
|
||||
f_out.write(line)
|
||||
|
||||
# Handle case where file ends in a JSON block
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Error: Expected 2 arguments, got {len(sys.argv) - 1}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
process_log_file(sys.argv[1], sys.argv[2])
|
||||
except Exception as e:
|
||||
print(f"Error processing log file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYTHON_SCRIPT
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log.tmp"
|
||||
set +e
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE" || cp "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/default.log.tmp"
|
||||
set +e
|
||||
run_newman > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/default.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE" || cp "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
temp_logfile="${logfile}.tmp"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( set +e; run_newman "$name" "$jsonfile" > "$temp_logfile" 2>&1; ec=$?; set -e; post_process_log "$temp_logfile" "$logfile" || cp "$temp_logfile" "$logfile"; rm -f "$temp_logfile"; exit $ec ) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
160
tests/e2e/api/runners/individual/run-newman-fallbacks-tests.sh
Executable file
160
tests/e2e/api/runners/individual/run-newman-fallbacks-tests.sh
Executable file
@@ -0,0 +1,160 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Fallbacks Newman Test Runner
|
||||
# Runs fallback failover tests.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-fallbacks.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/fallbacks"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Fallbacks Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All fallbacks tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
161
tests/e2e/api/runners/individual/run-newman-mgmt-flows-tests.sh
Executable file
161
tests/e2e/api/runners/individual/run-newman-mgmt-flows-tests.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Management E2E Flows Newman Test Runner
|
||||
# Runs full lifecycle flows: Provider+Key+Inference, Customer+Team+VK+Inference, VK lifecycle.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-mgmt-flows.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/mgmt-flows"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Prerequisites: Governance plugin must be configured."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Management E2E Flows Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All management flow tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
514
tests/e2e/api/runners/individual/run-newman-openai-integration.sh
Executable file
514
tests/e2e/api/runners/individual/run-newman-openai-integration.sh
Executable file
@@ -0,0 +1,514 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost OpenAI Integration API Newman Test Runner
|
||||
# This script runs the OpenAI integration API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
# Run from script directory so paths to collection and provider-capabilities.json work
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-openai-integration.postman_collection.json"
|
||||
ENVIRONMENT="bifrost-v1.postman_environment.json"
|
||||
REPORT_DIR="newman-reports/openai-integration"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost OpenAI Integration API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost OpenAI Integration API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if environment exists
|
||||
if [ ! -f "$ENVIRONMENT" ]; then
|
||||
echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT${NC}"
|
||||
echo "Using collection variables only"
|
||||
ENV_FLAG=""
|
||||
else
|
||||
ENV_FLAG="-e $ENVIRONMENT"
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# Load provider capabilities from provider-capabilities.json (single source of truth) into a Newman globals file
|
||||
if [ ! -f "$PROVIDER_CAPABILITIES_JSON" ]; then
|
||||
echo -e "${RED}Error: $PROVIDER_CAPABILITIES_JSON not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v jq &>/dev/null; then
|
||||
echo -e "${RED}Error: jq is required to load $PROVIDER_CAPABILITIES_JSON${NC}"
|
||||
exit 1
|
||||
fi
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE="--verbose" # Enable verbose by default to capture console.log statements
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="cli,html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="cli,json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_RESPONSES_MODEL Override Responses API model (default: BIFROST_MODEL)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local -a cmd=(newman run "$COLLECTION" -g "$GLOBALS_TMP")
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd+=(-e "${2}")
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local responses_model="${BIFROST_RESPONSES_MODEL:-$model}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
if [ -n "$ENV_FLAG" ]; then
|
||||
cmd+=(-e "$ENVIRONMENT")
|
||||
fi
|
||||
cmd+=(--env-var "base_url=$base_url" --env-var "provider=$provider" --env-var "model=$model" --env-var "responses_model=$responses_model" --env-var "embedding_model=$embedding_model" --env-var "speech_model=$speech_model" --env-var "transcription_model=$transcription_model" --env-var "image_model=$image_model")
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd+=(--folder "$FOLDER")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report_${1:-run}.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report_${1:-run}.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd+=(--env-var "CI=1")
|
||||
fi
|
||||
|
||||
"${cmd[@]}"
|
||||
}
|
||||
|
||||
# Post-process log file to pretty-print JSON blocks using jq
|
||||
post_process_log() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if [ ! -f "$input_file" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
cp "$input_file" "$output_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$input_file" "$output_file" << 'PYTHON_SCRIPT'
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def format_json_with_jq(json_text):
|
||||
"""Format JSON using jq if available, otherwise use Python's json module"""
|
||||
if shutil.which('jq'):
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
['jq', '.'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
stdout, stderr = process.communicate(input=json_text)
|
||||
if process.returncode == 0:
|
||||
return stdout
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to Python's json module
|
||||
try:
|
||||
parsed = json.loads(json_text)
|
||||
return json.dumps(parsed, indent=2)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return json_text
|
||||
|
||||
def process_log_file(input_file, output_file):
|
||||
"""Process log file and format JSON blocks"""
|
||||
with open(input_file, 'r', encoding='utf-8', errors='ignore') as f_in:
|
||||
with open(output_file, 'w', encoding='utf-8') as f_out:
|
||||
in_json_block = False
|
||||
json_lines = []
|
||||
|
||||
for line in f_in:
|
||||
# Check if we're entering a JSON block
|
||||
if 'REQUEST BODY:' in line or 'RESPONSE BODY:' in line:
|
||||
if in_json_block and json_lines:
|
||||
# Format previous JSON block
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = True
|
||||
f_out.write(line)
|
||||
continue
|
||||
|
||||
# Check if we're exiting a JSON block
|
||||
if in_json_block:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('=') or line.startswith('REQUEST:') or line.startswith('RESPONSE:'):
|
||||
# End of JSON block, format and write
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = False
|
||||
else:
|
||||
json_lines.append(line)
|
||||
continue
|
||||
|
||||
f_out.write(line)
|
||||
|
||||
# Handle case where file ends in a JSON block
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Error: Expected 2 arguments, got {len(sys.argv) - 1}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
process_log_file(sys.argv[1], sys.argv[2])
|
||||
except Exception as e:
|
||||
print(f"Error processing log file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYTHON_SCRIPT
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log.tmp"
|
||||
set +e
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE" || cp "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/default.log.tmp"
|
||||
set +e
|
||||
run_newman > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/default.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE" || cp "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
temp_logfile="${logfile}.tmp"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( set +e; run_newman "$name" "$jsonfile" > "$temp_logfile" 2>&1; ec=$?; set -e; post_process_log "$temp_logfile" "$logfile" || cp "$temp_logfile" "$logfile"; rm -f "$temp_logfile"; exit $ec ) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
161
tests/e2e/api/runners/individual/run-newman-rate-limit-tests.sh
Executable file
161
tests/e2e/api/runners/individual/run-newman-rate-limit-tests.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Rate Limit Newman Test Runner
|
||||
# Runs rate limit enforcement tests. Requires governance plugin.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-rate-limit.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/rate-limit"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Prerequisites: Governance plugin must be configured."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Rate Limit Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All rate limit tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
166
tests/e2e/api/runners/individual/run-newman-session-tests.sh
Executable file
166
tests/e2e/api/runners/individual/run-newman-session-tests.sh
Executable file
@@ -0,0 +1,166 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Session Stickiness Newman Test Runner
|
||||
# Runs session stickiness tests (x-bf-session-id, x-bf-session-ttl).
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-session.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/session"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Session Stickiness Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
BAIL=""
|
||||
HTML_REPORT=""
|
||||
JSON_REPORT=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) HTML_REPORT="yes"; shift ;;
|
||||
--json) JSON_REPORT="yes"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build reporters string
|
||||
REPORTERS="cli"
|
||||
[ -n "$HTML_REPORT" ] && REPORTERS="$REPORTERS,html"
|
||||
[ -n "$JSON_REPORT" ] && REPORTERS="$REPORTERS,json"
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All session tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
160
tests/e2e/api/runners/individual/run-newman-streaming-tests.sh
Executable file
160
tests/e2e/api/runners/individual/run-newman-streaming-tests.sh
Executable file
@@ -0,0 +1,160 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Streaming Newman Test Runner
|
||||
# Runs streaming SSE tests for chat completions and responses.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-streaming.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/streaming"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Streaming Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All streaming tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
211
tests/e2e/api/runners/individual/run-newman-vk-auth-tests.sh
Executable file
211
tests/e2e/api/runners/individual/run-newman-vk-auth-tests.sh
Executable file
@@ -0,0 +1,211 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Virtual Key Auth Newman Test Runner
|
||||
# Runs VK auth tests: creates VK, runs inference with/without VK, tests rejection cases, cleans up.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-v1-vk-auth.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/vk-auth"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
PROVIDER_ENV_FILE=""
|
||||
ENFORCE_AUTH=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--enforce-auth)
|
||||
ENFORCE_AUTH="1"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name (e.g. openai or provider_config/bifrost-v1-openai.postman_environment.json)"
|
||||
echo " --enforce-auth Enable auth enforcement mode (without-VK requests expect 401)"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 --env openai # Run with OpenAI provider"
|
||||
echo " $0 --env openai --enforce-auth # Run with auth enforcement"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Virtual Key Auth Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# Load provider capabilities into globals (for consistency with v1 runner)
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
# Parse remaining options
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="${REPORTERS},html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="${REPORTERS},json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Resolve provider env file
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Default to openai if no env specified
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Build Newman command
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
|
||||
# Pass enforce_auth when --enforce-auth was set
|
||||
if [ -n "$ENFORCE_AUTH" ]; then
|
||||
cmd+=(--env-var "enforce_auth=1")
|
||||
echo -e "Mode: ${YELLOW}enforce_auth=1${NC} (without-VK requests expect 401)"
|
||||
fi
|
||||
|
||||
# Base URL override
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All VK auth tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
161
tests/e2e/api/runners/individual/run-newman-vk-routing-tests.sh
Executable file
161
tests/e2e/api/runners/individual/run-newman-vk-routing-tests.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 VK Routing Newman Test Runner
|
||||
# Runs governance routing tests. Requires governance plugin.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-vk-routing.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/vk-routing"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Prerequisites: Governance plugin must be configured."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 VK Routing Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All VK routing tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
133
tests/e2e/api/runners/run-all-integration-tests.sh
Executable file
133
tests/e2e/api/runners/run-all-integration-tests.sh
Executable file
@@ -0,0 +1,133 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost All Integration Tests Runner
|
||||
# This script runs all integration test suites sequentially and aggregates results
|
||||
|
||||
set -e
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Print banner
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Bifrost All Integration Tests Runner${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
|
||||
# Parse command line arguments
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML reports"
|
||||
echo " --json Generate JSON reports"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --env <provider> Run tests with specific provider only"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "This script runs all integration test collections:"
|
||||
echo " 1. OpenAI Integration"
|
||||
echo " 2. Anthropic Integration"
|
||||
echo " 3. Bedrock Integration"
|
||||
echo " 4. Composite Integrations (GenAI, Cohere, LiteLLM, LangChain, PydanticAI)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run all tests for all providers"
|
||||
echo " $0 --env openai # Run all tests with OpenAI provider only"
|
||||
echo " $0 --html --verbose # Verbose with HTML reports"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Test scripts
|
||||
TEST_SCRIPTS=(
|
||||
"run-newman-openai-integration.sh"
|
||||
"run-newman-anthropic-integration.sh"
|
||||
"run-newman-bedrock-integration.sh"
|
||||
"run-newman-composite-integration.sh"
|
||||
)
|
||||
|
||||
# Test names for display
|
||||
TEST_NAMES=(
|
||||
"OpenAI Integration"
|
||||
"Anthropic Integration"
|
||||
"Bedrock Integration"
|
||||
"Composite Integrations"
|
||||
)
|
||||
|
||||
# Track results
|
||||
FAILED_TESTS=()
|
||||
PASSED_COUNT=0
|
||||
FAILED_COUNT=0
|
||||
|
||||
echo -e "${GREEN}Running ${#TEST_SCRIPTS[@]} integration test suites...${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each test suite
|
||||
for i in "${!TEST_SCRIPTS[@]}"; do
|
||||
script="${TEST_SCRIPTS[$i]}"
|
||||
name="${TEST_NAMES[$i]}"
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}[$((i+1))/${#TEST_SCRIPTS[@]}] Running ${name}${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
script_path="$SCRIPT_DIR/individual/$script"
|
||||
if [ -f "$script_path" ]; then
|
||||
if (cd "$SCRIPT_DIR/individual" && "./$script" "${ARGS[@]}"); then
|
||||
echo ""
|
||||
echo -e "${GREEN}✓ ${name} PASSED${NC}"
|
||||
PASSED_COUNT=$((PASSED_COUNT + 1))
|
||||
else
|
||||
echo ""
|
||||
echo -e "${RED}✗ ${name} FAILED${NC}"
|
||||
FAILED_TESTS+=("$name")
|
||||
FAILED_COUNT=$((FAILED_COUNT + 1))
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}Error: Test script not found: $script_path${NC}"
|
||||
FAILED_TESTS+=("$name (script not found)")
|
||||
FAILED_COUNT=$((FAILED_COUNT + 1))
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Print summary
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Test Summary${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo -e "Total test suites: ${#TEST_SCRIPTS[@]}"
|
||||
echo -e "${GREEN}Passed: ${PASSED_COUNT}${NC}"
|
||||
echo -e "${RED}Failed: ${FAILED_COUNT}${NC}"
|
||||
echo ""
|
||||
|
||||
if [ ${FAILED_COUNT} -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All integration test suites passed!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}✗ The following test suites failed:${NC}"
|
||||
for test in "${FAILED_TESTS[@]}"; do
|
||||
echo -e " ${RED}- ${test}${NC}"
|
||||
done
|
||||
echo ""
|
||||
echo -e "${YELLOW}Check individual test reports in newman-reports/ directories${NC}"
|
||||
exit 1
|
||||
fi
|
||||
293
tests/e2e/api/runners/run-newman-api-tests.sh
Executable file
293
tests/e2e/api/runners/run-newman-api-tests.sh
Executable file
@@ -0,0 +1,293 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost API Management & Health Tests
|
||||
# This script runs tests for /api/* and /health endpoints
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="$API_DIR/collections/bifrost-api-management.postman_collection.json"
|
||||
REPORT_DIR="$API_DIR/newman-reports/api-management"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Bifrost API Management & Health Tests${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create report directory and log directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
LOG_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Parse command line arguments
|
||||
VERBOSE="--verbose"
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
DB_VERIFY=""
|
||||
DB_URL="${BIFROST_DB_URL:-}"
|
||||
LOGS_DB_URL="${BIFROST_LOGS_DB_URL:-}"
|
||||
DB_CONFIG_PATH=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--no-verbose)
|
||||
VERBOSE=""
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="${REPORTERS},html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="${REPORTERS},json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--db-verify)
|
||||
DB_VERIFY="1"
|
||||
shift
|
||||
;;
|
||||
--db-url)
|
||||
DB_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--logs-db-url)
|
||||
LOGS_DB_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--config-path)
|
||||
DB_CONFIG_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --verbose Show detailed output (enabled by default)"
|
||||
echo " --no-verbose Disable verbose output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --db-verify Enable DB verification reporter (PostgreSQL or SQLite)"
|
||||
echo " --db-url <dsn> Explicit main DB connection string (overrides auto-detection)"
|
||||
echo " --logs-db-url <dsn> Explicit logs DB url (also reads BIFROST_LOGS_DB_URL; auto-detected)"
|
||||
echo " PostgreSQL: postgresql://user:pass@host:port/db"
|
||||
echo " SQLite: sqlite:///path/to/file.db"
|
||||
echo " --config-path <p> Path to Bifrost config.json for auto DB detection"
|
||||
echo " (default: ./config.json; also reads BIFROST_CONFIG_PATH env)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run API management tests"
|
||||
echo " $0 --html # Run with HTML report"
|
||||
echo " $0 --verbose # Run with verbose output"
|
||||
echo " $0 --db-verify # Run with DB verification"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo -e " Verbose: ${YELLOW}$([ -n "$VERBOSE" ] && echo "enabled" || echo "disabled")${NC}"
|
||||
if [ -n "$DB_VERIFY" ]; then
|
||||
if [ -n "$DB_URL" ]; then
|
||||
echo -e " DB Verify: ${YELLOW}enabled (url: $DB_URL)${NC}"
|
||||
elif [ -n "$DB_CONFIG_PATH" ]; then
|
||||
echo -e " DB Verify: ${YELLOW}enabled (config: $DB_CONFIG_PATH)${NC}"
|
||||
else
|
||||
echo -e " DB Verify: ${YELLOW}enabled (auto-detect from ./config.json)${NC}"
|
||||
fi
|
||||
else
|
||||
echo -e " DB Verify: ${YELLOW}disabled${NC}"
|
||||
fi
|
||||
# Repo root (tests/e2e/api -> ../../..)
|
||||
BIFROST_ROOT="$(cd "$API_DIR/../../.." && pwd)"
|
||||
PLUGIN_DIR="$BIFROST_ROOT/examples/plugins/hello-world"
|
||||
PLUGIN_SO="$PLUGIN_DIR/build/hello-world.so"
|
||||
|
||||
# Build hello-world plugin and resolve absolute path for plugin_path (before any test infra)
|
||||
if [ -d "$PLUGIN_DIR" ] && [ -f "$PLUGIN_DIR/Makefile" ]; then
|
||||
echo "Building hello-world plugin..."
|
||||
(cd "$PLUGIN_DIR" && make build) 2>/dev/null || (cd "$PLUGIN_DIR" && make dev) 2>/dev/null || true
|
||||
if [ -f "$PLUGIN_SO" ]; then
|
||||
PLUGIN_PATH_ABS="$(cd "$(dirname "$PLUGIN_SO")" && pwd)/$(basename "$PLUGIN_SO")"
|
||||
echo " Plugin: $PLUGIN_PATH_ABS"
|
||||
else
|
||||
PLUGIN_PATH_ABS=""
|
||||
fi
|
||||
else
|
||||
PLUGIN_PATH_ABS=""
|
||||
fi
|
||||
|
||||
# ── http-no-ping-server (MCP HTTP server on :3001) ───────────────────────────
|
||||
HTTP_SERVER_DIR="$BIFROST_ROOT/examples/mcps/http-no-ping-server"
|
||||
HTTP_SERVER_BIN="$HTTP_SERVER_DIR/http-server"
|
||||
HTTP_SERVER_PID=""
|
||||
|
||||
start_http_mcp_server() {
|
||||
# Skip if something is already listening on 3001
|
||||
if lsof -ti tcp:3001 &>/dev/null 2>&1; then
|
||||
echo " http-no-ping-server: port 3001 already in use, skipping start"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ ! -d "$HTTP_SERVER_DIR" ]; then
|
||||
echo " http-no-ping-server: directory not found ($HTTP_SERVER_DIR), skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Build binary if missing
|
||||
if [ ! -f "$HTTP_SERVER_BIN" ]; then
|
||||
echo " Building http-no-ping-server..."
|
||||
(cd "$HTTP_SERVER_DIR" && CGO_ENABLED=0 go build -o http-server main.go) || {
|
||||
echo " http-no-ping-server: build failed, skipping"
|
||||
return 0
|
||||
}
|
||||
fi
|
||||
|
||||
echo " Starting http-no-ping-server on port 3001..."
|
||||
"$HTTP_SERVER_BIN" &
|
||||
HTTP_SERVER_PID=$!
|
||||
|
||||
# Wait up to 10 s for it to accept connections
|
||||
for i in $(seq 1 10); do
|
||||
sleep 1
|
||||
if lsof -ti tcp:3001 &>/dev/null 2>&1; then
|
||||
echo " http-no-ping-server ready (PID $HTTP_SERVER_PID)"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo " WARNING: http-no-ping-server did not become ready in time"
|
||||
}
|
||||
|
||||
stop_http_mcp_server() {
|
||||
if [ -n "$HTTP_SERVER_PID" ] && kill -0 "$HTTP_SERVER_PID" 2>/dev/null; then
|
||||
echo "Stopping http-no-ping-server (PID $HTTP_SERVER_PID)..."
|
||||
kill "$HTTP_SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Register teardown so the server is stopped even if the script exits early
|
||||
trap stop_http_mcp_server EXIT
|
||||
|
||||
echo "Setting up MCP test servers..."
|
||||
start_http_mcp_server
|
||||
echo ""
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
# Add dbverify reporter if requested
|
||||
if [ -n "$DB_VERIFY" ]; then
|
||||
REPORTERS="$REPORTERS,dbverify"
|
||||
# Install dependencies for the dbverify reporter if not already present
|
||||
if [ ! -d "$API_DIR/node_modules" ]; then
|
||||
echo "Installing DB verify reporter dependencies..."
|
||||
(cd "$API_DIR" && npm install --silent)
|
||||
fi
|
||||
# Newman (global) resolves reporters via Node's module search. Prepend the
|
||||
# local node_modules so it can find newman-reporter-dbverify without a
|
||||
# global install.
|
||||
export NODE_PATH="$API_DIR/node_modules${NODE_PATH:+:$NODE_PATH}"
|
||||
fi
|
||||
|
||||
# Build Newman command
|
||||
cmd=(newman run "$COLLECTION" --timeout-script 120000 --timeout 900000 -r "$REPORTERS")
|
||||
|
||||
# Override plugin_path with resolved absolute path so Create Plugin / Get Plugin use the built .so
|
||||
# env-var takes precedence over collection variables in Newman's resolution order
|
||||
if [ -n "$PLUGIN_PATH_ABS" ]; then
|
||||
cmd+=(--env-var "plugin_path=$PLUGIN_PATH_ABS")
|
||||
fi
|
||||
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
fi
|
||||
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
fi
|
||||
|
||||
if [ -n "$DB_VERIFY" ]; then
|
||||
[ -n "$DB_URL" ] && cmd+=(--reporter-dbverify-db-url "$DB_URL")
|
||||
[ -n "$LOGS_DB_URL" ] && cmd+=(--reporter-dbverify-logs-db-url "$LOGS_DB_URL")
|
||||
[ -n "$DB_CONFIG_PATH" ] && cmd+=(--reporter-dbverify-config "$DB_CONFIG_PATH")
|
||||
fi
|
||||
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
# Run Newman and save output to log file while displaying to console (using tee)
|
||||
LOG_FILE="$LOG_DIR/api-management.log"
|
||||
|
||||
# Write resolved plugin path to log before running tests
|
||||
if [ -n "$PLUGIN_PATH_ABS" ]; then
|
||||
echo "[setup] plugin_path resolved to: $PLUGIN_PATH_ABS" | tee "$LOG_FILE"
|
||||
else
|
||||
echo "[setup] plugin_path not resolved (build may have failed)" | tee "$LOG_FILE"
|
||||
fi
|
||||
|
||||
set +e
|
||||
"${cmd[@]}" 2>&1 | tee -a "$LOG_FILE"
|
||||
EXIT_CODE=${PIPESTATUS[0]}
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
echo -e "Log saved to: ${YELLOW}$LOG_FILE${NC}"
|
||||
|
||||
exit $EXIT_CODE
|
||||
162
tests/e2e/api/runners/run-newman-inference-features-tests.sh
Executable file
162
tests/e2e/api/runners/run-newman-inference-features-tests.sh
Executable file
@@ -0,0 +1,162 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Inference with Bifrost Features Newman Test Runner
|
||||
# Runs combined: Async Inference, Fallbacks, Management Flows, Rate Limit, Session Stickiness, VK Routing.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-inference-features.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/inference-features"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Suites: Async Inference, Fallbacks, Management Flows, Rate Limit, Session Stickiness, VK Routing"
|
||||
echo "Prerequisites: governance plugin must be configured for management/rate-limit/routing suites."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Inference with Bifrost Features Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All inference features tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
406
tests/e2e/api/runners/run-newman-inference-tests.sh
Executable file
406
tests/e2e/api/runners/run-newman-inference-tests.sh
Executable file
@@ -0,0 +1,406 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 API Newman Test Runner
|
||||
# This script runs the complete Bifrost V1 API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
# Run from script directory so paths to collection and provider-capabilities.json work
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-v1-complete.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/v1"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost V1 API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost V1 API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# Load provider capabilities from provider-capabilities.json (single source of truth) into a Newman globals file
|
||||
if [ ! -f "$PROVIDER_CAPABILITIES_JSON" ]; then
|
||||
echo -e "${RED}Error: $PROVIDER_CAPABILITIES_JSON not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v jq &>/dev/null; then
|
||||
echo -e "${RED}Error: jq is required to load $PROVIDER_CAPABILITIES_JSON${NC}"
|
||||
exit 1
|
||||
fi
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
REPORTERS="cli,html,json"
|
||||
else
|
||||
REPORTERS="cli,html"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
REPORTERS="cli,html,json"
|
||||
else
|
||||
REPORTERS="cli,json"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " CI=1 When set, each failing request is retried up to 3 times"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_CHAT_MODEL Override chat completions model (default: BIFROST_MODEL)"
|
||||
echo " BIFROST_TEXT_COMPLETION_MODEL Override text completions model (default: BIFROST_MODEL)"
|
||||
echo " BIFROST_RESPONSES_MODEL Override Responses API model (default: BIFROST_MODEL)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo " AWS_S3_BUCKET For Bedrock: S3 bucket for file/batch (same as core tests)"
|
||||
echo " AWS_BEDROCK_ROLE_ARN For Bedrock: IAM role ARN for batch (same as core tests)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local -a cmd=(newman run "$COLLECTION" -g "$GLOBALS_TMP")
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd+=(-e "${2}")
|
||||
# Align with core Bedrock tests: pass AWS_S3_BUCKET / AWS_BEDROCK_ROLE_ARN when running with Bedrock env
|
||||
if [[ "${1:-}" == "bedrock" ]]; then
|
||||
[ -n "${AWS_S3_BUCKET:-}" ] && cmd+=(--env-var "s3_bucket=$AWS_S3_BUCKET" --env-var "s3_output_bucket=$AWS_S3_BUCKET" --env-var "output_s3_uri=s3://$AWS_S3_BUCKET/batch-output/")
|
||||
[ -n "${AWS_BEDROCK_ROLE_ARN:-}" ] && cmd+=(--env-var "role_arn=$AWS_BEDROCK_ROLE_ARN")
|
||||
fi
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local chat_model="${BIFROST_CHAT_MODEL:-$model}"
|
||||
local text_completion_model="${BIFROST_TEXT_COMPLETION_MODEL:-$model}"
|
||||
local responses_model="${BIFROST_RESPONSES_MODEL:-$model}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
cmd+=(--env-var "base_url=$base_url" --env-var "provider=$provider" --env-var "model=$model" --env-var "chat_model=$chat_model" --env-var "text_completion_model=$text_completion_model" --env-var "responses_model=$responses_model" --env-var "embedding_model=$embedding_model" --env-var "speech_model=$speech_model" --env-var "transcription_model=$transcription_model" --env-var "image_model=$image_model")
|
||||
fi
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd+=(--env-var "CI=1")
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd+=(--folder "$FOLDER")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report_${1:-run}.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report_${1:-run}.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
"${cmd[@]}"
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" && EXIT_CODE=0 || EXIT_CODE=$?
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
set +e
|
||||
run_newman
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( run_newman "$name" "$jsonfile" ) > "$logfile" 2>&1 &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
78
tests/e2e/api/setup-mcp.sh
Executable file
78
tests/e2e/api/setup-mcp.sh
Executable file
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# Global setup for MCP client E2E tests: start the http-no-ping-server on port 3001.
|
||||
# The API Management collection adds a test MCP client with connection_string http://localhost:3001/
|
||||
# so this server must be running for Add/Update/Delete MCP Client tests to pass.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
MCP_SERVER_DIR="$REPO_ROOT/examples/mcps/http-no-ping-server"
|
||||
MCP_PORT=3001
|
||||
PID_FILE="$SCRIPT_DIR/.mcp-server.pid"
|
||||
|
||||
# Check if MCP server is already listening on 3001 (e.g. from a previous run)
|
||||
if command -v nc &>/dev/null; then
|
||||
if nc -z 127.0.0.1 "$MCP_PORT" 2>/dev/null; then
|
||||
if pgrep -f "http-no-ping-server" >/dev/null 2>&1; then
|
||||
echo "MCP server already running on port $MCP_PORT (test MCP client will use http://localhost:$MCP_PORT/)."
|
||||
exit 0
|
||||
fi
|
||||
echo "Port $MCP_PORT is occupied by a non-MCP process. Aborting setup."
|
||||
exit 1
|
||||
fi
|
||||
elif command -v bash &>/dev/null && (echo >/dev/tcp/127.0.0.1/"$MCP_PORT") 2>/dev/null; then
|
||||
if pgrep -f "http-no-ping-server" >/dev/null 2>&1; then
|
||||
echo "MCP server already running on port $MCP_PORT (test MCP client will use http://localhost:$MCP_PORT/)."
|
||||
exit 0
|
||||
fi
|
||||
echo "Port $MCP_PORT is occupied by a non-MCP process. Aborting setup."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$MCP_SERVER_DIR" ]; then
|
||||
echo "MCP server source not found: $MCP_SERVER_DIR"
|
||||
echo "MCP client tests will use fallback (accept 404/500)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Build the server
|
||||
echo "Building MCP test server (http-no-ping-server)..."
|
||||
cd "$MCP_SERVER_DIR" || exit 0
|
||||
if ! go build -o http-no-ping-server . 2>/dev/null; then
|
||||
echo "WARNING: MCP server build failed. MCP client tests will use fallback (accept 404/500)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Start in background
|
||||
if [ ! -f "./http-no-ping-server" ]; then
|
||||
echo "WARNING: MCP server binary not found. MCP client tests will use fallback."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Clean up stale PID file safely (only kill if process is our MCP server)
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
old_pid="$(cat "$PID_FILE" 2>/dev/null || true)"
|
||||
if [[ -n "$old_pid" && "$old_pid" =~ ^[0-9]+$ ]] && ps -p "$old_pid" -o args= 2>/dev/null | grep -q "http-no-ping-server"; then
|
||||
kill "$old_pid" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
rm -f "$PID_FILE"
|
||||
|
||||
echo "Starting MCP server on http://localhost:$MCP_PORT/ ..."
|
||||
./http-no-ping-server &
|
||||
echo $! > "$PID_FILE"
|
||||
|
||||
# Wait for port to be open (max 10s)
|
||||
for i in $(seq 1 20); do
|
||||
if (command -v nc &>/dev/null && nc -z 127.0.0.1 "$MCP_PORT" 2>/dev/null) \
|
||||
|| (command -v bash &>/dev/null && (echo >/dev/tcp/127.0.0.1/"$MCP_PORT") 2>/dev/null); then
|
||||
echo "MCP server ready at http://localhost:$MCP_PORT/ (test MCP client will use this URL)."
|
||||
exit 0
|
||||
fi
|
||||
[ $i -eq 20 ] && break
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo "WARNING: MCP server may not have started in time. MCP client tests may fail or use fallback."
|
||||
exit 0
|
||||
37
tests/e2e/api/setup-plugin.sh
Executable file
37
tests/e2e/api/setup-plugin.sh
Executable file
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
# Build hello-world plugin for E2E tests
|
||||
# Run from tests/e2e/api/ (or any dir; script finds repo root)
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# Repo root is three levels up from tests/e2e/api
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
PLUGIN_DIR="$REPO_ROOT/examples/plugins/hello-world"
|
||||
BUILD_DIR="$PLUGIN_DIR/build"
|
||||
|
||||
echo "Building hello-world plugin..."
|
||||
|
||||
# Check if plugin source exists
|
||||
if [ ! -d "$PLUGIN_DIR" ]; then
|
||||
echo "ERROR: Plugin source directory not found: $PLUGIN_DIR"
|
||||
echo "Plugin tests will be skipped."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Create build directory
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
# Build the plugin (native for current OS/arch)
|
||||
cd "$PLUGIN_DIR" || exit 1
|
||||
if command -v make &>/dev/null; then
|
||||
make build-test-plugin 2>/dev/null || make dev 2>/dev/null || true
|
||||
else
|
||||
CGO_ENABLED=1 go build -buildmode=plugin -o "build/hello-world.so" . 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [ -f "build/hello-world.so" ]; then
|
||||
echo "Plugin built successfully: $PLUGIN_DIR/build/hello-world.so"
|
||||
else
|
||||
echo "WARNING: Plugin build failed or skipped (e.g. cross-compilation). Plugin tests may fail."
|
||||
fi
|
||||
228
tests/e2e/core/actions/api.ts
Normal file
228
tests/e2e/core/actions/api.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import { APIRequestContext, APIResponse } from '@playwright/test'
|
||||
|
||||
/**
|
||||
* API helper functions for test setup and cleanup
|
||||
*/
|
||||
|
||||
const API_BASE = '/api'
|
||||
|
||||
/**
|
||||
* Handle API response with error checking
|
||||
*/
|
||||
async function handleResponse<T>(response: APIResponse, operation: string): Promise<T> {
|
||||
if (!response.ok()) {
|
||||
throw new Error(`${operation} failed: ${response.status()} ${response.statusText()}`)
|
||||
}
|
||||
return response.json() as Promise<T>
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider API helpers
|
||||
*/
|
||||
export const providersApi = {
|
||||
/**
|
||||
* Get all providers
|
||||
*/
|
||||
async getAll(request: APIRequestContext) {
|
||||
const response = await request.get(`${API_BASE}/providers`)
|
||||
return handleResponse(response, 'Get all providers')
|
||||
},
|
||||
|
||||
/**
|
||||
* Get a specific provider
|
||||
*/
|
||||
async get(request: APIRequestContext, name: string) {
|
||||
const response = await request.get(`${API_BASE}/providers/${name}`)
|
||||
return handleResponse(response, `Get provider ${name}`)
|
||||
},
|
||||
|
||||
/**
|
||||
* Create a provider
|
||||
*/
|
||||
async create(request: APIRequestContext, data: unknown) {
|
||||
const response = await request.post(`${API_BASE}/providers`, {
|
||||
data,
|
||||
})
|
||||
return handleResponse(response, 'Create provider')
|
||||
},
|
||||
|
||||
/**
|
||||
* Update a provider
|
||||
*/
|
||||
async update(request: APIRequestContext, name: string, data: unknown) {
|
||||
const response = await request.put(`${API_BASE}/providers/${name}`, {
|
||||
data,
|
||||
})
|
||||
return handleResponse(response, `Update provider ${name}`)
|
||||
},
|
||||
|
||||
/**
|
||||
* Delete a provider
|
||||
*/
|
||||
async delete(request: APIRequestContext, name: string) {
|
||||
const response = await request.delete(`${API_BASE}/providers/${name}`)
|
||||
return response.ok()
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Virtual Keys API helpers
|
||||
*/
|
||||
export const virtualKeysApi = {
|
||||
/**
|
||||
* Get all virtual keys
|
||||
*/
|
||||
async getAll(request: APIRequestContext) {
|
||||
const response = await request.get(`${API_BASE}/governance/virtual-keys`)
|
||||
return handleResponse(response, 'Get all virtual keys')
|
||||
},
|
||||
|
||||
/**
|
||||
* Get a specific virtual key
|
||||
*/
|
||||
async get(request: APIRequestContext, id: string) {
|
||||
const response = await request.get(`${API_BASE}/governance/virtual-keys/${id}`)
|
||||
return handleResponse(response, `Get virtual key ${id}`)
|
||||
},
|
||||
|
||||
/**
|
||||
* Create a virtual key
|
||||
*/
|
||||
async create(request: APIRequestContext, data: unknown) {
|
||||
const response = await request.post(`${API_BASE}/governance/virtual-keys`, {
|
||||
data,
|
||||
})
|
||||
return handleResponse(response, 'Create virtual key')
|
||||
},
|
||||
|
||||
/**
|
||||
* Update a virtual key
|
||||
*/
|
||||
async update(request: APIRequestContext, id: string, data: unknown) {
|
||||
const response = await request.put(`${API_BASE}/governance/virtual-keys/${id}`, {
|
||||
data,
|
||||
})
|
||||
return handleResponse(response, `Update virtual key ${id}`)
|
||||
},
|
||||
|
||||
/**
|
||||
* Delete a virtual key
|
||||
*/
|
||||
async delete(request: APIRequestContext, id: string) {
|
||||
const response = await request.delete(`${API_BASE}/governance/virtual-keys/${id}`)
|
||||
return response.ok()
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Teams API helpers
|
||||
*/
|
||||
export const teamsApi = {
|
||||
/**
|
||||
* Get all teams
|
||||
*/
|
||||
async getAll(request: APIRequestContext) {
|
||||
const response = await request.get(`${API_BASE}/governance/teams`)
|
||||
return handleResponse(response, 'Get all teams')
|
||||
},
|
||||
|
||||
/**
|
||||
* Create a team
|
||||
*/
|
||||
async create(request: APIRequestContext, data: unknown) {
|
||||
const response = await request.post(`${API_BASE}/governance/teams`, {
|
||||
data,
|
||||
})
|
||||
return handleResponse(response, 'Create team')
|
||||
},
|
||||
|
||||
/**
|
||||
* Delete a team
|
||||
*/
|
||||
async delete(request: APIRequestContext, id: string) {
|
||||
const response = await request.delete(`${API_BASE}/governance/teams/${id}`)
|
||||
return response.ok()
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Customers API helpers
|
||||
*/
|
||||
export const customersApi = {
|
||||
/**
|
||||
* Get all customers
|
||||
*/
|
||||
async getAll(request: APIRequestContext) {
|
||||
const response = await request.get(`${API_BASE}/governance/customers`)
|
||||
return handleResponse(response, 'Get all customers')
|
||||
},
|
||||
|
||||
/**
|
||||
* Create a customer
|
||||
*/
|
||||
async create(request: APIRequestContext, data: unknown) {
|
||||
const response = await request.post(`${API_BASE}/governance/customers`, {
|
||||
data,
|
||||
})
|
||||
return handleResponse(response, 'Create customer')
|
||||
},
|
||||
|
||||
/**
|
||||
* Delete a customer
|
||||
*/
|
||||
async delete(request: APIRequestContext, id: string) {
|
||||
const response = await request.delete(`${API_BASE}/governance/customers/${id}`)
|
||||
return response.ok()
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup helper - delete all test data
|
||||
*/
|
||||
export async function cleanupTestData(
|
||||
request: APIRequestContext,
|
||||
options: {
|
||||
virtualKeyIds?: string[]
|
||||
teamIds?: string[]
|
||||
customerIds?: string[]
|
||||
providerNames?: string[]
|
||||
}
|
||||
): Promise<void> {
|
||||
const { virtualKeyIds = [], teamIds = [], customerIds = [], providerNames = [] } = options
|
||||
|
||||
// Delete virtual keys first (they may depend on teams/customers)
|
||||
for (const id of virtualKeyIds) {
|
||||
try {
|
||||
await virtualKeysApi.delete(request, id)
|
||||
} catch (e) {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
|
||||
// Delete teams
|
||||
for (const id of teamIds) {
|
||||
try {
|
||||
await teamsApi.delete(request, id)
|
||||
} catch (e) {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
|
||||
// Delete customers
|
||||
for (const id of customerIds) {
|
||||
try {
|
||||
await customersApi.delete(request, id)
|
||||
} catch (e) {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
|
||||
// Delete custom providers
|
||||
for (const name of providerNames) {
|
||||
try {
|
||||
await providersApi.delete(request, name)
|
||||
} catch (e) {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
}
|
||||
86
tests/e2e/core/actions/navigation.ts
Normal file
86
tests/e2e/core/actions/navigation.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
import { Page } from '@playwright/test'
|
||||
import { waitForNetworkIdle } from '../utils/test-helpers'
|
||||
|
||||
/**
|
||||
* Navigation helper functions
|
||||
*/
|
||||
|
||||
/**
|
||||
* Navigate to the workspace root
|
||||
*/
|
||||
export async function goToWorkspace(page: Page): Promise<void> {
|
||||
await page.goto('/workspace')
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Providers page
|
||||
*/
|
||||
export async function goToProviders(page: Page): Promise<void> {
|
||||
await page.goto('/workspace/providers')
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Virtual Keys page
|
||||
*/
|
||||
export async function goToVirtualKeys(page: Page): Promise<void> {
|
||||
await page.goto('/workspace/virtual-keys')
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to User Groups page
|
||||
*/
|
||||
export async function goToUserGroups(page: Page): Promise<void> {
|
||||
await page.goto('/workspace/user-groups')
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to MCP Clients page
|
||||
*/
|
||||
export async function goToMCPClients(page: Page): Promise<void> {
|
||||
await page.goto('/workspace/mcp-clients')
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Logs page
|
||||
*/
|
||||
export async function goToLogs(page: Page): Promise<void> {
|
||||
await page.goto('/workspace/logs')
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Plugins page
|
||||
*/
|
||||
export async function goToPlugins(page: Page): Promise<void> {
|
||||
await page.goto('/workspace/plugins')
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Config page
|
||||
*/
|
||||
export async function goToConfig(page: Page): Promise<void> {
|
||||
await page.goto('/workspace/config')
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to a specific provider
|
||||
*/
|
||||
export async function goToProvider(page: Page, providerName: string): Promise<void> {
|
||||
await page.goto(`/workspace/providers?provider=${encodeURIComponent(providerName)}`)
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to a specific virtual key
|
||||
*/
|
||||
export async function goToVirtualKey(page: Page, vkId: string): Promise<void> {
|
||||
await page.goto(`/workspace/virtual-keys?vk=${encodeURIComponent(vkId)}`)
|
||||
await waitForNetworkIdle(page)
|
||||
}
|
||||
123
tests/e2e/core/fixtures/base.fixture.ts
Normal file
123
tests/e2e/core/fixtures/base.fixture.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
import { test as base, expect } from '@playwright/test'
|
||||
import { SidebarPage } from '../pages/sidebar.page'
|
||||
import { ProvidersPage } from '../../features/providers/pages/providers.page'
|
||||
import { VirtualKeysPage } from '../../features/virtual-keys/pages/virtual-keys.page'
|
||||
import { DashboardPage } from '../../features/dashboard/pages/dashboard.page'
|
||||
import { LogsPage } from '../../features/logs/pages/logs.page'
|
||||
import { MCPLogsPage } from '../../features/mcp-logs/pages/mcp-logs.page'
|
||||
import { RoutingRulesPage } from '../../features/routing-rules/pages/routing-rules.page'
|
||||
import { MCPRegistryPage } from '../../features/mcp-registry/pages/mcp-registry.page'
|
||||
import { PluginsPage } from '../../features/plugins/pages/plugins.page'
|
||||
import { ObservabilityPage } from '../../features/observability/pages/observability.page'
|
||||
import { ConfigSettingsPage } from '../../features/config/pages/config-settings.page'
|
||||
import { GovernancePage } from '../../features/governance/pages/governance.page'
|
||||
import { MCPAuthConfigPage } from '../../features/mcp-auth-config/pages/mcp-auth-config.page'
|
||||
import { MCPSettingsPage } from '../../features/mcp-settings/pages/mcp-settings.page'
|
||||
import { MCPToolGroupsPage } from '../../features/mcp-tool-groups/pages/mcp-tool-groups.page'
|
||||
import { ModelLimitsPage } from '../../features/model-limits/pages/model-limits.page'
|
||||
|
||||
/**
|
||||
* Custom test fixtures type
|
||||
*/
|
||||
type BifrostFixtures = {
|
||||
closeDevProfiler: void
|
||||
sidebarPage: SidebarPage
|
||||
providersPage: ProvidersPage
|
||||
virtualKeysPage: VirtualKeysPage
|
||||
dashboardPage: DashboardPage
|
||||
logsPage: LogsPage
|
||||
mcpLogsPage: MCPLogsPage
|
||||
routingRulesPage: RoutingRulesPage
|
||||
mcpRegistryPage: MCPRegistryPage
|
||||
pluginsPage: PluginsPage
|
||||
observabilityPage: ObservabilityPage
|
||||
configSettingsPage: ConfigSettingsPage
|
||||
governancePage: GovernancePage
|
||||
modelLimitsPage: ModelLimitsPage
|
||||
mcpSettingsPage: MCPSettingsPage
|
||||
mcpToolGroupsPage: MCPToolGroupsPage
|
||||
mcpAuthConfigPage: MCPAuthConfigPage
|
||||
}
|
||||
|
||||
/**
|
||||
* Extended test with Bifrost-specific fixtures
|
||||
*/
|
||||
export const test = base.extend<BifrostFixtures>({
|
||||
closeDevProfiler: [async ({ page }, use) => {
|
||||
// Automatically dismiss the Dev Profiler overlay whenever it appears.
|
||||
// Uses addLocatorHandler so it triggers before any test action if the profiler is visible.
|
||||
await page.addLocatorHandler(
|
||||
page.getByText('Dev Profiler', { exact: true }),
|
||||
async () => {
|
||||
await page.locator('button[title="Dismiss"]').click({ force: true })
|
||||
}
|
||||
)
|
||||
await use()
|
||||
}, { auto: true }],
|
||||
|
||||
sidebarPage: async ({ page }, use) => {
|
||||
await use(new SidebarPage(page))
|
||||
},
|
||||
|
||||
providersPage: async ({ page }, use) => {
|
||||
await use(new ProvidersPage(page))
|
||||
},
|
||||
|
||||
virtualKeysPage: async ({ page }, use) => {
|
||||
await use(new VirtualKeysPage(page))
|
||||
},
|
||||
|
||||
dashboardPage: async ({ page }, use) => {
|
||||
await use(new DashboardPage(page))
|
||||
},
|
||||
|
||||
logsPage: async ({ page }, use) => {
|
||||
await use(new LogsPage(page))
|
||||
},
|
||||
|
||||
mcpLogsPage: async ({ page }, use) => {
|
||||
await use(new MCPLogsPage(page))
|
||||
},
|
||||
|
||||
routingRulesPage: async ({ page }, use) => {
|
||||
await use(new RoutingRulesPage(page))
|
||||
},
|
||||
|
||||
mcpRegistryPage: async ({ page }, use) => {
|
||||
await use(new MCPRegistryPage(page))
|
||||
},
|
||||
|
||||
pluginsPage: async ({ page }, use) => {
|
||||
await use(new PluginsPage(page))
|
||||
},
|
||||
|
||||
observabilityPage: async ({ page }, use) => {
|
||||
await use(new ObservabilityPage(page))
|
||||
},
|
||||
|
||||
configSettingsPage: async ({ page }, use) => {
|
||||
await use(new ConfigSettingsPage(page))
|
||||
},
|
||||
|
||||
governancePage: async ({ page }, use) => {
|
||||
await use(new GovernancePage(page))
|
||||
},
|
||||
|
||||
modelLimitsPage: async ({ page }, use) => {
|
||||
await use(new ModelLimitsPage(page))
|
||||
},
|
||||
|
||||
mcpSettingsPage: async ({ page }, use) => {
|
||||
await use(new MCPSettingsPage(page))
|
||||
},
|
||||
|
||||
mcpToolGroupsPage: async ({ page }, use) => {
|
||||
await use(new MCPToolGroupsPage(page))
|
||||
},
|
||||
|
||||
mcpAuthConfigPage: async ({ page }, use) => {
|
||||
await use(new MCPAuthConfigPage(page))
|
||||
},
|
||||
})
|
||||
|
||||
export { expect }
|
||||
172
tests/e2e/core/fixtures/test-data.fixture.ts
Normal file
172
tests/e2e/core/fixtures/test-data.fixture.ts
Normal file
@@ -0,0 +1,172 @@
|
||||
import { test as base } from '@playwright/test'
|
||||
import { randomUUID } from 'crypto'
|
||||
|
||||
/**
|
||||
* Test data types
|
||||
*/
|
||||
export interface ProviderKeyConfig {
|
||||
name: string
|
||||
value: string
|
||||
models?: string[]
|
||||
weight?: number
|
||||
}
|
||||
|
||||
export interface CustomProviderConfig {
|
||||
name: string
|
||||
baseProviderType: 'openai' | 'anthropic' | 'gemini' | 'cohere' | 'bedrock' | string
|
||||
baseUrl?: string
|
||||
authType?: 'api_key' | 'bearer' | 'basic' | 'none'
|
||||
isKeyless?: boolean
|
||||
}
|
||||
|
||||
export interface VirtualKeyConfig {
|
||||
name: string
|
||||
description?: string
|
||||
isActive?: boolean
|
||||
providerConfigs?: ProviderConfigItem[]
|
||||
budget?: BudgetConfig
|
||||
rateLimit?: RateLimitConfig
|
||||
teamId?: string
|
||||
customerId?: string
|
||||
}
|
||||
|
||||
export interface ProviderConfigItem {
|
||||
provider: string
|
||||
weight?: number
|
||||
allowedModels?: string[]
|
||||
keyIds?: string[]
|
||||
budget?: BudgetConfig
|
||||
rateLimit?: RateLimitConfig
|
||||
}
|
||||
|
||||
export interface BudgetConfig {
|
||||
maxLimit: number
|
||||
resetDuration: string
|
||||
}
|
||||
|
||||
export interface RateLimitConfig {
|
||||
tokenMaxLimit?: number
|
||||
tokenResetDuration?: string
|
||||
requestMaxLimit?: number
|
||||
requestResetDuration?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Test data fixture type
|
||||
*/
|
||||
type TestDataFixtures = {
|
||||
testData: TestDataFactory
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory for creating test data with unique identifiers
|
||||
*/
|
||||
export class TestDataFactory {
|
||||
private counter = 0
|
||||
private runId = randomUUID()
|
||||
|
||||
/**
|
||||
* Generate a unique ID for test data
|
||||
*/
|
||||
uniqueId(prefix = 'test'): string {
|
||||
this.counter++
|
||||
return `${prefix}-${this.runId}-${this.counter}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Create provider key test data
|
||||
*/
|
||||
createProviderKey(overrides: Partial<ProviderKeyConfig> = {}): ProviderKeyConfig {
|
||||
return {
|
||||
name: this.uniqueId('key'),
|
||||
value: `sk-test-${this.uniqueId()}`,
|
||||
models: ['*'],
|
||||
weight: 1.0,
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create custom provider test data
|
||||
*/
|
||||
createCustomProvider(overrides: Partial<CustomProviderConfig> = {}): CustomProviderConfig {
|
||||
return {
|
||||
name: this.uniqueId('provider'),
|
||||
baseProviderType: 'openai',
|
||||
baseUrl: 'https://api.example.com',
|
||||
authType: 'api_key',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create virtual key test data
|
||||
*/
|
||||
createVirtualKey(overrides: Partial<VirtualKeyConfig> = {}): VirtualKeyConfig {
|
||||
return {
|
||||
name: this.uniqueId('vk'),
|
||||
description: 'Test virtual key',
|
||||
isActive: true,
|
||||
providerConfigs: [],
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create virtual key with budget
|
||||
*/
|
||||
createVirtualKeyWithBudget(
|
||||
budgetOverrides: Partial<BudgetConfig> = {},
|
||||
vkOverrides: Partial<VirtualKeyConfig> = {}
|
||||
): VirtualKeyConfig {
|
||||
return this.createVirtualKey({
|
||||
budget: {
|
||||
maxLimit: 100,
|
||||
resetDuration: '1M',
|
||||
...budgetOverrides,
|
||||
},
|
||||
...vkOverrides,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Create virtual key with rate limits
|
||||
*/
|
||||
createVirtualKeyWithRateLimit(
|
||||
rateLimitOverrides: Partial<RateLimitConfig> = {},
|
||||
vkOverrides: Partial<VirtualKeyConfig> = {}
|
||||
): VirtualKeyConfig {
|
||||
return this.createVirtualKey({
|
||||
rateLimit: {
|
||||
tokenMaxLimit: 10000,
|
||||
tokenResetDuration: '1h',
|
||||
requestMaxLimit: 1000,
|
||||
requestResetDuration: '1h',
|
||||
...rateLimitOverrides,
|
||||
},
|
||||
...vkOverrides,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Create provider config item for virtual key
|
||||
*/
|
||||
createProviderConfigItem(overrides: Partial<ProviderConfigItem> = {}): ProviderConfigItem {
|
||||
return {
|
||||
provider: 'openai',
|
||||
weight: 1.0,
|
||||
allowedModels: ['*'],
|
||||
keyIds: ['*'],
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extended test with test data fixture
|
||||
*/
|
||||
export const testWithData = base.extend<TestDataFixtures>({
|
||||
testData: async (_, use) => {
|
||||
await use(new TestDataFactory())
|
||||
},
|
||||
})
|
||||
27
tests/e2e/core/index.ts
Normal file
27
tests/e2e/core/index.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Core module exports
|
||||
*/
|
||||
|
||||
// Fixtures
|
||||
export { test, expect } from './fixtures/base.fixture'
|
||||
export { testWithData, TestDataFactory } from './fixtures/test-data.fixture'
|
||||
export type {
|
||||
ProviderKeyConfig,
|
||||
CustomProviderConfig,
|
||||
VirtualKeyConfig,
|
||||
ProviderConfigItem,
|
||||
BudgetConfig,
|
||||
RateLimitConfig,
|
||||
} from './fixtures/test-data.fixture'
|
||||
|
||||
// Page Objects
|
||||
export { BasePage } from './pages/base.page'
|
||||
export { SidebarPage } from './pages/sidebar.page'
|
||||
|
||||
// Actions
|
||||
export * from './actions/navigation'
|
||||
export * from './actions/api'
|
||||
|
||||
// Utils
|
||||
export { Selectors } from './utils/selectors'
|
||||
export * from './utils/test-helpers'
|
||||
219
tests/e2e/core/pages/base.page.ts
Normal file
219
tests/e2e/core/pages/base.page.ts
Normal file
@@ -0,0 +1,219 @@
|
||||
import { Locator, Page, expect } from '@playwright/test'
|
||||
|
||||
/**
|
||||
* Base page object with common methods shared across all pages
|
||||
*/
|
||||
export class BasePage {
|
||||
readonly page: Page
|
||||
|
||||
constructor(page: Page) {
|
||||
this.page = page
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the page to finish loading
|
||||
*/
|
||||
async waitForPageLoad(): Promise<void> {
|
||||
await this.page.waitForLoadState('networkidle')
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the toast notification element (first/most recent one)
|
||||
* Filters out toasts that are being removed to avoid matching stale toasts.
|
||||
* Optionally filters by toast type (success, error, loading, default).
|
||||
*/
|
||||
getToast(type?: 'success' | 'error' | 'loading' | 'default'): Locator {
|
||||
const selector = type
|
||||
? `[data-sonner-toast][data-type="${type}"]:not([data-removed="true"])`
|
||||
: '[data-sonner-toast]:not([data-removed="true"])'
|
||||
return this.page.locator(selector).first()
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for a success toast to appear
|
||||
*/
|
||||
async waitForSuccessToast(message?: string): Promise<void> {
|
||||
const toast = this.getToast('success')
|
||||
await expect(toast).toBeVisible({ timeout: 10000 })
|
||||
if (message) {
|
||||
await expect(toast).toContainText(message)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for an error toast to appear
|
||||
*/
|
||||
async waitForErrorToast(message?: string): Promise<void> {
|
||||
const toast = this.getToast('error')
|
||||
await expect(toast).toBeVisible({ timeout: 10000 })
|
||||
if (message) {
|
||||
await expect(toast).toContainText(message)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for all toasts to disappear
|
||||
*/
|
||||
async waitForToastsToDisappear(timeout = 5000): Promise<void> {
|
||||
const toasts = this.page.locator('[data-sonner-toast]:not([data-removed="true"])')
|
||||
try {
|
||||
// Wait for all toasts to be detached from DOM
|
||||
await toasts.first().waitFor({ state: 'detached', timeout }).catch(() => {
|
||||
// If no toasts exist, that's fine
|
||||
})
|
||||
// Also check if count is 0
|
||||
const count = await toasts.count()
|
||||
if (count > 0) {
|
||||
// Wait for toasts to be hidden
|
||||
await expect(toasts.first()).not.toBeVisible({ timeout: 3000 }).catch(() => {})
|
||||
}
|
||||
} catch {
|
||||
// No toasts present, which is fine
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for a sheet/dialog to be fully visible (animation complete)
|
||||
*/
|
||||
async waitForSheetAnimation(): Promise<void> {
|
||||
// Wait for any sheet transition to complete by checking for stable state
|
||||
await this.page.waitForFunction(() => {
|
||||
const sheet = document.querySelector('[role="dialog"]')
|
||||
if (!sheet) return true
|
||||
const style = window.getComputedStyle(sheet)
|
||||
return style.opacity === '1' && style.transform === 'none'
|
||||
}, { timeout: 2000 }).catch(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for element state to change (useful for toggles)
|
||||
*/
|
||||
async waitForStateChange(locator: Locator, attribute: string, expectedValue: string, timeout = 5000): Promise<void> {
|
||||
await expect(locator).toHaveAttribute(attribute, expectedValue, { timeout })
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for URL to contain a specific parameter
|
||||
*/
|
||||
async waitForUrlParam(param: string, value: string, timeout = 5000): Promise<void> {
|
||||
await expect(this.page).toHaveURL(new RegExp(`${param}=${value}`), { timeout })
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for charts/data to load after page navigation
|
||||
*/
|
||||
async waitForChartsToLoad(): Promise<void> {
|
||||
// Wait for network to be idle (data fetching complete)
|
||||
await this.page.waitForLoadState('networkidle')
|
||||
// Wait for any loading skeletons to disappear
|
||||
const skeletons = this.page.locator('[data-testid="skeleton"], .skeleton, [data-loading="true"]')
|
||||
if (await skeletons.count() > 0) {
|
||||
await skeletons.first().waitFor({ state: 'hidden', timeout: 10000 }).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dismiss all visible toasts by waiting for them to disappear
|
||||
*/
|
||||
async dismissToasts(): Promise<void> {
|
||||
// Just wait for toasts to auto-dismiss
|
||||
await this.waitForToastsToDisappear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Force dismiss all toasts by clicking away and waiting
|
||||
*/
|
||||
async forceCloseToasts(): Promise<void> {
|
||||
// Click somewhere neutral to potentially dismiss toasts
|
||||
await this.page.locator('body').click({ position: { x: 10, y: 10 }, force: true }).catch(() => {})
|
||||
|
||||
// Wait for toasts to auto-dismiss (they typically auto-dismiss after 4-5 seconds)
|
||||
await this.waitForToastsToDisappear(8000)
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the Dev Profiler overlay if it is visible.
|
||||
* Clicks the dismiss (X) button on the profiler panel. Silently continues if not present.
|
||||
*/
|
||||
async closeDevProfiler(): Promise<void> {
|
||||
const profilerHeader = this.page.locator('text=Dev Profiler')
|
||||
const isVisible = await profilerHeader.isVisible().catch(() => false)
|
||||
if (isVisible) {
|
||||
const dismissBtn = this.page.locator('button[title="Dismiss"]')
|
||||
if (await dismissBtn.isVisible().catch(() => false)) {
|
||||
await dismissBtn.click()
|
||||
await profilerHeader.waitFor({ state: 'hidden', timeout: 3000 }).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill a form field by label
|
||||
*/
|
||||
async fillByLabel(label: string, value: string): Promise<void> {
|
||||
await this.page.getByLabel(label).fill(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill a form field by placeholder
|
||||
*/
|
||||
async fillByPlaceholder(placeholder: string, value: string): Promise<void> {
|
||||
await this.page.getByPlaceholder(placeholder).fill(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill a form field by test id
|
||||
*/
|
||||
async fillByTestId(testId: string, value: string): Promise<void> {
|
||||
await this.page.getByTestId(testId).fill(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Click a button by text
|
||||
*/
|
||||
async clickButton(text: string): Promise<void> {
|
||||
await this.page.getByRole('button', { name: text }).click()
|
||||
}
|
||||
|
||||
/**
|
||||
* Click a button by test id
|
||||
*/
|
||||
async clickByTestId(testId: string): Promise<void> {
|
||||
await this.page.getByTestId(testId).click()
|
||||
}
|
||||
|
||||
/**
|
||||
* Select an option from a dropdown by label
|
||||
*/
|
||||
async selectOption(label: string, value: string): Promise<void> {
|
||||
await this.page.getByLabel(label).selectOption(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an element is visible
|
||||
*/
|
||||
async isVisible(selector: string): Promise<boolean> {
|
||||
return await this.page.locator(selector).isVisible()
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for an element to be visible
|
||||
*/
|
||||
async waitForSelector(selector: string, timeout = 10000): Promise<void> {
|
||||
await this.page.waitForSelector(selector, { timeout })
|
||||
}
|
||||
|
||||
/**
|
||||
* Get text content of an element
|
||||
*/
|
||||
async getTextContent(selector: string): Promise<string | null> {
|
||||
return await this.page.locator(selector).textContent()
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a screenshot
|
||||
*/
|
||||
async screenshot(name: string): Promise<void> {
|
||||
await this.page.screenshot({ path: `./screenshots/${name}.png` })
|
||||
}
|
||||
}
|
||||
83
tests/e2e/core/pages/sidebar.page.ts
Normal file
83
tests/e2e/core/pages/sidebar.page.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import { Page, Locator } from '@playwright/test'
|
||||
import { BasePage } from './base.page'
|
||||
|
||||
/**
|
||||
* Sidebar navigation page object
|
||||
*/
|
||||
export class SidebarPage extends BasePage {
|
||||
// Navigation links
|
||||
readonly providersLink: Locator
|
||||
readonly virtualKeysLink: Locator
|
||||
readonly logsLink: Locator
|
||||
readonly mcpClientsLink: Locator
|
||||
readonly userGroupsLink: Locator
|
||||
readonly pluginsLink: Locator
|
||||
readonly configLink: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
this.providersLink = page.getByRole('link', { name: /providers/i })
|
||||
this.virtualKeysLink = page.getByRole('link', { name: /virtual keys/i })
|
||||
this.logsLink = page.getByRole('link', { name: /logs/i })
|
||||
this.mcpClientsLink = page.getByRole('link', { name: /mcp/i })
|
||||
this.userGroupsLink = page.getByRole('link', { name: /user groups/i })
|
||||
this.pluginsLink = page.getByRole('link', { name: /plugins/i })
|
||||
this.configLink = page.getByRole('link', { name: /config/i })
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Providers page
|
||||
*/
|
||||
async goToProviders(): Promise<void> {
|
||||
await this.providersLink.click()
|
||||
await this.waitForPageLoad()
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Virtual Keys page
|
||||
*/
|
||||
async goToVirtualKeys(): Promise<void> {
|
||||
await this.virtualKeysLink.click()
|
||||
await this.waitForPageLoad()
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Logs page
|
||||
*/
|
||||
async goToLogs(): Promise<void> {
|
||||
await this.logsLink.click()
|
||||
await this.waitForPageLoad()
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to MCP Clients page
|
||||
*/
|
||||
async goToMCPClients(): Promise<void> {
|
||||
await this.mcpClientsLink.click()
|
||||
await this.waitForPageLoad()
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to User Groups page
|
||||
*/
|
||||
async goToUserGroups(): Promise<void> {
|
||||
await this.userGroupsLink.click()
|
||||
await this.waitForPageLoad()
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Plugins page
|
||||
*/
|
||||
async goToPlugins(): Promise<void> {
|
||||
await this.pluginsLink.click()
|
||||
await this.waitForPageLoad()
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to Config page
|
||||
*/
|
||||
async goToConfig(): Promise<void> {
|
||||
await this.configLink.click()
|
||||
await this.waitForPageLoad()
|
||||
}
|
||||
}
|
||||
102
tests/e2e/core/utils/selectors.ts
Normal file
102
tests/e2e/core/utils/selectors.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Centralized selectors for Bifrost UI elements
|
||||
* Using data-testid attributes where available, falling back to other strategies
|
||||
*/
|
||||
|
||||
export const Selectors = {
|
||||
// Common
|
||||
toast: '[data-sonner-toast]:not([data-removed="true"])',
|
||||
loadingSpinner: '[data-testid="loading-spinner"]',
|
||||
|
||||
// Providers Page
|
||||
providers: {
|
||||
// Sidebar list
|
||||
providerList: '[data-testid="provider-list"]',
|
||||
providerItem: (name: string) => `[data-testid="provider-item-${name.replace(/[^a-z0-9]+/gi, "-").toLowerCase()}"]`,
|
||||
addProviderBtn: '[data-testid="add-provider-btn"]',
|
||||
/** Add New Provider dropdown > Custom provider... (opens custom provider sheet) */
|
||||
addProviderOptionCustom: '[data-testid="add-provider-option-custom"]',
|
||||
|
||||
// Provider config
|
||||
providerConfig: '[data-testid="provider-config"]',
|
||||
addKeyBtn: '[data-testid="add-key-btn"]',
|
||||
keysTable: '[data-testid="keys-table"]',
|
||||
keyRow: (name: string) => `[data-testid="key-row-${name}"]`,
|
||||
|
||||
// Key form
|
||||
keyForm: {
|
||||
container: '[data-testid="key-form"]',
|
||||
nameInput: '[data-testid="key-name-input"]',
|
||||
valueInput: '[data-testid="key-value-input"]',
|
||||
modelsInput: '[data-testid="key-models-input"]',
|
||||
weightInput: '[data-testid="key-weight-input"]',
|
||||
saveBtn: '[data-testid="key-save-btn"]',
|
||||
cancelBtn: '[data-testid="key-cancel-btn"]',
|
||||
},
|
||||
|
||||
// Custom provider sheet
|
||||
customProviderSheet: {
|
||||
container: '[data-testid="custom-provider-sheet"]',
|
||||
nameInput: '[data-testid="custom-provider-name"]',
|
||||
baseProviderSelect: '[data-testid="base-provider-select"]',
|
||||
baseUrlInput: '[data-testid="base-url-input"]',
|
||||
saveBtn: '[data-testid="custom-provider-save-btn"]',
|
||||
cancelBtn: '[data-testid="custom-provider-cancel-btn"]',
|
||||
},
|
||||
},
|
||||
|
||||
// Virtual Keys Page
|
||||
virtualKeys: {
|
||||
// Table
|
||||
table: '[data-testid="vk-table"]',
|
||||
row: (name: string) => `[data-testid="vk-row-${name}"]`,
|
||||
createBtn: '[data-testid="create-vk-btn"]',
|
||||
|
||||
// Sheet/Form
|
||||
sheet: {
|
||||
container: '[data-testid="vk-sheet"]',
|
||||
nameInput: '[data-testid="vk-name-input"]',
|
||||
descriptionInput: '[data-testid="vk-description-input"]',
|
||||
isActiveToggle: '[data-testid="vk-is-active-toggle"]',
|
||||
|
||||
// Provider configs
|
||||
providerSelect: '[data-testid="vk-provider-select"]',
|
||||
|
||||
// Entity assignment
|
||||
entityTypeSelect: '[data-testid="vk-entity-type-select"]',
|
||||
teamSelect: '[data-testid="vk-team-select"]',
|
||||
customerSelect: '[data-testid="vk-customer-select"]',
|
||||
|
||||
// Actions
|
||||
saveBtn: '[data-testid="vk-save-btn"]',
|
||||
cancelBtn: '[data-testid="vk-cancel-btn"]',
|
||||
},
|
||||
},
|
||||
|
||||
// User Groups Page
|
||||
userGroups: {
|
||||
teamsTab: '[data-testid="teams-tab"]',
|
||||
customersTab: '[data-testid="customers-tab"]',
|
||||
teamsTable: '[data-testid="teams-table"]',
|
||||
customersTable: '[data-testid="customers-table"]',
|
||||
createTeamBtn: '[data-testid="create-team-btn"]',
|
||||
createCustomerBtn: '[data-testid="customer-button-create"]',
|
||||
},
|
||||
|
||||
// Common form elements
|
||||
form: {
|
||||
input: (name: string) => `[data-testid="input-${name}"]`,
|
||||
select: (name: string) => `[data-testid="select-${name}"]`,
|
||||
toggle: (name: string) => `[data-testid="toggle-${name}"]`,
|
||||
saveBtn: '[data-testid="btn-save"]',
|
||||
cancelBtn: '[data-testid="btn-cancel"]',
|
||||
deleteBtn: '[data-testid="btn-delete"]',
|
||||
},
|
||||
|
||||
// Dialogs
|
||||
dialog: {
|
||||
container: '[role="dialog"]',
|
||||
confirmBtn: '[data-testid="dialog-confirm-btn"]',
|
||||
cancelBtn: '[data-testid="dialog-cancel-btn"]',
|
||||
},
|
||||
}
|
||||
169
tests/e2e/core/utils/test-helpers.ts
Normal file
169
tests/e2e/core/utils/test-helpers.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
import { Page, expect } from '@playwright/test';
|
||||
|
||||
/**
|
||||
* Wait for network to be idle
|
||||
*/
|
||||
export async function waitForNetworkIdle(page: Page, timeout = 5000): Promise<void> {
|
||||
await page.waitForLoadState('networkidle', { timeout })
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for a specific number of milliseconds
|
||||
*/
|
||||
export async function wait(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry a function until it succeeds or times out
|
||||
*/
|
||||
export async function retry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: { retries?: number; delay?: number } = {}
|
||||
): Promise<T> {
|
||||
const { retries = 3, delay = 1000 } = options
|
||||
let lastError: Error | undefined
|
||||
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
return await fn()
|
||||
} catch (error) {
|
||||
lastError = error as Error
|
||||
if (i < retries - 1) {
|
||||
await wait(delay)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a random string
|
||||
*/
|
||||
export function randomString(length = 8): string {
|
||||
return Math.random().toString(36).substring(2).padEnd(length, '0').substring(0, length)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique test name
|
||||
*/
|
||||
export function uniqueTestName(prefix: string): string {
|
||||
return `${prefix}-${Date.now()}-${randomString(4)}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that a toast message appears
|
||||
*/
|
||||
export async function assertToast(
|
||||
page: Page,
|
||||
expectedText: string,
|
||||
type: 'success' | 'error' | 'info' = 'success'
|
||||
): Promise<void> {
|
||||
const selector = `[data-sonner-toast][data-type="${type}"]:not([data-removed="true"])`
|
||||
const toast = page.locator(selector).first()
|
||||
await expect(toast).toBeVisible({ timeout: 10000 })
|
||||
await expect(toast).toContainText(expectedText)
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that page URL matches expected pattern
|
||||
*/
|
||||
export async function assertUrl(page: Page, pattern: string | RegExp): Promise<void> {
|
||||
await expect(page).toHaveURL(pattern)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill a Radix/Shadcn Select component
|
||||
*/
|
||||
export async function fillSelect(
|
||||
page: Page,
|
||||
triggerSelector: string,
|
||||
optionText: string
|
||||
): Promise<void> {
|
||||
// Click the trigger to open the dropdown
|
||||
await page.locator(triggerSelector).click()
|
||||
|
||||
// Wait for the dropdown content to appear
|
||||
await page.waitForSelector('[role="listbox"]', { timeout: 5000 })
|
||||
|
||||
// Click the option
|
||||
await page.getByRole('option', { name: optionText }).click()
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill a multi-select component
|
||||
*/
|
||||
export async function fillMultiSelect(
|
||||
page: Page,
|
||||
inputSelector: string,
|
||||
values: string[]
|
||||
): Promise<void> {
|
||||
const input = page.locator(inputSelector)
|
||||
|
||||
for (const value of values) {
|
||||
await input.fill(value)
|
||||
await page.keyboard.press('Enter')
|
||||
await wait(100) // Small delay between entries
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear and fill an input
|
||||
*/
|
||||
export async function clearAndFill(page: Page, selector: string, value: string): Promise<void> {
|
||||
const input = page.locator(selector)
|
||||
await input.clear()
|
||||
await input.fill(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get table row count
|
||||
*/
|
||||
export async function getTableRowCount(page: Page, tableSelector: string): Promise<number> {
|
||||
const rows = page.locator(`${tableSelector} tbody tr`)
|
||||
return await rows.count()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if table contains a row with specific text
|
||||
*/
|
||||
export async function tableContainsRow(
|
||||
page: Page,
|
||||
tableSelector: string,
|
||||
text: string
|
||||
): Promise<boolean> {
|
||||
const table = page.locator(tableSelector)
|
||||
const row = table.locator('tbody tr', { hasText: text })
|
||||
return await row.count() > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for table to load (no loading indicator)
|
||||
*/
|
||||
export async function waitForTableLoad(page: Page, tableSelector: string): Promise<void> {
|
||||
// Wait for table to be visible
|
||||
await page.locator(tableSelector).waitFor({ state: 'visible' })
|
||||
|
||||
// Wait for any loading spinners to disappear
|
||||
const loadingIndicator = page.locator('[data-testid="loading-spinner"]')
|
||||
if (await loadingIndicator.count() > 0) {
|
||||
await loadingIndicator.waitFor({ state: 'hidden', timeout: 10000 })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Screenshot on failure helper
|
||||
*/
|
||||
export async function screenshotOnError(
|
||||
page: Page,
|
||||
testName: string,
|
||||
fn: () => Promise<void>
|
||||
): Promise<void> {
|
||||
try {
|
||||
await fn()
|
||||
} catch (error) {
|
||||
await page.screenshot({ path: `./screenshots/error-${testName}-${Date.now()}.png` })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
57
tests/e2e/features/config/config.data.ts
Normal file
57
tests/e2e/features/config/config.data.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Test data factories for config settings tests
|
||||
*/
|
||||
|
||||
/**
|
||||
* Config toggle state interface
|
||||
*/
|
||||
export interface ConfigToggleState {
|
||||
name: string
|
||||
enabled: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Client settings data factory
|
||||
*/
|
||||
export function createClientSettingsData(overrides: Partial<{
|
||||
dropExcessRequests: boolean
|
||||
enableLiteLLMFallbacks: boolean
|
||||
disableDBPings: boolean
|
||||
}> = {}) {
|
||||
return {
|
||||
dropExcessRequests: false,
|
||||
enableLiteLLMFallbacks: true,
|
||||
disableDBPings: false,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Logging settings data factory
|
||||
*/
|
||||
export function createLoggingSettingsData(overrides: Partial<{
|
||||
enableLogging: boolean
|
||||
disableContentLogging: boolean
|
||||
retentionDays: number
|
||||
}> = {}) {
|
||||
return {
|
||||
enableLogging: true,
|
||||
disableContentLogging: false,
|
||||
retentionDays: 30,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Performance tuning settings data factory
|
||||
*/
|
||||
export function createPerformanceTuningData(overrides: Partial<{
|
||||
workerPoolSize: number
|
||||
maxRequestBodySize: number
|
||||
}> = {}) {
|
||||
return {
|
||||
workerPoolSize: 100,
|
||||
maxRequestBodySize: 10485760, // 10MB
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
547
tests/e2e/features/config/config.spec.ts
Normal file
547
tests/e2e/features/config/config.spec.ts
Normal file
@@ -0,0 +1,547 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
import { ConfigSettingsState } from './pages/config-settings.page'
|
||||
|
||||
test.describe('Config Settings', () => {
|
||||
// Run all config tests serially to avoid parallel writes to the same config/store
|
||||
test.describe.configure({ mode: 'serial' })
|
||||
|
||||
test.describe('Navigation', () => {
|
||||
test('should navigate to client settings', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('client-settings')
|
||||
await expect(configSettingsPage.saveBtn).toBeVisible()
|
||||
// Use heading to avoid matching sidebar link
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Client Settings/i })).toBeVisible()
|
||||
})
|
||||
|
||||
test('should navigate to caching config', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('caching')
|
||||
// Caching page exists - verify page loaded
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Caching/i })).toBeVisible()
|
||||
})
|
||||
|
||||
test('should navigate to logging config', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('logging')
|
||||
await expect(configSettingsPage.saveBtn).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Logging/i })).toBeVisible()
|
||||
})
|
||||
|
||||
test('should navigate to security config', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('security')
|
||||
await expect(configSettingsPage.saveBtn).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Security/i })).toBeVisible()
|
||||
})
|
||||
|
||||
test('should navigate to performance tuning config', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('performance-tuning')
|
||||
await expect(configSettingsPage.saveBtn).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Performance Tuning/i })).toBeVisible()
|
||||
})
|
||||
|
||||
test('should navigate to pricing config', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('pricing-config')
|
||||
await expect(configSettingsPage.saveBtn).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Pricing/i })).toBeVisible()
|
||||
})
|
||||
|
||||
test('should navigate to MCP settings', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('mcp-gateway')
|
||||
await expect(configSettingsPage.page.getByTestId('mcp-settings-view')).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByTestId('mcp-agent-depth-input')).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByTestId('mcp-tool-timeout-input')).toBeVisible()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('MCP Settings', () => {
|
||||
test('should display MCP settings form', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('mcp-gateway')
|
||||
|
||||
await expect(configSettingsPage.page.getByTestId('mcp-settings-view')).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByTestId('mcp-agent-depth-input')).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByTestId('mcp-tool-timeout-input')).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByTestId('mcp-binding-level')).toBeVisible()
|
||||
})
|
||||
|
||||
test('should have save button disabled when no changes', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('mcp-gateway')
|
||||
|
||||
const saveBtn = configSettingsPage.page.getByTestId('mcp-settings-save-btn')
|
||||
await expect(saveBtn).toBeVisible()
|
||||
await expect(saveBtn).toBeDisabled()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Pricing Config', () => {
|
||||
let originalPricingUrl: string | null = null
|
||||
|
||||
test.beforeEach(async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('pricing-config')
|
||||
originalPricingUrl = await configSettingsPage.pricingDatasheetUrlInput.inputValue()
|
||||
})
|
||||
|
||||
test.afterEach(async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('pricing-config')
|
||||
const canEdit = await configSettingsPage.pricingDatasheetUrlInput.isEditable().catch(() => false)
|
||||
if (!canEdit || originalPricingUrl === null) return
|
||||
await configSettingsPage.setPricingDatasheetUrl(originalPricingUrl)
|
||||
const isSaveEnabled = await configSettingsPage.pricingSaveBtn.isDisabled().then((d) => !d)
|
||||
if (isSaveEnabled) {
|
||||
await configSettingsPage.savePricingConfig()
|
||||
await configSettingsPage.dismissToasts()
|
||||
}
|
||||
})
|
||||
|
||||
test('should display pricing config view', async ({ configSettingsPage }) => {
|
||||
await expect(configSettingsPage.pricingConfigView).toBeVisible()
|
||||
await expect(configSettingsPage.pricingDatasheetUrlInput).toBeVisible()
|
||||
await expect(configSettingsPage.pricingForceSyncBtn).toBeVisible()
|
||||
await expect(configSettingsPage.pricingSaveBtn).toBeVisible()
|
||||
})
|
||||
|
||||
test('should set and save datasheet URL', async ({ configSettingsPage }) => {
|
||||
const testUrl = 'https://example.com/pricing.json'
|
||||
await configSettingsPage.setPricingDatasheetUrl(testUrl)
|
||||
|
||||
const isSaveEnabled = await configSettingsPage.pricingSaveBtn.isDisabled().then((d) => !d)
|
||||
if (!isSaveEnabled) {
|
||||
test.skip(true, 'Save button disabled (no changes detected or RBAC)')
|
||||
return
|
||||
}
|
||||
|
||||
await configSettingsPage.savePricingConfig()
|
||||
await configSettingsPage.dismissToasts()
|
||||
})
|
||||
|
||||
test('should trigger force sync', async ({ configSettingsPage }) => {
|
||||
const isForceSyncEnabled = await configSettingsPage.pricingForceSyncBtn.isDisabled().then((d) => !d)
|
||||
if (!isForceSyncEnabled) {
|
||||
test.skip(true, 'Force sync button disabled (RBAC or no datasheet URL)')
|
||||
return
|
||||
}
|
||||
|
||||
await configSettingsPage.triggerForceSync()
|
||||
await configSettingsPage.dismissToasts()
|
||||
})
|
||||
|
||||
test('should validate URL format', async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.pricingDatasheetUrlInput.fill('invalid-url-no-http')
|
||||
const canSave = await configSettingsPage.pricingSaveBtn.isDisabled().then((d) => !d)
|
||||
if (!canSave) {
|
||||
test.skip(true, 'Save button disabled (RBAC)')
|
||||
return
|
||||
}
|
||||
await configSettingsPage.pricingSaveBtn.click()
|
||||
|
||||
await expect(configSettingsPage.page.getByText(/URL must start with http|valid URL/i)).toBeVisible()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Client Settings', () => {
|
||||
let originalState: ConfigSettingsState
|
||||
|
||||
test.beforeEach(async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('client-settings')
|
||||
// Capture original state for restoration
|
||||
originalState = await configSettingsPage.getCurrentSettings('client-settings')
|
||||
})
|
||||
|
||||
test.afterEach(async ({ configSettingsPage }) => {
|
||||
// Restore original settings
|
||||
if (originalState) {
|
||||
await configSettingsPage.restoreSettings(originalState)
|
||||
}
|
||||
})
|
||||
|
||||
test('should display client settings controls', async ({ configSettingsPage }) => {
|
||||
// Check for main controls
|
||||
await expect(configSettingsPage.dropExcessRequestsSwitch).toBeVisible()
|
||||
await expect(configSettingsPage.enableLiteLLMFallbacksSwitch).toBeVisible()
|
||||
await expect(configSettingsPage.disableDBPingsSwitch).toBeVisible()
|
||||
})
|
||||
|
||||
test('should display async job result TTL input when available', async ({ configSettingsPage }) => {
|
||||
const isVisible = await configSettingsPage.asyncJobResultTtlInput.isVisible().catch(() => false)
|
||||
if (isVisible) {
|
||||
await expect(configSettingsPage.asyncJobResultTtlInput).toBeVisible()
|
||||
} else {
|
||||
test.skip(true, 'Async job result TTL not available')
|
||||
}
|
||||
})
|
||||
|
||||
test('should toggle drop excess requests', async ({ configSettingsPage }) => {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.dropExcessRequestsSwitch)
|
||||
|
||||
await configSettingsPage.toggleDropExcessRequests()
|
||||
|
||||
const newState = await configSettingsPage.getSwitchState(configSettingsPage.dropExcessRequestsSwitch)
|
||||
expect(newState).toBe(!initialState)
|
||||
|
||||
// Verify changes are pending
|
||||
const hasChanges = await configSettingsPage.hasPendingChanges()
|
||||
expect(hasChanges).toBe(true)
|
||||
})
|
||||
|
||||
test('should save and persist drop excess requests toggle', async ({ configSettingsPage }) => {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.dropExcessRequestsSwitch)
|
||||
|
||||
await configSettingsPage.toggleDropExcessRequests()
|
||||
await configSettingsPage.saveSettings()
|
||||
await configSettingsPage.goto('client-settings')
|
||||
|
||||
const expectedState = !initialState
|
||||
await expect(configSettingsPage.dropExcessRequestsSwitch).toHaveAttribute(
|
||||
'data-state',
|
||||
expectedState ? 'checked' : 'unchecked'
|
||||
)
|
||||
})
|
||||
|
||||
test('should toggle LiteLLM fallbacks', async ({ configSettingsPage }) => {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.enableLiteLLMFallbacksSwitch)
|
||||
|
||||
await configSettingsPage.toggleLiteLLMFallbacks()
|
||||
|
||||
const newState = await configSettingsPage.getSwitchState(configSettingsPage.enableLiteLLMFallbacksSwitch)
|
||||
expect(newState).toBe(!initialState)
|
||||
})
|
||||
|
||||
test('should save and persist LiteLLM fallbacks toggle', async ({ configSettingsPage }) => {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.enableLiteLLMFallbacksSwitch)
|
||||
|
||||
await configSettingsPage.toggleLiteLLMFallbacks()
|
||||
await configSettingsPage.saveSettings()
|
||||
await configSettingsPage.goto('client-settings')
|
||||
|
||||
// Wait for persisted state (form is populated async after navigation)
|
||||
const expectedState = !initialState
|
||||
await expect(configSettingsPage.enableLiteLLMFallbacksSwitch).toHaveAttribute(
|
||||
'data-state',
|
||||
expectedState ? 'checked' : 'unchecked'
|
||||
)
|
||||
})
|
||||
|
||||
test('should toggle disable DB pings', async ({ configSettingsPage }) => {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.disableDBPingsSwitch)
|
||||
|
||||
await configSettingsPage.toggleDisableDBPings()
|
||||
|
||||
const newState = await configSettingsPage.getSwitchState(configSettingsPage.disableDBPingsSwitch)
|
||||
expect(newState).toBe(!initialState)
|
||||
})
|
||||
|
||||
test('should save and persist disable DB pings toggle', async ({ configSettingsPage }) => {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.disableDBPingsSwitch)
|
||||
|
||||
await configSettingsPage.toggleDisableDBPings()
|
||||
await configSettingsPage.saveSettings()
|
||||
await configSettingsPage.goto('client-settings')
|
||||
|
||||
const expectedState = !initialState
|
||||
await expect(configSettingsPage.disableDBPingsSwitch).toHaveAttribute(
|
||||
'data-state',
|
||||
expectedState ? 'checked' : 'unchecked'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Logging Settings', () => {
|
||||
let originalState: ConfigSettingsState
|
||||
|
||||
test.beforeEach(async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('logging')
|
||||
// Capture original state for restoration
|
||||
originalState = await configSettingsPage.getCurrentSettings('logging')
|
||||
})
|
||||
|
||||
test.afterEach(async ({ configSettingsPage }) => {
|
||||
// Restore original settings
|
||||
if (originalState) {
|
||||
await configSettingsPage.restoreSettings(originalState)
|
||||
}
|
||||
})
|
||||
|
||||
test('should display logging settings controls', async ({ configSettingsPage }) => {
|
||||
// Check for main logging controls
|
||||
await expect(configSettingsPage.page.getByText(/Enable Logs/i)).toBeVisible()
|
||||
await expect(configSettingsPage.page.getByText(/Log Retention/i)).toBeVisible()
|
||||
await expect(configSettingsPage.hideDeletedVirtualKeysInFiltersSwitch).toBeVisible()
|
||||
})
|
||||
|
||||
test('should toggle hide deleted virtual keys in filters', async ({ configSettingsPage }) => {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.hideDeletedVirtualKeysInFiltersSwitch)
|
||||
|
||||
await configSettingsPage.toggleHideDeletedVirtualKeysInFilters()
|
||||
|
||||
const newState = await configSettingsPage.getSwitchState(configSettingsPage.hideDeletedVirtualKeysInFiltersSwitch)
|
||||
expect(newState).toBe(!initialState)
|
||||
|
||||
const hasChanges = await configSettingsPage.hasPendingChanges()
|
||||
expect(hasChanges).toBe(true)
|
||||
})
|
||||
|
||||
test('should save and persist hide deleted virtual keys in filters toggle', async ({ configSettingsPage }) => {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.hideDeletedVirtualKeysInFiltersSwitch)
|
||||
|
||||
await configSettingsPage.toggleHideDeletedVirtualKeysInFilters()
|
||||
await configSettingsPage.saveSettings()
|
||||
await configSettingsPage.goto('logging')
|
||||
|
||||
const expectedState = !initialState
|
||||
await expect(configSettingsPage.hideDeletedVirtualKeysInFiltersSwitch).toHaveAttribute(
|
||||
'data-state',
|
||||
expectedState ? 'checked' : 'unchecked'
|
||||
)
|
||||
})
|
||||
|
||||
test('should display workspace logging headers textarea when available', async ({ configSettingsPage }) => {
|
||||
const isVisible = await configSettingsPage.workspaceLoggingHeadersTextarea.isVisible().catch(() => false)
|
||||
if (isVisible) {
|
||||
await expect(configSettingsPage.workspaceLoggingHeadersTextarea).toBeVisible()
|
||||
} else {
|
||||
test.skip(true, 'Workspace logging headers not available (depends on log connector)')
|
||||
}
|
||||
})
|
||||
|
||||
test('should toggle content logging when available', async ({ configSettingsPage }) => {
|
||||
// Check if the switch is available (depends on logs being connected)
|
||||
const disableContentLoggingVisible = await configSettingsPage.disableContentLoggingSwitch.isVisible().catch(() => false)
|
||||
|
||||
if (disableContentLoggingVisible) {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.disableContentLoggingSwitch)
|
||||
|
||||
await configSettingsPage.toggleDisableContentLogging()
|
||||
|
||||
const newState = await configSettingsPage.getSwitchState(configSettingsPage.disableContentLoggingSwitch)
|
||||
expect(newState).toBe(!initialState)
|
||||
} else {
|
||||
// Skip if logging not available
|
||||
test.skip()
|
||||
}
|
||||
})
|
||||
|
||||
test('should save and persist content logging toggle when available', async ({ configSettingsPage }) => {
|
||||
// Check if the switch is available (depends on logs being connected)
|
||||
const disableContentLoggingVisible = await configSettingsPage.disableContentLoggingSwitch.isVisible().catch(() => false)
|
||||
|
||||
if (disableContentLoggingVisible) {
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.disableContentLoggingSwitch)
|
||||
|
||||
// Toggle
|
||||
await configSettingsPage.toggleDisableContentLogging()
|
||||
|
||||
// Save
|
||||
await configSettingsPage.saveSettings()
|
||||
|
||||
// Reload the page
|
||||
await configSettingsPage.goto('logging')
|
||||
|
||||
// Verify change persisted
|
||||
const savedState = await configSettingsPage.getSwitchState(configSettingsPage.disableContentLoggingSwitch)
|
||||
expect(savedState).toBe(!initialState)
|
||||
} else {
|
||||
// Skip if logging not available
|
||||
test.skip()
|
||||
}
|
||||
})
|
||||
|
||||
test('should change log retention days', async ({ configSettingsPage }) => {
|
||||
const retentionInput = configSettingsPage.logRetentionDaysInput
|
||||
const isVisible = await retentionInput.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
const originalValue = await retentionInput.inputValue()
|
||||
const newValue = originalValue === '30' ? '60' : '30'
|
||||
|
||||
await retentionInput.clear()
|
||||
await retentionInput.fill(newValue)
|
||||
|
||||
const currentValue = await retentionInput.inputValue()
|
||||
expect(currentValue).toBe(newValue)
|
||||
|
||||
// Verify changes are pending
|
||||
const hasChanges = await configSettingsPage.hasPendingChanges()
|
||||
expect(hasChanges).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
test('should save and persist log retention days', async ({ configSettingsPage }) => {
|
||||
const retentionInput = configSettingsPage.logRetentionDaysInput
|
||||
const isVisible = await retentionInput.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
const originalValue = await retentionInput.inputValue()
|
||||
const newValue = originalValue === '30' ? '60' : '30'
|
||||
|
||||
// Change value
|
||||
await retentionInput.clear()
|
||||
await retentionInput.fill(newValue)
|
||||
|
||||
// Save
|
||||
await configSettingsPage.saveSettings()
|
||||
|
||||
// Reload the page
|
||||
await configSettingsPage.goto('logging')
|
||||
|
||||
// Verify change persisted
|
||||
const savedValue = await retentionInput.inputValue()
|
||||
expect(savedValue).toBe(newValue)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Security Settings', () => {
|
||||
let originalState: ConfigSettingsState
|
||||
|
||||
test.beforeEach(async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('security')
|
||||
// Capture original state for restoration
|
||||
originalState = await configSettingsPage.getCurrentSettings('security')
|
||||
})
|
||||
|
||||
test.afterEach(async ({ configSettingsPage }) => {
|
||||
// Restore original settings
|
||||
if (originalState) {
|
||||
await configSettingsPage.restoreSettings(originalState)
|
||||
}
|
||||
})
|
||||
|
||||
test('should display security settings', async ({ configSettingsPage }) => {
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Security/i })).toBeVisible()
|
||||
await expect(configSettingsPage.saveBtn).toBeVisible()
|
||||
})
|
||||
|
||||
test('should display enforce auth on inference switch', async ({ configSettingsPage }) => {
|
||||
const isVisible = await configSettingsPage.enforceAuthOnInferenceSwitch.isVisible().catch(() => false)
|
||||
if (!isVisible) {
|
||||
test.skip(true, 'Enforce auth on inference not available')
|
||||
return
|
||||
}
|
||||
await expect(configSettingsPage.enforceAuthOnInferenceSwitch).toBeVisible()
|
||||
})
|
||||
|
||||
test('should toggle enforce auth on inference', async ({ configSettingsPage }) => {
|
||||
const isVisible = await configSettingsPage.enforceAuthOnInferenceSwitch.isVisible().catch(() => false)
|
||||
if (!isVisible) {
|
||||
test.skip(true, 'Enforce auth on inference not available')
|
||||
return
|
||||
}
|
||||
const initialState = await configSettingsPage.getSwitchState(configSettingsPage.enforceAuthOnInferenceSwitch)
|
||||
await configSettingsPage.toggleEnforceAuthOnInference()
|
||||
const newState = await configSettingsPage.getSwitchState(configSettingsPage.enforceAuthOnInferenceSwitch)
|
||||
expect(newState).toBe(!initialState)
|
||||
await configSettingsPage.toggleEnforceAuthOnInference()
|
||||
if (await configSettingsPage.hasPendingChanges()) {
|
||||
await configSettingsPage.saveSettings()
|
||||
}
|
||||
})
|
||||
|
||||
test('should display required headers textarea', async ({ configSettingsPage }) => {
|
||||
const isVisible = await configSettingsPage.requiredHeadersTextarea.isVisible().catch(() => false)
|
||||
if (!isVisible) {
|
||||
test.skip(true, 'Required headers control not available')
|
||||
return
|
||||
}
|
||||
await expect(configSettingsPage.requiredHeadersTextarea).toBeVisible()
|
||||
})
|
||||
|
||||
test('should display rate limiting section', async ({ configSettingsPage }) => {
|
||||
const isVisible = await configSettingsPage.isRateLimitingSectionVisible()
|
||||
expect(isVisible).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Performance Tuning Settings', () => {
|
||||
let originalState: ConfigSettingsState
|
||||
|
||||
test.beforeEach(async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('performance-tuning')
|
||||
originalState = await configSettingsPage.getCurrentSettings('performance-tuning')
|
||||
})
|
||||
|
||||
test.afterEach(async ({ configSettingsPage }) => {
|
||||
if (originalState) {
|
||||
await configSettingsPage.restoreSettings(originalState)
|
||||
}
|
||||
})
|
||||
|
||||
test('should display performance tuning settings', async ({ configSettingsPage }) => {
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Performance Tuning/i })).toBeVisible()
|
||||
})
|
||||
|
||||
test('should change worker pool size', async ({ configSettingsPage }) => {
|
||||
const workerPoolInput = configSettingsPage.workerPoolSizeInput
|
||||
const isVisible = await workerPoolInput.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
const originalValue = await workerPoolInput.inputValue()
|
||||
const newValue = parseInt(originalValue) === 100 ? '200' : '100'
|
||||
|
||||
await workerPoolInput.clear()
|
||||
await workerPoolInput.fill(newValue)
|
||||
|
||||
const currentValue = await workerPoolInput.inputValue()
|
||||
expect(currentValue).toBe(newValue)
|
||||
}
|
||||
})
|
||||
|
||||
test('should save and persist worker pool size', async ({ configSettingsPage }) => {
|
||||
const workerPoolInput = configSettingsPage.workerPoolSizeInput
|
||||
const isVisible = await workerPoolInput.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
const originalValue = await workerPoolInput.inputValue()
|
||||
const newValue = parseInt(originalValue) === 100 ? '200' : '100'
|
||||
|
||||
// Change value
|
||||
await workerPoolInput.clear()
|
||||
await workerPoolInput.fill(newValue)
|
||||
|
||||
// Save
|
||||
await configSettingsPage.saveSettings()
|
||||
|
||||
// Reload the page
|
||||
await configSettingsPage.goto('performance-tuning')
|
||||
|
||||
// Verify change persisted
|
||||
const savedValue = await workerPoolInput.inputValue()
|
||||
expect(savedValue).toBe(newValue)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Pricing Config Settings', () => {
|
||||
let originalState: ConfigSettingsState
|
||||
|
||||
test.beforeEach(async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('pricing-config')
|
||||
originalState = await configSettingsPage.getCurrentSettings('pricing-config')
|
||||
})
|
||||
|
||||
test.afterEach(async ({ configSettingsPage }) => {
|
||||
if (originalState) {
|
||||
await configSettingsPage.restoreSettings(originalState)
|
||||
}
|
||||
})
|
||||
|
||||
test('should display pricing config settings', async ({ configSettingsPage }) => {
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Pricing/i })).toBeVisible()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Caching Settings', () => {
|
||||
let originalState: ConfigSettingsState
|
||||
|
||||
test.beforeEach(async ({ configSettingsPage }) => {
|
||||
await configSettingsPage.goto('caching')
|
||||
originalState = await configSettingsPage.getCurrentSettings('caching')
|
||||
})
|
||||
|
||||
test.afterEach(async ({ configSettingsPage }) => {
|
||||
if (originalState) {
|
||||
await configSettingsPage.restoreSettings(originalState)
|
||||
}
|
||||
})
|
||||
|
||||
test('should display caching settings', async ({ configSettingsPage }) => {
|
||||
await expect(configSettingsPage.page.getByRole('heading', { name: /Caching/i })).toBeVisible()
|
||||
})
|
||||
})
|
||||
})
|
||||
343
tests/e2e/features/config/pages/config-settings.page.ts
Normal file
343
tests/e2e/features/config/pages/config-settings.page.ts
Normal file
@@ -0,0 +1,343 @@
|
||||
import { Locator, Page } from '@playwright/test'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
/**
|
||||
* Config settings state interface
|
||||
*/
|
||||
export interface ConfigSettingsState {
|
||||
toggleStates: Record<string, boolean>
|
||||
inputValues: Record<string, string>
|
||||
configPath: string
|
||||
}
|
||||
|
||||
export class ConfigSettingsPage extends BasePage {
|
||||
readonly saveBtn: Locator
|
||||
|
||||
// Client Settings
|
||||
readonly dropExcessRequestsSwitch: Locator
|
||||
readonly enableLiteLLMFallbacksSwitch: Locator
|
||||
readonly disableDBPingsSwitch: Locator
|
||||
readonly asyncJobResultTtlInput: Locator
|
||||
|
||||
// Logging Settings
|
||||
readonly enableLoggingSwitch: Locator
|
||||
readonly disableContentLoggingSwitch: Locator
|
||||
readonly hideDeletedVirtualKeysInFiltersSwitch: Locator
|
||||
readonly logRetentionDaysInput: Locator
|
||||
readonly workspaceLoggingHeadersTextarea: Locator
|
||||
|
||||
// Security Settings
|
||||
readonly rateLimitingSection: Locator
|
||||
readonly enforceAuthOnInferenceSwitch: Locator
|
||||
readonly requiredHeadersTextarea: Locator
|
||||
|
||||
// Performance Tuning Settings
|
||||
readonly workerPoolSizeInput: Locator
|
||||
readonly maxRequestBodySizeInput: Locator
|
||||
|
||||
// Observability Settings
|
||||
readonly observabilityToggles: Locator
|
||||
|
||||
// Pricing Config
|
||||
readonly pricingConfigView: Locator
|
||||
readonly pricingDatasheetUrlInput: Locator
|
||||
readonly pricingForceSyncBtn: Locator
|
||||
readonly pricingSaveBtn: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
this.saveBtn = page.getByRole('button', { name: /Save/i })
|
||||
|
||||
// Client Settings locators
|
||||
this.dropExcessRequestsSwitch = page.locator('#drop-excess-requests')
|
||||
this.enableLiteLLMFallbacksSwitch = page.locator('#enable-litellm-fallbacks')
|
||||
this.disableDBPingsSwitch = page.locator('#disable-db-pings-in-health')
|
||||
this.asyncJobResultTtlInput = page.getByTestId('client-settings-async-job-result-ttl-input')
|
||||
|
||||
// Logging Settings locators
|
||||
this.enableLoggingSwitch = page.locator('#enable-logging')
|
||||
this.disableContentLoggingSwitch = page.locator('#disable-content-logging')
|
||||
this.hideDeletedVirtualKeysInFiltersSwitch = page.getByTestId('hide-deleted-virtual-keys-in-filters-switch')
|
||||
this.logRetentionDaysInput = page.getByLabel(/Log Retention Days/i).or(
|
||||
page.locator('#log-n-days')
|
||||
)
|
||||
this.workspaceLoggingHeadersTextarea = page.getByTestId('workspace-logging-headers-textarea')
|
||||
|
||||
// Security Settings locators
|
||||
this.rateLimitingSection = page.locator('text=Rate Limiting').locator('..')
|
||||
this.enforceAuthOnInferenceSwitch = page.getByTestId('enforce-auth-on-inference-switch')
|
||||
this.requiredHeadersTextarea = page.getByTestId('required-headers-textarea')
|
||||
|
||||
// Performance Tuning locators
|
||||
this.workerPoolSizeInput = page.getByLabel(/Worker Pool Size/i)
|
||||
this.maxRequestBodySizeInput = page.getByLabel(/Max Request Body Size/i)
|
||||
|
||||
// Observability locators
|
||||
this.observabilityToggles = page.locator('button[role="switch"]')
|
||||
|
||||
// Pricing Config locators
|
||||
this.pricingConfigView = page.getByTestId('pricing-config-view')
|
||||
this.pricingDatasheetUrlInput = page.getByTestId('pricing-datasheet-url-input')
|
||||
this.pricingForceSyncBtn = page.getByTestId('pricing-force-sync-btn')
|
||||
this.pricingSaveBtn = page.getByTestId('pricing-save-btn')
|
||||
}
|
||||
|
||||
async goto(path: string): Promise<void> {
|
||||
await this.page.goto(`/workspace/config/${path}`)
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
async saveSettings(): Promise<void> {
|
||||
await this.saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if save button is enabled (changes pending)
|
||||
*/
|
||||
async hasPendingChanges(): Promise<boolean> {
|
||||
const isDisabled = await this.saveBtn.isDisabled()
|
||||
return !isDisabled
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle a switch element
|
||||
*/
|
||||
async toggleSwitch(switchLocator: Locator): Promise<void> {
|
||||
await switchLocator.click()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the state of a switch
|
||||
*/
|
||||
async getSwitchState(switchLocator: Locator): Promise<boolean> {
|
||||
const state = await switchLocator.getAttribute('data-state')
|
||||
return state === 'checked'
|
||||
}
|
||||
|
||||
/**
|
||||
* Set input value
|
||||
*/
|
||||
async setInputValue(inputLocator: Locator, value: string): Promise<void> {
|
||||
await inputLocator.clear()
|
||||
await inputLocator.fill(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input value
|
||||
*/
|
||||
async getInputValue(inputLocator: Locator): Promise<string> {
|
||||
return await inputLocator.inputValue()
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture current settings state for a config page
|
||||
*/
|
||||
async getCurrentSettings(configPath: string): Promise<ConfigSettingsState> {
|
||||
const state: ConfigSettingsState = {
|
||||
toggleStates: {},
|
||||
inputValues: {},
|
||||
configPath,
|
||||
}
|
||||
|
||||
// Get all switch states on the page
|
||||
const switches = this.page.locator('button[role="switch"]')
|
||||
const switchCount = await switches.count()
|
||||
|
||||
for (let i = 0; i < switchCount; i++) {
|
||||
const switchEl = switches.nth(i)
|
||||
const elId = await switchEl.getAttribute('id')
|
||||
if (!elId) {
|
||||
console.warn(`Switch at index ${i} has no id attribute — using positional fallback "switch-${i}" which may mismatch on restore`)
|
||||
}
|
||||
const id = elId || `switch-${i}`
|
||||
const isChecked = await switchEl.getAttribute('data-state') === 'checked'
|
||||
state.toggleStates[id] = isChecked
|
||||
}
|
||||
|
||||
// Get all number input values on the page
|
||||
const numberInputs = this.page.locator('input[type="number"]')
|
||||
const inputCount = await numberInputs.count()
|
||||
|
||||
for (let i = 0; i < inputCount; i++) {
|
||||
const input = numberInputs.nth(i)
|
||||
const elId = await input.getAttribute('id')
|
||||
if (!elId) {
|
||||
console.warn(`Input at index ${i} has no id attribute — using positional fallback "input-${i}" which may mismatch on restore`)
|
||||
}
|
||||
const id = elId || `input-${i}`
|
||||
const value = await input.inputValue()
|
||||
state.inputValues[id] = value
|
||||
}
|
||||
|
||||
return state
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore settings to a previous state
|
||||
*/
|
||||
async restoreSettings(state: ConfigSettingsState): Promise<void> {
|
||||
// Navigate to the config page if not already there
|
||||
const currentUrl = this.page.url()
|
||||
if (!currentUrl.includes(state.configPath)) {
|
||||
await this.goto(state.configPath)
|
||||
}
|
||||
|
||||
let hasChanges = false
|
||||
|
||||
// Restore switch states
|
||||
const switches = this.page.locator('button[role="switch"]')
|
||||
const switchCount = await switches.count()
|
||||
|
||||
for (let i = 0; i < switchCount; i++) {
|
||||
const switchEl = switches.nth(i)
|
||||
const elId = await switchEl.getAttribute('id')
|
||||
if (!elId) {
|
||||
console.warn(`Switch at index ${i} has no id attribute — using positional fallback "switch-${i}" which may mismatch on restore`)
|
||||
}
|
||||
const id = elId || `switch-${i}`
|
||||
|
||||
if (state.toggleStates[id] !== undefined) {
|
||||
const currentState = await switchEl.getAttribute('data-state') === 'checked'
|
||||
if (currentState !== state.toggleStates[id]) {
|
||||
await switchEl.click()
|
||||
hasChanges = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Restore input values
|
||||
const numberInputs = this.page.locator('input[type="number"]')
|
||||
const inputCount = await numberInputs.count()
|
||||
|
||||
for (let i = 0; i < inputCount; i++) {
|
||||
const input = numberInputs.nth(i)
|
||||
const elId = await input.getAttribute('id')
|
||||
if (!elId) {
|
||||
console.warn(`Input at index ${i} has no id attribute — using positional fallback "input-${i}" which may mismatch on restore`)
|
||||
}
|
||||
const id = elId || `input-${i}`
|
||||
|
||||
if (state.inputValues[id] !== undefined) {
|
||||
const currentValue = await input.inputValue()
|
||||
if (currentValue !== state.inputValues[id]) {
|
||||
await input.clear()
|
||||
await input.fill(state.inputValues[id])
|
||||
hasChanges = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save if changes were made
|
||||
if (hasChanges) {
|
||||
const canSave = await this.hasPendingChanges()
|
||||
if (canSave) {
|
||||
await this.saveSettings()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// === Client Settings Methods ===
|
||||
|
||||
async toggleDropExcessRequests(): Promise<void> {
|
||||
await this.dropExcessRequestsSwitch.click()
|
||||
}
|
||||
|
||||
async toggleLiteLLMFallbacks(): Promise<void> {
|
||||
await this.enableLiteLLMFallbacksSwitch.click()
|
||||
}
|
||||
|
||||
async toggleDisableDBPings(): Promise<void> {
|
||||
await this.disableDBPingsSwitch.click()
|
||||
}
|
||||
|
||||
// === Logging Settings Methods ===
|
||||
|
||||
async toggleEnableLogging(): Promise<void> {
|
||||
await this.enableLoggingSwitch.click()
|
||||
}
|
||||
|
||||
async toggleDisableContentLogging(): Promise<void> {
|
||||
await this.disableContentLoggingSwitch.click()
|
||||
}
|
||||
|
||||
async toggleHideDeletedVirtualKeysInFilters(): Promise<void> {
|
||||
await this.hideDeletedVirtualKeysInFiltersSwitch.click()
|
||||
}
|
||||
|
||||
async setLogRetentionDays(days: number): Promise<void> {
|
||||
const input = this.page.locator('input[type="number"]').first()
|
||||
await input.clear()
|
||||
await input.fill(days.toString())
|
||||
}
|
||||
|
||||
async getLogRetentionDays(): Promise<number> {
|
||||
const input = this.page.locator('input[type="number"]').first()
|
||||
const value = await input.inputValue()
|
||||
return parseInt(value, 10)
|
||||
}
|
||||
|
||||
// === Security Settings Methods ===
|
||||
|
||||
async isRateLimitingSectionVisible(): Promise<boolean> {
|
||||
return await this.page.getByText(/Rate Limiting/i).isVisible()
|
||||
}
|
||||
|
||||
async toggleEnforceAuthOnInference(): Promise<void> {
|
||||
await this.enforceAuthOnInferenceSwitch.click()
|
||||
}
|
||||
|
||||
async setRequiredHeaders(value: string): Promise<void> {
|
||||
await this.requiredHeadersTextarea.clear()
|
||||
await this.requiredHeadersTextarea.fill(value)
|
||||
}
|
||||
|
||||
async setWorkspaceLoggingHeaders(value: string): Promise<void> {
|
||||
await this.workspaceLoggingHeadersTextarea.clear()
|
||||
await this.workspaceLoggingHeadersTextarea.fill(value)
|
||||
}
|
||||
|
||||
async setAsyncJobResultTtl(value: string): Promise<void> {
|
||||
await this.asyncJobResultTtlInput.clear()
|
||||
await this.asyncJobResultTtlInput.fill(value)
|
||||
}
|
||||
|
||||
// === Observability Settings Methods ===
|
||||
|
||||
async getObservabilityConnectors(): Promise<string[]> {
|
||||
const connectorHeadings = this.page.locator('h3, h4').filter({ hasText: /Datadog|New Relic|OTel|OpenTelemetry|Maxim/i })
|
||||
const count = await connectorHeadings.count()
|
||||
const connectors: string[] = []
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
const text = await connectorHeadings.nth(i).textContent()
|
||||
if (text) connectors.push(text)
|
||||
}
|
||||
|
||||
return connectors
|
||||
}
|
||||
|
||||
async toggleObservabilityConnector(connectorName: string): Promise<void> {
|
||||
const connectorSection = this.page.locator('div').filter({ hasText: new RegExp(connectorName, 'i') }).first()
|
||||
const toggleSwitch = connectorSection.locator('button[role="switch"]').first()
|
||||
await toggleSwitch.click()
|
||||
}
|
||||
|
||||
// === Pricing Config Methods ===
|
||||
|
||||
async setPricingDatasheetUrl(url: string): Promise<void> {
|
||||
await this.pricingDatasheetUrlInput.clear()
|
||||
await this.pricingDatasheetUrlInput.fill(url)
|
||||
}
|
||||
|
||||
async triggerForceSync(): Promise<void> {
|
||||
await this.pricingForceSyncBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
}
|
||||
|
||||
async savePricingConfig(): Promise<void> {
|
||||
await this.pricingSaveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
}
|
||||
}
|
||||
385
tests/e2e/features/dashboard/dashboard.spec.ts
Normal file
385
tests/e2e/features/dashboard/dashboard.spec.ts
Normal file
@@ -0,0 +1,385 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
import { waitForNetworkIdle } from '../../core/utils/test-helpers'
|
||||
import { DashboardPage } from './pages/dashboard.page'
|
||||
|
||||
test.describe('Dashboard', () => {
|
||||
test.beforeEach(async ({ dashboardPage }) => {
|
||||
await dashboardPage.goto()
|
||||
})
|
||||
|
||||
test.describe('Dashboard Display', () => {
|
||||
test('should display dashboard page', async ({ dashboardPage }) => {
|
||||
await expect(dashboardPage.pageTitle).toBeVisible()
|
||||
})
|
||||
|
||||
test('should display all chart cards', async ({ dashboardPage }) => {
|
||||
// Check that all four main charts are visible
|
||||
await expect(dashboardPage.logVolumeChart).toBeVisible()
|
||||
await expect(dashboardPage.tokenUsageChart).toBeVisible()
|
||||
await expect(dashboardPage.costChart).toBeVisible()
|
||||
await expect(dashboardPage.modelUsageChart).toBeVisible()
|
||||
})
|
||||
|
||||
test('should display date time picker', async ({ dashboardPage }) => {
|
||||
// Date picker should be visible (may be a button with date text)
|
||||
const datePicker = dashboardPage.page.locator('button').filter({ hasText: /Last/i }).or(
|
||||
dashboardPage.page.locator('[data-testid="dashboard-date-picker"]')
|
||||
)
|
||||
await expect(datePicker.first()).toBeVisible()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Time Period Selection', () => {
|
||||
test('should filter by time period (full flow)', async ({ dashboardPage }) => {
|
||||
// Time period control must exist and be visible (no skip)
|
||||
const trigger = dashboardPage.getDatePickerTrigger()
|
||||
await expect(trigger).toBeVisible({ timeout: 10000 })
|
||||
|
||||
// Let initial chart load finish so the refetch we wait for is the one from the period change
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Wait for the chart data request that fires when we change the period (proves filter is applied)
|
||||
const responsePromise = dashboardPage.page.waitForResponse(
|
||||
(res) => res.url().includes('/logs/histogram') && res.status() === 200,
|
||||
{ timeout: 15000 }
|
||||
)
|
||||
|
||||
await dashboardPage.selectTimePeriod('1h')
|
||||
|
||||
// UI: trigger shows the selected period
|
||||
const label = await dashboardPage.getSelectedPeriodLabel()
|
||||
expect(label).toContain('Last hour')
|
||||
|
||||
// URL: selection is reflected in query state
|
||||
const url = dashboardPage.page.url()
|
||||
expect(url).toMatch(/period=1h|start_time=\d+&end_time=\d+/)
|
||||
|
||||
// Data: dashboard refetched with the new range
|
||||
await responsePromise
|
||||
})
|
||||
|
||||
test('should change time period to last hour', async ({ dashboardPage }) => {
|
||||
await dashboardPage.selectTimePeriod('1h')
|
||||
|
||||
const label = await dashboardPage.getSelectedPeriodLabel()
|
||||
expect(label).toContain('Last hour')
|
||||
|
||||
const url = dashboardPage.page.url()
|
||||
expect(url).toMatch(/period=1h|start_time=\d+&end_time=\d+/)
|
||||
})
|
||||
|
||||
test('should change time period to last 7 days', async ({ dashboardPage }) => {
|
||||
await dashboardPage.selectTimePeriod('7d')
|
||||
|
||||
const label = await dashboardPage.getSelectedPeriodLabel()
|
||||
expect(label).toContain('Last 7 days')
|
||||
|
||||
const url = dashboardPage.page.url()
|
||||
expect(url).toMatch(/period=7d|start_time=\d+&end_time=\d+/)
|
||||
})
|
||||
|
||||
test('should change time period to last 30 days', async ({ dashboardPage }) => {
|
||||
await dashboardPage.selectTimePeriod('30d')
|
||||
|
||||
const label = await dashboardPage.getSelectedPeriodLabel()
|
||||
expect(label).toContain('Last 30 days')
|
||||
|
||||
const url = dashboardPage.page.url()
|
||||
expect(url).toMatch(/period=30d|start_time=\d+&end_time=\d+/)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Chart Type Toggling', () => {
|
||||
test('should toggle volume chart type', async ({ dashboardPage }) => {
|
||||
// Get initial toggle state from DOM
|
||||
const initialToggle = dashboardPage.volumeChartToggle
|
||||
const initialState = await dashboardPage.getChartToggleState(initialToggle)
|
||||
|
||||
// Toggle the chart (method handles waiting internally)
|
||||
await dashboardPage.toggleVolumeChartType()
|
||||
|
||||
// Get new toggle state
|
||||
const newToggle = dashboardPage.volumeChartToggle
|
||||
const newState = await dashboardPage.getChartToggleState(newToggle)
|
||||
|
||||
// Chart type should have changed (state should be different)
|
||||
expect(newState).not.toBe(initialState)
|
||||
})
|
||||
|
||||
test('should toggle token chart type', async ({ dashboardPage }) => {
|
||||
const initialToggle = dashboardPage.tokenChartToggle
|
||||
const initialState = await dashboardPage.getChartToggleState(initialToggle)
|
||||
|
||||
await dashboardPage.toggleTokenChartType()
|
||||
|
||||
const newToggle = dashboardPage.tokenChartToggle
|
||||
const newState = await dashboardPage.getChartToggleState(newToggle)
|
||||
|
||||
expect(newState).not.toBe(initialState)
|
||||
})
|
||||
|
||||
test('should toggle cost chart type', async ({ dashboardPage }) => {
|
||||
const initialToggle = dashboardPage.costChartToggle
|
||||
const initialState = await dashboardPage.getChartToggleState(initialToggle)
|
||||
|
||||
await dashboardPage.toggleCostChartType()
|
||||
|
||||
const newToggle = dashboardPage.costChartToggle
|
||||
const newState = await dashboardPage.getChartToggleState(newToggle)
|
||||
|
||||
expect(newState).not.toBe(initialState)
|
||||
})
|
||||
|
||||
test('should toggle model chart type', async ({ dashboardPage }) => {
|
||||
const initialToggle = dashboardPage.modelChartToggle
|
||||
const initialState = await dashboardPage.getChartToggleState(initialToggle)
|
||||
|
||||
await dashboardPage.toggleModelChartType()
|
||||
|
||||
const newToggle = dashboardPage.modelChartToggle
|
||||
const newState = await dashboardPage.getChartToggleState(newToggle)
|
||||
|
||||
expect(newState).not.toBe(initialState)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Model Filtering', () => {
|
||||
test('should filter cost chart by model', async ({ dashboardPage }) => {
|
||||
// Wait for charts to fully load
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Try to filter by a specific model if available
|
||||
const costModelFilter = dashboardPage.costModelFilter
|
||||
const isVisible = await costModelFilter.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
await dashboardPage.filterCostChartByModel('all')
|
||||
|
||||
// Check that filter value is "All Models"
|
||||
const newSelected = await dashboardPage.getSelectedModel(costModelFilter)
|
||||
expect(newSelected).toContain('All Models')
|
||||
}
|
||||
})
|
||||
|
||||
test('should filter usage chart by model', async ({ dashboardPage }) => {
|
||||
// Wait for charts to fully load
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
const usageModelFilter = dashboardPage.usageModelFilter
|
||||
const isVisible = await usageModelFilter.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
await dashboardPage.filterUsageChartByModel('all')
|
||||
|
||||
// Check that filter value is "All Models"
|
||||
const newSelected = await dashboardPage.getSelectedModel(usageModelFilter)
|
||||
expect(newSelected).toContain('All Models')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Chart Loading States', () => {
|
||||
test('should show loading state initially', async ({ dashboardPage }) => {
|
||||
// Navigate to a fresh dashboard
|
||||
await dashboardPage.page.reload()
|
||||
await dashboardPage.waitForPageLoad()
|
||||
|
||||
// Charts may show loading state briefly
|
||||
// This test verifies the page loads without errors
|
||||
await expect(dashboardPage.pageTitle).toBeVisible({ timeout: 10000 })
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('URL State Management', () => {
|
||||
test('should preserve chart state in URL', async ({ dashboardPage }) => {
|
||||
// Change some settings
|
||||
await dashboardPage.selectTimePeriod('7d')
|
||||
await dashboardPage.toggleVolumeChartType()
|
||||
|
||||
// Check URL for period (time period should still be in URL)
|
||||
const url = dashboardPage.page.url()
|
||||
expect(url).toContain('period=7d')
|
||||
|
||||
// Check DOM state for chart toggle (may or may not be in URL)
|
||||
const toggleState = await dashboardPage.getChartToggleState(dashboardPage.volumeChartToggle)
|
||||
expect(toggleState).toBeTruthy()
|
||||
})
|
||||
|
||||
test('should restore state from URL on page load', async ({ dashboardPage }) => {
|
||||
// Set URL with specific state
|
||||
await dashboardPage.page.goto('/workspace/dashboard?period=7d&volume_chart=line')
|
||||
await waitForNetworkIdle(dashboardPage.page)
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Verify page loaded with correct state from URL
|
||||
const url = dashboardPage.page.url()
|
||||
expect(url).toContain('period=7d')
|
||||
// volume_chart=line was in the URL - verify exact value persisted
|
||||
expect(url).toContain('volume_chart=line')
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Chart Data Validation', () => {
|
||||
test('should render chart elements after data loads', async ({ dashboardPage }) => {
|
||||
// Wait for charts to load
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Check that each chart card has a canvas or chart surface SVG (recharts-surface = actual chart, not icons)
|
||||
const volumeChartContent = dashboardPage.logVolumeChart.locator('canvas, svg.recharts-surface')
|
||||
const tokenChartContent = dashboardPage.tokenUsageChart.locator('canvas, svg.recharts-surface')
|
||||
const costChartContent = dashboardPage.costChart.locator('canvas, svg.recharts-surface')
|
||||
const modelChartContent = dashboardPage.modelUsageChart.locator('canvas, svg.recharts-surface')
|
||||
|
||||
// Each chart card should have canvas or SVG content (chart library renders into these)
|
||||
const volumeCount = await volumeChartContent.count()
|
||||
const tokenCount = await tokenChartContent.count()
|
||||
const costCount = await costChartContent.count()
|
||||
const modelCount = await modelChartContent.count()
|
||||
|
||||
// All four chart cards should have rendered content (count > 0)
|
||||
expect(volumeCount).toBeGreaterThan(0)
|
||||
expect(tokenCount).toBeGreaterThan(0)
|
||||
expect(costCount).toBeGreaterThan(0)
|
||||
expect(modelCount).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
test('should show chart legends', async ({ dashboardPage }) => {
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Check that chart actions (legends/toggles) are visible
|
||||
const volumeActions = dashboardPage.page.locator('[data-testid="chart-log-volume-actions"]')
|
||||
const tokenActions = dashboardPage.page.locator('[data-testid="chart-token-usage-actions"]')
|
||||
|
||||
// Actions should be visible (they contain legends and toggles)
|
||||
await expect(volumeActions).toBeVisible()
|
||||
await expect(tokenActions).toBeVisible()
|
||||
})
|
||||
|
||||
test('should not show loading skeletons after data loads', async ({ dashboardPage }) => {
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Check that no skeletons are visible (data has loaded)
|
||||
const skeletons = dashboardPage.page.locator('[data-testid="skeleton"]')
|
||||
const skeletonCount = await skeletons.count()
|
||||
|
||||
expect(skeletonCount).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Chart Interactions', () => {
|
||||
test('should toggle between bar and line chart for volume', async ({ dashboardPage }) => {
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Get initial toggle state
|
||||
const initialState = await dashboardPage.getChartToggleState(dashboardPage.volumeChartToggle)
|
||||
|
||||
// Toggle volume chart type
|
||||
await dashboardPage.toggleVolumeChartType()
|
||||
|
||||
// DOM state should change (chart type toggles are in DOM, not URL)
|
||||
const newState = await dashboardPage.getChartToggleState(dashboardPage.volumeChartToggle)
|
||||
expect(newState).not.toBe(initialState)
|
||||
})
|
||||
|
||||
test('should update chart when time period changes', async ({ dashboardPage }) => {
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
const initialUrl = dashboardPage.page.url()
|
||||
|
||||
await dashboardPage.selectTimePeriod('1h')
|
||||
|
||||
// Trigger should show new period (filter was applied)
|
||||
const label = await dashboardPage.getSelectedPeriodLabel()
|
||||
expect(label).toContain('Last hour')
|
||||
|
||||
const newUrl = dashboardPage.page.url()
|
||||
expect(newUrl).toMatch(/period=1h|start_time=\d+&end_time=\d+/)
|
||||
expect(newUrl).not.toBe(initialUrl)
|
||||
})
|
||||
|
||||
test('should sync model filter between cost and usage charts', async ({ dashboardPage }) => {
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Check if model filters are visible
|
||||
const costFilterVisible = await dashboardPage.costModelFilter.isVisible().catch(() => false)
|
||||
const usageFilterVisible = await dashboardPage.usageModelFilter.isVisible().catch(() => false)
|
||||
|
||||
if (costFilterVisible && usageFilterVisible) {
|
||||
// Filter cost chart
|
||||
await dashboardPage.filterCostChartByModel('all')
|
||||
|
||||
// Verify filter was applied (check DOM state, not URL)
|
||||
const selectedModel = await dashboardPage.getSelectedModel(dashboardPage.costModelFilter)
|
||||
expect(selectedModel).toContain('All Models')
|
||||
}
|
||||
})
|
||||
|
||||
test('should display correct time period labels', async ({ dashboardPage }) => {
|
||||
const periods: Array<'1h' | '6h' | '24h' | '7d' | '30d'> = ['1h', '6h', '24h', '7d', '30d']
|
||||
|
||||
for (const period of periods) {
|
||||
await dashboardPage.selectTimePeriod(period)
|
||||
// Assert the date picker trigger shows the selected period (actual selected value)
|
||||
const label = await dashboardPage.getSelectedPeriodLabel()
|
||||
const expected = DashboardPage.PERIOD_LABELS[period]
|
||||
expect(label).toContain(expected)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Error States', () => {
|
||||
test('should handle empty data gracefully', async ({ dashboardPage }) => {
|
||||
// Navigate with very short time range that may have no data
|
||||
await dashboardPage.page.goto('/workspace/dashboard?period=1h')
|
||||
await waitForNetworkIdle(dashboardPage.page)
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Page should still render without errors
|
||||
await expect(dashboardPage.pageTitle).toBeVisible()
|
||||
|
||||
// All chart containers should still be visible
|
||||
await expect(dashboardPage.logVolumeChart).toBeVisible()
|
||||
await expect(dashboardPage.tokenUsageChart).toBeVisible()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Custom Date Range', () => {
|
||||
test('should open custom date range picker', async ({ dashboardPage }) => {
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Look for date picker button
|
||||
const datePicker = dashboardPage.page.getByRole('button').filter({ hasText: /Last|Custom/i }).first()
|
||||
const isVisible = await datePicker.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
await datePicker.click()
|
||||
|
||||
// Should see date range options or calendar
|
||||
const calendarVisible = await dashboardPage.page.locator('[role="dialog"], [role="listbox"]').isVisible().catch(() => false)
|
||||
const optionsVisible = await dashboardPage.page.getByRole('option').first().isVisible().catch(() => false)
|
||||
|
||||
expect(calendarVisible || optionsVisible).toBe(true)
|
||||
|
||||
// Close the picker
|
||||
await dashboardPage.page.keyboard.press('Escape')
|
||||
}
|
||||
})
|
||||
|
||||
test('should handle empty data for custom range', async ({ dashboardPage }) => {
|
||||
// Set a custom time range that likely has no data
|
||||
await dashboardPage.page.goto('/workspace/dashboard?period=1h')
|
||||
await waitForNetworkIdle(dashboardPage.page)
|
||||
await dashboardPage.waitForChartsToLoad()
|
||||
|
||||
// Charts should still be visible even with no data
|
||||
await expect(dashboardPage.logVolumeChart).toBeVisible()
|
||||
await expect(dashboardPage.costChart).toBeVisible()
|
||||
|
||||
// Page should not show error alerts (not matching chart legend "Error")
|
||||
const errorAlert = dashboardPage.page.locator('[role="alert"][data-variant="destructive"], .text-destructive, [data-sonner-toast][data-type="error"]')
|
||||
const hasErrorAlert = await errorAlert.count() > 0
|
||||
expect(hasErrorAlert).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
349
tests/e2e/features/dashboard/pages/dashboard.page.ts
Normal file
349
tests/e2e/features/dashboard/pages/dashboard.page.ts
Normal file
@@ -0,0 +1,349 @@
|
||||
import { Locator, Page, expect } from '@playwright/test'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
/**
|
||||
* Page object for the Dashboard page
|
||||
*/
|
||||
export class DashboardPage extends BasePage {
|
||||
// Main elements
|
||||
readonly pageTitle: Locator
|
||||
readonly dateTimePicker: Locator
|
||||
|
||||
// Chart cards
|
||||
readonly logVolumeChart: Locator
|
||||
readonly tokenUsageChart: Locator
|
||||
readonly costChart: Locator
|
||||
readonly modelUsageChart: Locator
|
||||
|
||||
// Chart type toggles
|
||||
readonly volumeChartToggle: Locator
|
||||
readonly tokenChartToggle: Locator
|
||||
readonly costChartToggle: Locator
|
||||
readonly modelChartToggle: Locator
|
||||
|
||||
// Model filters
|
||||
readonly costModelFilter: Locator
|
||||
readonly usageModelFilter: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
|
||||
// Main elements
|
||||
this.pageTitle = page.getByRole('heading', { name: /Dashboard/i })
|
||||
this.dateTimePicker = page.locator('[data-testid="dashboard-date-picker"]')
|
||||
|
||||
// Chart cards - using data-testid for robust selectors
|
||||
this.logVolumeChart = page.locator('[data-testid="chart-log-volume"]')
|
||||
this.tokenUsageChart = page.locator('[data-testid="chart-token-usage"]')
|
||||
this.costChart = page.locator('[data-testid="chart-cost-total"]')
|
||||
this.modelUsageChart = page.locator('[data-testid="chart-model-usage"]')
|
||||
|
||||
// Chart type toggles - using data-testid with actions suffix
|
||||
// Volume and token charts have only ChartTypeToggle in the actions bar
|
||||
this.volumeChartToggle = page.locator('[data-testid="chart-log-volume-actions"]').locator('button').filter({ has: page.locator('svg') })
|
||||
this.tokenChartToggle = page.locator('[data-testid="chart-token-usage-actions"]').locator('button').filter({ has: page.locator('svg') })
|
||||
// Cost and model charts have model filter + ChartTypeToggle; scope to ChartTypeToggle buttons only so getChartToggleState reads the right element
|
||||
this.costChartToggle = page.locator('[data-testid="chart-cost-total-actions"]').locator('> div > div').last().locator('button')
|
||||
this.modelChartToggle = page.locator('[data-testid="chart-model-usage-actions"]').locator('> div > div').last().locator('button')
|
||||
|
||||
// Model filters - select trigger inside each chart's actions area (opens dropdown; Radix uses role=combobox or data-slot=select-trigger)
|
||||
this.costModelFilter = page.locator('[data-testid="chart-cost-total-actions"]').locator('[role="combobox"], [data-slot="select-trigger"]').first()
|
||||
this.usageModelFilter = page.locator('[data-testid="chart-model-usage-actions"]').locator('[role="combobox"], [data-slot="select-trigger"]').first()
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to the dashboard page
|
||||
*/
|
||||
async goto(): Promise<void> {
|
||||
await this.page.goto('/workspace/dashboard')
|
||||
await waitForNetworkIdle(this.page)
|
||||
// Wait for charts to load
|
||||
await this.waitForChartsToLoad()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if dashboard is loaded
|
||||
*/
|
||||
async isLoaded(): Promise<boolean> {
|
||||
try {
|
||||
await expect(this.pageTitle).toBeVisible({ timeout: 5000 })
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close any open popups (date picker, dropdowns, etc.)
|
||||
*/
|
||||
async closePopups(): Promise<void> {
|
||||
// Check for open date picker dialog and close it
|
||||
const datePickerDialog = this.page.locator('[data-radix-popper-content-wrapper]')
|
||||
if (await datePickerDialog.isVisible().catch(() => false)) {
|
||||
await this.page.keyboard.press('Escape')
|
||||
await datePickerDialog.waitFor({ state: 'hidden', timeout: 2000 }).catch(() => {})
|
||||
}
|
||||
// Check for open listbox and close it
|
||||
const listbox = this.page.locator('[role="listbox"]')
|
||||
if (await listbox.isVisible().catch(() => false)) {
|
||||
await this.page.keyboard.press('Escape')
|
||||
await listbox.waitFor({ state: 'hidden', timeout: 2000 }).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
/** Period label map used by the date picker (must match UI) */
|
||||
static readonly PERIOD_LABELS: Record<string, string> = {
|
||||
'1h': 'Last hour',
|
||||
'6h': 'Last 6 hours',
|
||||
'24h': 'Last 24 hours',
|
||||
'7d': 'Last 7 days',
|
||||
'30d': 'Last 30 days',
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the date picker trigger button (the button that shows the current period and opens the popover).
|
||||
* Identified by having the calendar icon so we don't match preset buttons inside the popover.
|
||||
*/
|
||||
getDatePickerTrigger(): Locator {
|
||||
return this.page.locator('button').filter({ has: this.page.locator('svg') }).filter({ hasText: /Last|Pick/i }).first()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the currently displayed period label from the date picker trigger (what the user sees as selected).
|
||||
*/
|
||||
async getSelectedPeriodLabel(): Promise<string> {
|
||||
const trigger = this.getDatePickerTrigger()
|
||||
await trigger.waitFor({ state: 'visible', timeout: 5000 })
|
||||
const text = await trigger.textContent()
|
||||
return (text ?? '').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Select a predefined time period
|
||||
*/
|
||||
async selectTimePeriod(period: '1h' | '6h' | '24h' | '7d' | '30d'): Promise<void> {
|
||||
await this.closePopups()
|
||||
|
||||
const trigger = this.getDatePickerTrigger()
|
||||
await trigger.click()
|
||||
|
||||
// Wait for dialog to open
|
||||
await this.page.waitForSelector('[data-radix-popper-content-wrapper]', { timeout: 5000 }).catch(() => {})
|
||||
|
||||
const label = DashboardPage.PERIOD_LABELS[period]
|
||||
await this.page.getByRole('button', { name: label }).click()
|
||||
|
||||
// Wait for dialog to close
|
||||
await this.page.locator('[data-radix-popper-content-wrapper]').waitFor({ state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the inactive toggle button from a set of buttons (the one to click to switch chart type).
|
||||
*/
|
||||
private async getInactiveToggleButtonFrom(buttons: Locator): Promise<Locator> {
|
||||
const count = await buttons.count()
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
const btn = buttons.nth(i)
|
||||
const className = await btn.getAttribute('class').catch(() => '')
|
||||
const hasActive = await btn.evaluate((el) => el.hasAttribute('active')).catch(() => false)
|
||||
|
||||
if (!className?.includes('bg-secondary') && !hasActive) {
|
||||
return btn
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`No inactive toggle button found among ${count} buttons`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the inactive toggle button (the one to click to switch chart type) from a full actions container.
|
||||
*/
|
||||
private async getInactiveToggleButton(actionsContainer: Locator): Promise<Locator> {
|
||||
const buttons = actionsContainer.locator('button')
|
||||
return this.getInactiveToggleButtonFrom(buttons)
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle chart type for volume chart (clicks inactive button to switch)
|
||||
*/
|
||||
async toggleVolumeChartType(): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.closePopups()
|
||||
const actionsContainer = this.page.locator('[data-testid="chart-log-volume-actions"]')
|
||||
const toggleBtn = await this.getInactiveToggleButton(actionsContainer)
|
||||
await toggleBtn.waitFor({ state: 'visible' })
|
||||
await toggleBtn.click()
|
||||
await this.page.waitForLoadState('networkidle').catch(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle chart type for token chart
|
||||
*/
|
||||
async toggleTokenChartType(): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.closePopups()
|
||||
const actionsContainer = this.page.locator('[data-testid="chart-token-usage-actions"]')
|
||||
const toggleBtn = await this.getInactiveToggleButton(actionsContainer)
|
||||
await toggleBtn.waitFor({ state: 'visible' })
|
||||
await toggleBtn.click()
|
||||
await this.page.waitForLoadState('networkidle').catch(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle chart type for cost chart.
|
||||
* Scopes to the ChartTypeToggle only (excludes the model dropdown in the same actions bar).
|
||||
*/
|
||||
async toggleCostChartType(): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.closePopups()
|
||||
const actionsContainer = this.page.locator('[data-testid="chart-cost-total-actions"]')
|
||||
// ChartTypeToggle is the last child in the actions bar; its two buttons are the bar/line toggles only
|
||||
const chartTypeButtons = actionsContainer.locator('> div > div').last().locator('button')
|
||||
const toggleBtn = await this.getInactiveToggleButtonFrom(chartTypeButtons)
|
||||
await toggleBtn.waitFor({ state: 'visible' })
|
||||
await toggleBtn.click()
|
||||
await this.page.waitForLoadState('networkidle').catch(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle chart type for model chart.
|
||||
* Scopes to the ChartTypeToggle only (excludes the model dropdown in the same actions bar).
|
||||
*/
|
||||
async toggleModelChartType(): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.closePopups()
|
||||
const actionsContainer = this.page.locator('[data-testid="chart-model-usage-actions"]')
|
||||
// ChartTypeToggle is the last child in the actions bar; its two buttons are the bar/line toggles only
|
||||
const chartTypeButtons = actionsContainer.locator('> div > div').last().locator('button')
|
||||
const toggleBtn = await this.getInactiveToggleButtonFrom(chartTypeButtons)
|
||||
await toggleBtn.waitFor({ state: 'visible' })
|
||||
await toggleBtn.click()
|
||||
await this.page.waitForLoadState('networkidle').catch(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter cost chart by model. Opens the model dropdown, then selects the option.
|
||||
*/
|
||||
async filterCostChartByModel(model: string): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.costModelFilter.waitFor({ state: 'visible' })
|
||||
// Open the dropdown by clicking the trigger
|
||||
await this.costModelFilter.click()
|
||||
// Wait for dropdown to open (option becomes visible in portal)
|
||||
const optionName = model === 'all' ? 'All Models' : model
|
||||
await this.page.getByRole('option', { name: optionName }).waitFor({ state: 'visible', timeout: 5000 })
|
||||
await this.page.getByRole('option', { name: optionName }).click()
|
||||
// Wait for dropdown to close and data to refresh
|
||||
await this.page.getByRole('option', { name: optionName }).waitFor({ state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter usage chart by model. Opens the model dropdown, then selects the option.
|
||||
*/
|
||||
async filterUsageChartByModel(model: string): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.usageModelFilter.waitFor({ state: 'visible' })
|
||||
// Open the dropdown by clicking the trigger
|
||||
await this.usageModelFilter.click()
|
||||
// Wait for dropdown to open (option becomes visible in portal)
|
||||
const optionName = model === 'all' ? 'All Models' : model
|
||||
await this.page.getByRole('option', { name: optionName }).waitFor({ state: 'visible', timeout: 5000 })
|
||||
await this.page.getByRole('option', { name: optionName }).click()
|
||||
// Wait for dropdown to close and data to refresh
|
||||
await this.page.getByRole('option', { name: optionName }).waitFor({ state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if chart is visible
|
||||
*/
|
||||
async isChartVisible(chartTitle: string): Promise<boolean> {
|
||||
// Map chart titles to test IDs
|
||||
const testIdMap: Record<string, string> = {
|
||||
'Request Volume': 'chart-log-volume',
|
||||
'Token Usage': 'chart-token-usage',
|
||||
'Cost': 'chart-cost-total',
|
||||
'Model Usage': 'chart-model-usage',
|
||||
}
|
||||
const testId = testIdMap[chartTitle]
|
||||
if (testId) {
|
||||
return await this.page.locator(`[data-testid="${testId}"]`).isVisible()
|
||||
}
|
||||
// Fallback for unknown titles
|
||||
const chart = this.page.locator(`text=${chartTitle}`).locator('..').locator('..')
|
||||
return await chart.isVisible()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if chart is loading
|
||||
*/
|
||||
async isChartLoading(chartTitle: string): Promise<boolean> {
|
||||
// Map chart titles to test IDs
|
||||
const testIdMap: Record<string, string> = {
|
||||
'Request Volume': 'chart-log-volume',
|
||||
'Token Usage': 'chart-token-usage',
|
||||
'Cost': 'chart-cost-total',
|
||||
'Model Usage': 'chart-model-usage',
|
||||
}
|
||||
const testId = testIdMap[chartTitle]
|
||||
if (testId) {
|
||||
const chartCard = this.page.locator(`[data-testid="${testId}"]`)
|
||||
const skeleton = chartCard.locator('[data-testid="skeleton"]')
|
||||
return await skeleton.isVisible().catch(() => false)
|
||||
}
|
||||
// Fallback for unknown titles
|
||||
const chartCard = this.page.locator(`text=${chartTitle}`).locator('..').locator('..')
|
||||
const skeleton = chartCard.locator('[data-testid="skeleton"]').or(chartCard.locator('.skeleton'))
|
||||
return await skeleton.isVisible().catch(() => false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get URL parameters
|
||||
*/
|
||||
getUrlParams(): URLSearchParams {
|
||||
return new URLSearchParams(this.page.url().split('?')[1] || '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Get chart toggle state (checks aria-pressed, data-state, or active class)
|
||||
*/
|
||||
async getChartToggleState(toggle: Locator): Promise<string | null> {
|
||||
// Handle case where toggle might match multiple elements
|
||||
const firstToggle = toggle.first()
|
||||
|
||||
// Try aria-pressed first (for button toggles)
|
||||
const ariaPressed = await firstToggle.getAttribute('aria-pressed').catch(() => null)
|
||||
if (ariaPressed) {
|
||||
return ariaPressed
|
||||
}
|
||||
// Try data-state (for switch components)
|
||||
const dataState = await firstToggle.getAttribute('data-state').catch(() => null)
|
||||
if (dataState) {
|
||||
return dataState
|
||||
}
|
||||
// Check if button is active (has active class or attribute)
|
||||
const classAttr = await firstToggle.getAttribute('class').catch(() => null)
|
||||
if (classAttr?.includes('bg-secondary')) {
|
||||
return 'active'
|
||||
}
|
||||
// Check for [active] attribute
|
||||
const isActive = await firstToggle.evaluate((el) => el.hasAttribute('active')).catch(() => false)
|
||||
if (isActive) {
|
||||
return 'active'
|
||||
}
|
||||
return 'inactive'
|
||||
}
|
||||
|
||||
/**
|
||||
* Get selected model from filter combobox
|
||||
*/
|
||||
async getSelectedModel(filter: Locator): Promise<string | null> {
|
||||
const selectedText = await filter.textContent()
|
||||
return selectedText
|
||||
}
|
||||
}
|
||||
19
tests/e2e/features/governance/governance.data.ts
Normal file
19
tests/e2e/features/governance/governance.data.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { CustomerConfig, TeamConfig } from './pages/governance.page'
|
||||
|
||||
export function createTeamData(overrides: Partial<TeamConfig> = {}): TeamConfig {
|
||||
const timestamp = Date.now()
|
||||
return {
|
||||
name: `E2E Team ${timestamp}`,
|
||||
budget: { maxLimit: 100, resetDuration: '1M' },
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
export function createCustomerData(overrides: Partial<CustomerConfig> = {}): CustomerConfig {
|
||||
const timestamp = Date.now()
|
||||
return {
|
||||
name: `E2E Customer ${timestamp}`,
|
||||
budget: { maxLimit: 50, resetDuration: '1d' },
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
173
tests/e2e/features/governance/governance.spec.ts
Normal file
173
tests/e2e/features/governance/governance.spec.ts
Normal file
@@ -0,0 +1,173 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
import { createCustomerData, createTeamData } from './governance.data'
|
||||
|
||||
const createdTeams: string[] = []
|
||||
const createdCustomers: string[] = []
|
||||
|
||||
test.describe('Governance - Teams', () => {
|
||||
test.describe.configure({ mode: 'serial' })
|
||||
test.beforeEach(async ({ governancePage }) => {
|
||||
await governancePage.gotoTeams()
|
||||
})
|
||||
|
||||
test.afterEach(async ({ governancePage }) => {
|
||||
await governancePage.closeTeamDialog()
|
||||
for (const name of [...createdTeams]) {
|
||||
try {
|
||||
const exists = await governancePage.teamExists(name)
|
||||
if (exists) {
|
||||
await governancePage.deleteTeam(name)
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[CLEANUP] Failed to delete team ${name}:`, e)
|
||||
}
|
||||
}
|
||||
createdTeams.length = 0
|
||||
for (const name of [...createdCustomers]) {
|
||||
try {
|
||||
await governancePage.gotoCustomers()
|
||||
const exists = await governancePage.customerExists(name)
|
||||
if (exists) {
|
||||
await governancePage.deleteCustomer(name)
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[CLEANUP] Failed to delete customer ${name}:`, e)
|
||||
}
|
||||
}
|
||||
createdCustomers.length = 0
|
||||
})
|
||||
|
||||
test('should display create team button or empty state', async ({ governancePage }) => {
|
||||
const createVisible = await governancePage.teamsCreateBtn.isVisible().catch(() => false)
|
||||
const emptyAddVisible = await governancePage.page.getByTestId('team-button-add').isVisible().catch(() => false)
|
||||
expect(createVisible || emptyAddVisible).toBe(true)
|
||||
})
|
||||
|
||||
test('should create a team', async ({ governancePage }) => {
|
||||
const teamData = createTeamData({ name: `E2E Test Team ${Date.now()}` })
|
||||
createdTeams.push(teamData.name)
|
||||
|
||||
await governancePage.createTeam(teamData)
|
||||
|
||||
const exists = await governancePage.teamExists(teamData.name)
|
||||
expect(exists).toBe(true)
|
||||
})
|
||||
|
||||
test('should edit a team', async ({ governancePage }) => {
|
||||
const teamData = createTeamData({ name: `E2E Edit Team ${Date.now()}` })
|
||||
createdTeams.push(teamData.name)
|
||||
await governancePage.createTeam(teamData)
|
||||
|
||||
await governancePage.editTeam(teamData.name, { budget: { maxLimit: 129 } })
|
||||
|
||||
const exists = await governancePage.teamExists(teamData.name)
|
||||
expect(exists).toBe(true)
|
||||
})
|
||||
|
||||
test('should create team with customer assignment', async ({ governancePage }) => {
|
||||
// 1. Create a customer (UI)
|
||||
const customerData = createCustomerData({ name: `E2E Customer For Team ${Date.now()}` })
|
||||
createdCustomers.push(customerData.name)
|
||||
await governancePage.gotoCustomers()
|
||||
await governancePage.createCustomer(customerData)
|
||||
|
||||
// 2. Go to Teams and create a team, assign the customer from the create-team dropdown (UI)
|
||||
await governancePage.gotoTeams()
|
||||
const teamData = createTeamData({
|
||||
name: `E2E Team With Customer ${Date.now()}`,
|
||||
customerName: customerData.name,
|
||||
})
|
||||
createdTeams.push(teamData.name)
|
||||
await governancePage.createTeam(teamData)
|
||||
|
||||
// 3. Validate in UI that the customer was assigned (via data-testid)
|
||||
const exists = await governancePage.teamExists(teamData.name)
|
||||
expect(exists).toBe(true)
|
||||
const customerCell = governancePage.getTeamRowCustomerCell(teamData.name)
|
||||
await expect(customerCell).toContainText(customerData.name)
|
||||
})
|
||||
|
||||
test('should delete a team', async ({ governancePage }) => {
|
||||
const teamData = createTeamData({ name: `E2E Delete Team ${Date.now()}` })
|
||||
createdTeams.push(teamData.name)
|
||||
await governancePage.createTeam(teamData)
|
||||
|
||||
let exists = await governancePage.teamExists(teamData.name)
|
||||
expect(exists).toBe(true)
|
||||
|
||||
await governancePage.deleteTeam(teamData.name)
|
||||
const idx = createdTeams.indexOf(teamData.name)
|
||||
if (idx >= 0) createdTeams.splice(idx, 1)
|
||||
|
||||
exists = await governancePage.teamExists(teamData.name)
|
||||
expect(exists).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Governance - Customers', () => {
|
||||
test.describe.configure({ mode: 'serial' })
|
||||
test.beforeEach(async ({ governancePage }) => {
|
||||
await governancePage.gotoCustomers()
|
||||
})
|
||||
|
||||
test.afterEach(async ({ governancePage }) => {
|
||||
for (const name of [...createdCustomers]) {
|
||||
try {
|
||||
const exists = await governancePage.customerExists(name)
|
||||
if (exists) {
|
||||
await governancePage.deleteCustomer(name)
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[CLEANUP] Failed to delete customer ${name}:`, e)
|
||||
}
|
||||
}
|
||||
createdCustomers.length = 0
|
||||
})
|
||||
|
||||
test('should display create customer button or empty state', async ({ governancePage }) => {
|
||||
const createVisible = await governancePage.customersCreateBtn.isVisible().catch(() => false)
|
||||
const emptyCreateVisible = await governancePage.page.getByTestId('customer-button-create').isVisible().catch(() => false)
|
||||
expect(createVisible || emptyCreateVisible).toBe(true)
|
||||
})
|
||||
|
||||
test('should create a customer', async ({ governancePage }) => {
|
||||
const customerData = createCustomerData({ name: `E2E Test Customer ${Date.now()}` })
|
||||
createdCustomers.push(customerData.name)
|
||||
|
||||
await governancePage.createCustomer(customerData)
|
||||
|
||||
const exists = await governancePage.customerExists(customerData.name)
|
||||
expect(exists).toBe(true)
|
||||
})
|
||||
|
||||
test('should edit a customer', async ({ governancePage }) => {
|
||||
const customerData = createCustomerData({ name: `E2E Edit Customer ${Date.now()}` })
|
||||
createdCustomers.push(customerData.name)
|
||||
await governancePage.createCustomer(customerData)
|
||||
|
||||
const newName = `E2E Edited Customer ${Date.now()}`
|
||||
createdCustomers[createdCustomers.length - 1] = newName
|
||||
await governancePage.editCustomer(customerData.name, { name: newName })
|
||||
|
||||
const oldExists = await governancePage.customerExists(customerData.name)
|
||||
const newExists = await governancePage.customerExists(newName)
|
||||
expect(oldExists).toBe(false)
|
||||
expect(newExists).toBe(true)
|
||||
})
|
||||
|
||||
test('should delete a customer', async ({ governancePage }) => {
|
||||
const customerData = createCustomerData({ name: `E2E Delete Customer ${Date.now()}` })
|
||||
createdCustomers.push(customerData.name)
|
||||
await governancePage.createCustomer(customerData)
|
||||
|
||||
let exists = await governancePage.customerExists(customerData.name)
|
||||
expect(exists).toBe(true)
|
||||
|
||||
await governancePage.deleteCustomer(customerData.name)
|
||||
const idx = createdCustomers.indexOf(customerData.name)
|
||||
if (idx >= 0) createdCustomers.splice(idx, 1)
|
||||
|
||||
exists = await governancePage.customerExists(customerData.name)
|
||||
expect(exists).toBe(false)
|
||||
})
|
||||
})
|
||||
229
tests/e2e/features/governance/pages/governance.page.ts
Normal file
229
tests/e2e/features/governance/pages/governance.page.ts
Normal file
@@ -0,0 +1,229 @@
|
||||
import { Locator, Page } from '@playwright/test'
|
||||
import { expect } from '../../../core/fixtures/base.fixture'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { fillSelect, waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
export interface TeamConfig {
|
||||
name: string
|
||||
/** Assign by customer id (from API). Prefer customerName for UI-only flow. */
|
||||
customerId?: string
|
||||
/** Assign by customer name in the create-team dropdown (UI-only, no API). */
|
||||
customerName?: string
|
||||
budget?: { maxLimit: number; resetDuration?: string }
|
||||
rateLimit?: {
|
||||
tokenMaxLimit?: number
|
||||
tokenResetDuration?: string
|
||||
requestMaxLimit?: number
|
||||
requestResetDuration?: string
|
||||
}
|
||||
}
|
||||
|
||||
export interface CustomerConfig {
|
||||
name: string
|
||||
budget?: { maxLimit: number; resetDuration?: string }
|
||||
rateLimit?: {
|
||||
tokenMaxLimit?: number
|
||||
tokenResetDuration?: string
|
||||
requestMaxLimit?: number
|
||||
requestResetDuration?: string
|
||||
}
|
||||
}
|
||||
|
||||
export class GovernancePage extends BasePage {
|
||||
// Teams
|
||||
readonly teamsCreateBtn: Locator
|
||||
readonly teamsTable: Locator
|
||||
readonly teamDialog: Locator
|
||||
readonly teamNameInput: Locator
|
||||
|
||||
// Customers
|
||||
readonly customersCreateBtn: Locator
|
||||
readonly customersTable: Locator
|
||||
readonly customerDialog: Locator
|
||||
readonly customerNameInput: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
|
||||
this.teamsCreateBtn = page.getByTestId('create-team-btn').or(page.getByTestId('team-button-add'))
|
||||
this.teamsTable = page.getByTestId('teams-table')
|
||||
this.teamDialog = page.getByTestId('team-dialog-content')
|
||||
this.teamNameInput = page.getByTestId('team-name-input')
|
||||
|
||||
this.customersCreateBtn = page.getByTestId('customer-button-create')
|
||||
this.customersTable = page.getByTestId('customer-table-container')
|
||||
this.customerDialog = page.getByTestId('customer-dialog-content')
|
||||
this.customerNameInput = page.getByTestId('customer-name-input')
|
||||
}
|
||||
|
||||
async gotoTeams(): Promise<void> {
|
||||
await this.page.goto('/workspace/governance/teams')
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
async gotoCustomers(): Promise<void> {
|
||||
await this.page.goto('/workspace/governance/customers')
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
getTeamRow(name: string): Locator {
|
||||
return this.page.getByTestId(`team-row-${name}`)
|
||||
}
|
||||
|
||||
/** Customer cell for a team row (use for asserting assigned customer in UI). */
|
||||
getTeamRowCustomerCell(teamName: string): Locator {
|
||||
return this.page.getByTestId(`team-row-${teamName}-customer`)
|
||||
}
|
||||
|
||||
async teamExists(name: string): Promise<boolean> {
|
||||
const row = this.getTeamRow(name)
|
||||
return (await row.count()) > 0
|
||||
}
|
||||
|
||||
async createTeam(config: TeamConfig): Promise<void> {
|
||||
await this.teamsCreateBtn.click()
|
||||
await expect(this.teamDialog).toBeVisible({ timeout: 5000 })
|
||||
await this.waitForSheetAnimation()
|
||||
|
||||
await this.teamNameInput.fill(config.name)
|
||||
|
||||
if (config.customerId !== undefined || config.customerName !== undefined) {
|
||||
const trigger = this.page.getByTestId('team-customer-select-trigger')
|
||||
await trigger.click()
|
||||
if (config.customerId === '') {
|
||||
await this.page.getByTestId('team-customer-option-none').click()
|
||||
} else if (config.customerName !== undefined) {
|
||||
const customerOption = this.page
|
||||
.locator('[data-testid^="team-customer-option-"]')
|
||||
.filter({ hasText: config.customerName })
|
||||
await customerOption.waitFor({ state: 'visible', timeout: 5000 })
|
||||
await customerOption.click()
|
||||
} else if (config.customerId !== undefined && config.customerId !== '') {
|
||||
const customerOption = this.page.getByTestId(`team-customer-option-${config.customerId}`)
|
||||
await customerOption.waitFor({ state: 'visible', timeout: 5000 })
|
||||
await customerOption.click()
|
||||
}
|
||||
}
|
||||
|
||||
if (config.budget?.maxLimit !== undefined) {
|
||||
const budgetInput = this.page.getByTestId('budget-max-limit-input')
|
||||
await budgetInput.fill(String(config.budget.maxLimit))
|
||||
}
|
||||
|
||||
const saveBtn = this.teamDialog.getByRole('button', { name: /Create Team/i })
|
||||
await expect(saveBtn).toBeEnabled()
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.teamDialog).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
|
||||
async deleteTeam(name: string): Promise<void> {
|
||||
const deleteBtn = this.page.getByTestId(`team-delete-btn-${name}`)
|
||||
await deleteBtn.click()
|
||||
|
||||
const confirmDialog = this.page.locator('[role="alertdialog"]')
|
||||
await confirmDialog.getByRole('button', { name: /Delete/i }).click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect.poll(() => this.teamExists(name), { timeout: 10000 }).toBe(false)
|
||||
}
|
||||
|
||||
async closeTeamDialog(): Promise<void> {
|
||||
if (await this.teamDialog.isVisible().catch(() => false)) {
|
||||
await this.teamDialog.getByRole('button', { name: /Cancel/i }).click()
|
||||
await expect(this.teamDialog).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
}
|
||||
|
||||
getCustomerRow(name: string): Locator {
|
||||
return this.customersTable.getByTestId(`customer-row-${name}`)
|
||||
}
|
||||
|
||||
async customerExists(name: string): Promise<boolean> {
|
||||
const row = this.getCustomerRow(name)
|
||||
return (await row.count()) > 0
|
||||
}
|
||||
|
||||
async createCustomer(config: CustomerConfig): Promise<void> {
|
||||
await this.customersCreateBtn.click()
|
||||
await expect(this.customerDialog).toBeVisible({ timeout: 5000 })
|
||||
await this.waitForSheetAnimation()
|
||||
|
||||
await this.customerNameInput.fill(config.name)
|
||||
|
||||
if (config.budget?.maxLimit !== undefined) {
|
||||
const budgetInput = this.page.getByTestId('budget-max-limit-input')
|
||||
await budgetInput.fill(String(config.budget.maxLimit))
|
||||
}
|
||||
|
||||
const saveBtn = this.customerDialog.getByRole('button', { name: /Create Customer/i })
|
||||
await expect(saveBtn).toBeEnabled()
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.customerDialog).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
|
||||
async deleteCustomer(name: string): Promise<void> {
|
||||
const row = this.getCustomerRow(name)
|
||||
const deleteBtn = row.locator('[data-testid^="customer-button-delete-"]')
|
||||
await deleteBtn.click()
|
||||
|
||||
const confirmBtn = this.page.getByTestId('customer-button-delete-confirm')
|
||||
await confirmBtn.waitFor({ state: 'visible', timeout: 5000 })
|
||||
await confirmBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect.poll(() => this.customerExists(name), { timeout: 10000 }).toBe(false)
|
||||
}
|
||||
|
||||
async editTeam(name: string, updates: Partial<TeamConfig>): Promise<void> {
|
||||
const editBtn = this.page.getByTestId(`team-edit-btn-${name}`)
|
||||
await editBtn.click()
|
||||
await expect(this.teamDialog).toBeVisible({ timeout: 5000 })
|
||||
await this.waitForSheetAnimation()
|
||||
|
||||
if (updates.name) {
|
||||
await this.teamNameInput.clear()
|
||||
await this.teamNameInput.fill(updates.name)
|
||||
}
|
||||
|
||||
if (updates.customerId !== undefined) {
|
||||
const trigger = this.page.getByTestId('team-customer-select-trigger')
|
||||
await trigger.click()
|
||||
if (updates.customerId === '') {
|
||||
await this.page.getByTestId('team-customer-option-none').click()
|
||||
} else {
|
||||
await this.page.getByTestId(`team-customer-option-${updates.customerId}`).click()
|
||||
}
|
||||
}
|
||||
|
||||
if (updates.budget?.maxLimit !== undefined) {
|
||||
const budgetInput = this.page.getByTestId('budget-max-limit-input')
|
||||
await budgetInput.clear()
|
||||
await budgetInput.fill(String(updates.budget.maxLimit))
|
||||
}
|
||||
|
||||
const saveBtn = this.teamDialog.getByRole('button', { name: /Save|Update/i })
|
||||
await expect(saveBtn).toBeEnabled()
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.teamDialog).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
|
||||
async editCustomer(name: string, updates: Partial<CustomerConfig>): Promise<void> {
|
||||
const row = this.getCustomerRow(name)
|
||||
const editBtn = row.locator('[data-testid^="customer-button-edit-"]')
|
||||
await editBtn.click()
|
||||
await expect(this.customerDialog).toBeVisible({ timeout: 5000 })
|
||||
await this.waitForSheetAnimation()
|
||||
|
||||
if (updates.name) {
|
||||
await this.customerNameInput.clear()
|
||||
await this.customerNameInput.fill(updates.name)
|
||||
}
|
||||
|
||||
const saveBtn = this.customerDialog.getByRole('button', { name: /Save|Update/i })
|
||||
await expect(saveBtn).toBeEnabled()
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.customerDialog).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
}
|
||||
35
tests/e2e/features/logs/logs.data.ts
Normal file
35
tests/e2e/features/logs/logs.data.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Test data factories for logs tests
|
||||
*/
|
||||
|
||||
/**
|
||||
* Sample log entry data for testing
|
||||
*/
|
||||
export interface SampleLogData {
|
||||
provider: string
|
||||
model: string
|
||||
status: 'success' | 'error' | 'pending'
|
||||
content?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Create sample log search query
|
||||
*/
|
||||
export function createLogSearchQuery(overrides: Partial<{ query: string }> = {}): string {
|
||||
return overrides.query || `test-query-${Date.now()}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample providers for filtering
|
||||
*/
|
||||
export const SAMPLE_PROVIDERS = ['openai', 'anthropic', 'gemini'] as const
|
||||
|
||||
/**
|
||||
* Sample models for filtering
|
||||
*/
|
||||
export const SAMPLE_MODELS = ['gpt-4', 'claude-3-opus', 'gemini-pro'] as const
|
||||
|
||||
/**
|
||||
* Sample statuses for filtering
|
||||
*/
|
||||
export const SAMPLE_STATUSES = ['success', 'error', 'pending'] as const
|
||||
427
tests/e2e/features/logs/logs.spec.ts
Normal file
427
tests/e2e/features/logs/logs.spec.ts
Normal file
@@ -0,0 +1,427 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
import { createLogSearchQuery, SAMPLE_MODELS, SAMPLE_PROVIDERS } from './logs.data'
|
||||
|
||||
test.describe('LLM Logs', () => {
|
||||
test.beforeEach(async ({ logsPage }) => {
|
||||
await logsPage.goto()
|
||||
})
|
||||
|
||||
test.describe('Logs Display', () => {
|
||||
test('should display logs table', async ({ logsPage }) => {
|
||||
// Table should be visible after goto (which waits for load)
|
||||
const tableExists = await logsPage.logsTable.isVisible().catch(() => false)
|
||||
expect(tableExists).toBe(true)
|
||||
})
|
||||
|
||||
test('should display stats cards', async ({ logsPage }) => {
|
||||
const statsVisible = await logsPage.areStatsVisible()
|
||||
expect(statsVisible).toBe(true)
|
||||
})
|
||||
|
||||
test('should display filters section', async ({ logsPage }) => {
|
||||
// Check if the search input or filters button is visible
|
||||
// These are always visible when the page loads (not inside empty state)
|
||||
const searchVisible = await logsPage.searchInput.isVisible().catch(() => false)
|
||||
const filtersButtonVisible = await logsPage.filtersButton.isVisible().catch(() => false)
|
||||
|
||||
// Either search input OR filters button should be visible
|
||||
expect(searchVisible || filtersButtonVisible).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Log Filtering', () => {
|
||||
test('should filter logs by provider', async ({ logsPage }) => {
|
||||
// Try to filter by first available provider
|
||||
const providerFilter = logsPage.providerFilter
|
||||
const isVisible = await providerFilter.isVisible().catch(() => false)
|
||||
|
||||
if (!isVisible || SAMPLE_PROVIDERS.length === 0) {
|
||||
test.skip(!isVisible || SAMPLE_PROVIDERS.length === 0, 'Provider filter not visible or no sample providers')
|
||||
return
|
||||
}
|
||||
|
||||
// Get initial filter state
|
||||
const initialValue = await providerFilter.textContent().catch(() => '')
|
||||
|
||||
await logsPage.filterByProvider(SAMPLE_PROVIDERS[0])
|
||||
|
||||
// Check that filter value changed (or verify filter is applied via DOM)
|
||||
const newValue = await providerFilter.textContent().catch(() => '')
|
||||
// Filter should have changed or show selected provider
|
||||
expect(newValue || initialValue).toBeTruthy()
|
||||
})
|
||||
|
||||
test('should filter logs by model', async ({ logsPage }) => {
|
||||
const modelFilter = logsPage.modelFilter
|
||||
const isVisible = await modelFilter.isVisible().catch(() => false)
|
||||
|
||||
if (!isVisible || SAMPLE_MODELS.length === 0) {
|
||||
test.skip(!isVisible || SAMPLE_MODELS.length === 0, 'Model filter not visible or no sample models')
|
||||
return
|
||||
}
|
||||
|
||||
// Get initial filter state
|
||||
const initialValue = await modelFilter.textContent().catch(() => '')
|
||||
|
||||
await logsPage.filterByModel(SAMPLE_MODELS[0])
|
||||
|
||||
// Check that filter value changed (or verify filter is applied via DOM)
|
||||
const newValue = await modelFilter.textContent().catch(() => '')
|
||||
// Filter should have changed or show selected model
|
||||
expect(newValue || initialValue).toBeTruthy()
|
||||
})
|
||||
|
||||
test('should filter logs by status', async ({ logsPage, page }) => {
|
||||
const filtersVisible = await logsPage.filtersButton.isVisible().catch(() => false)
|
||||
if (!filtersVisible) {
|
||||
test.skip(true, 'Filters button not visible')
|
||||
return
|
||||
}
|
||||
|
||||
await logsPage.filterByStatus('success')
|
||||
|
||||
// Assert status filter is applied: logs page persists filters in URL (e.g. status=success)
|
||||
await expect
|
||||
.poll(() => page.url(), { timeout: 5000, intervals: [200, 300, 500] })
|
||||
.toMatch(/status=success/)
|
||||
})
|
||||
|
||||
test('should search logs by content', async ({ logsPage }) => {
|
||||
const searchInput = logsPage.searchInput
|
||||
const isVisible = await searchInput.isVisible().catch(() => false)
|
||||
|
||||
if (!isVisible) {
|
||||
test.skip(true, 'Search input not visible')
|
||||
return
|
||||
}
|
||||
|
||||
const query = createLogSearchQuery()
|
||||
await logsPage.searchLogs(query)
|
||||
|
||||
// Check that search input contains the query (DOM state)
|
||||
const inputValue = await searchInput.inputValue().catch(() => '')
|
||||
expect(inputValue).toContain(query)
|
||||
})
|
||||
|
||||
test('should clear search', async ({ logsPage }) => {
|
||||
const searchInput = logsPage.searchInput
|
||||
const isVisible = await searchInput.isVisible().catch(() => false)
|
||||
|
||||
if (!isVisible) {
|
||||
test.skip(true, 'Search input not visible')
|
||||
return
|
||||
}
|
||||
|
||||
await logsPage.searchLogs('test query')
|
||||
await logsPage.clearSearch()
|
||||
|
||||
// Search should be cleared
|
||||
const inputValue = await searchInput.inputValue().catch(() => '')
|
||||
expect(inputValue).toBe('')
|
||||
})
|
||||
|
||||
test('should filter by time period', async ({ logsPage }) => {
|
||||
const datePicker = logsPage.dateRangePicker
|
||||
const isVisible = await datePicker.isVisible().catch(() => false)
|
||||
|
||||
if (!isVisible) {
|
||||
test.skip(true, 'Date range picker not visible')
|
||||
return
|
||||
}
|
||||
|
||||
// Get initial date picker value
|
||||
const initialValue = await datePicker.textContent().catch(() => '')
|
||||
|
||||
await logsPage.selectTimePeriod('7d')
|
||||
|
||||
// Check that date picker value changed (DOM state)
|
||||
const newValue = await datePicker.textContent().catch(() => '')
|
||||
// Date picker should show "Last 7 days" or similar
|
||||
expect(newValue || initialValue).toBeTruthy()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Log Details', () => {
|
||||
test('should open log details sheet', async ({ logsPage }) => {
|
||||
// Wait a bit for logs to potentially load
|
||||
await logsPage.page.waitForTimeout(1000)
|
||||
|
||||
const logCount = await logsPage.getLogCount()
|
||||
|
||||
if (logCount > 0) {
|
||||
await logsPage.viewLogDetails(0)
|
||||
|
||||
// Wait for sheet animation
|
||||
await logsPage.page.waitForTimeout(500)
|
||||
|
||||
// Detail sheet should be visible
|
||||
const sheetVisible = await logsPage.logDetailSheet.isVisible().catch(() => false)
|
||||
expect(sheetVisible).toBe(true)
|
||||
|
||||
// Close the sheet
|
||||
await logsPage.closeLogDetails()
|
||||
} else {
|
||||
// If no logs exist, the test passes (nothing to click)
|
||||
expect(logCount).toBe(0)
|
||||
}
|
||||
})
|
||||
|
||||
test('should close log details sheet', async ({ logsPage }) => {
|
||||
const logCount = await logsPage.getLogCount()
|
||||
|
||||
if (logCount > 0) {
|
||||
await logsPage.viewLogDetails(0)
|
||||
await logsPage.closeLogDetails()
|
||||
|
||||
// Sheet should be closed
|
||||
const sheetVisible = await logsPage.logDetailSheet.isVisible().catch(() => false)
|
||||
expect(sheetVisible).toBe(false)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Pagination', () => {
|
||||
test('should navigate to next page', async ({ logsPage }) => {
|
||||
// Wait for pagination to settle (useTablePageSize may adjust limit dynamically)
|
||||
await logsPage.page.waitForTimeout(2000)
|
||||
const paginationVisible = await logsPage.paginationControls.isVisible().catch(() => false)
|
||||
if (!paginationVisible) {
|
||||
test.skip(true, 'Pagination controls not visible')
|
||||
return
|
||||
}
|
||||
const nextBtn = logsPage.nextPageBtn.first()
|
||||
const isEnabled = await nextBtn.isEnabled().catch(() => false)
|
||||
|
||||
if (!isEnabled) {
|
||||
test.skip(true, 'Only one page of results; skipping pagination test')
|
||||
return
|
||||
}
|
||||
|
||||
const initialPage = logsPage.getCurrentPageNumber()
|
||||
expect(initialPage).toBe(1)
|
||||
await logsPage.goToNextPage()
|
||||
|
||||
await expect
|
||||
.poll(() => logsPage.getCurrentPageNumber(), { timeout: 5000 })
|
||||
.toBe(initialPage + 1)
|
||||
})
|
||||
|
||||
test('should navigate to previous page', async ({ logsPage }) => {
|
||||
// Wait for pagination to settle (useTablePageSize may adjust limit dynamically)
|
||||
await logsPage.page.waitForTimeout(2000)
|
||||
const paginationVisible = await logsPage.paginationControls.isVisible().catch(() => false)
|
||||
if (!paginationVisible) {
|
||||
test.skip(true, 'Pagination controls not visible')
|
||||
return
|
||||
}
|
||||
const nextBtn = logsPage.nextPageBtn.first()
|
||||
const nextEnabled = await nextBtn.isEnabled().catch(() => false)
|
||||
|
||||
if (!nextEnabled) {
|
||||
test.skip(true, 'Only one page of results; skipping pagination test')
|
||||
return
|
||||
}
|
||||
|
||||
await logsPage.goToNextPage()
|
||||
|
||||
await expect
|
||||
.poll(() => logsPage.getCurrentPageNumber(), { timeout: 5000 })
|
||||
.toBe(2)
|
||||
|
||||
const prevBtn = logsPage.prevPageBtn.first()
|
||||
const prevEnabled = await prevBtn.isEnabled().catch(() => false)
|
||||
|
||||
if (!prevEnabled) {
|
||||
test.skip(true, 'Only one page of results; skipping previous-page test')
|
||||
return
|
||||
}
|
||||
|
||||
await logsPage.goToPreviousPage()
|
||||
|
||||
await expect
|
||||
.poll(() => logsPage.getCurrentPageNumber(), { timeout: 5000 })
|
||||
.toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Table Sorting', () => {
|
||||
test('should sort by timestamp', async ({ logsPage }) => {
|
||||
// Timestamp is the default sort column (desc), so clicking it toggles to asc
|
||||
await logsPage.sortBy('timestamp')
|
||||
|
||||
// Timestamp sort toggles order; wait for URL to reflect the change
|
||||
await logsPage.page.waitForURL(/order=asc|sort_by=timestamp/, { timeout: 5000 })
|
||||
})
|
||||
|
||||
test('should sort by latency', async ({ logsPage }) => {
|
||||
await logsPage.sortBy('latency')
|
||||
|
||||
// Wait for URL to update
|
||||
await logsPage.page.waitForURL(/sort_by=latency/, { timeout: 5000 })
|
||||
|
||||
// Check URL state for latency sort
|
||||
const sortState = await logsPage.getSortState('latency')
|
||||
expect(sortState).toBeTruthy()
|
||||
})
|
||||
|
||||
test('should sort by cost', async ({ logsPage }) => {
|
||||
await logsPage.sortBy('cost')
|
||||
|
||||
// Wait for URL to update
|
||||
await logsPage.page.waitForURL(/sort_by=cost/, { timeout: 5000 })
|
||||
|
||||
// Check URL state for cost sort
|
||||
const sortState = await logsPage.getSortState('cost')
|
||||
expect(sortState).toBeTruthy()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Live Updates', () => {
|
||||
test('should toggle live updates', async ({ logsPage }) => {
|
||||
const liveToggle = logsPage.liveToggle
|
||||
const isVisible = await liveToggle.isVisible().catch(() => false)
|
||||
|
||||
if (!isVisible) {
|
||||
test.skip(true, 'Live toggle not visible')
|
||||
return
|
||||
}
|
||||
|
||||
// Default is live_enabled=true (but URL may not have it since it's the default)
|
||||
// Check for live_enabled=false to determine if currently disabled
|
||||
const initialUrl = logsPage.page.url()
|
||||
const initialLiveDisabled = initialUrl.includes('live_enabled=false')
|
||||
|
||||
await logsPage.toggleLiveUpdates()
|
||||
|
||||
// Wait for URL to reflect live_enabled toggle
|
||||
await logsPage.page.waitForURL(/live_enabled=/, { timeout: 5000 })
|
||||
|
||||
const newUrl = logsPage.page.url()
|
||||
const newLiveDisabled = newUrl.includes('live_enabled=false')
|
||||
|
||||
// Live enabled state should have toggled
|
||||
// If initially enabled (not disabled), after toggle it should be disabled
|
||||
// If initially disabled, after toggle it should be enabled (no live_enabled=false)
|
||||
expect(newLiveDisabled).not.toBe(initialLiveDisabled)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Empty State', () => {
|
||||
test('should show empty state when no logs', async ({ logsPage }) => {
|
||||
// Try to filter by a non-existent provider
|
||||
const searchInput = logsPage.searchInput
|
||||
const isVisible = await searchInput.isVisible().catch(() => false)
|
||||
|
||||
if (!isVisible) {
|
||||
test.skip(true, 'Search input not visible')
|
||||
return
|
||||
}
|
||||
|
||||
await logsPage.searchLogs(`nonexistent-query-${Date.now()}`)
|
||||
|
||||
// After searching for a non-existent query, empty state should appear (wait for API + render)
|
||||
await expect(
|
||||
logsPage.page.locator('text=/No results found|No logs found/i')
|
||||
).toBeVisible({ timeout: 10000 })
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Advanced Filtering', () => {
|
||||
test('should combine multiple filters', async ({ logsPage }) => {
|
||||
// Apply multiple filters if they're visible
|
||||
const searchVisible = await logsPage.searchInput.isVisible().catch(() => false)
|
||||
const providerVisible = await logsPage.providerFilter.isVisible().catch(() => false)
|
||||
|
||||
if (!searchVisible || !providerVisible) {
|
||||
test.skip(true, 'Search input or provider filter not visible')
|
||||
return
|
||||
}
|
||||
|
||||
// Apply search filter
|
||||
await logsPage.searchLogs('test')
|
||||
|
||||
// Apply provider filter
|
||||
if (SAMPLE_PROVIDERS.length > 0) {
|
||||
await logsPage.filterByProvider(SAMPLE_PROVIDERS[0])
|
||||
}
|
||||
|
||||
// Both filters should be applied
|
||||
const searchValue = await logsPage.searchInput.inputValue().catch(() => '')
|
||||
expect(searchValue).toContain('test')
|
||||
})
|
||||
|
||||
test('should clear all filters', async ({ logsPage }) => {
|
||||
const searchVisible = await logsPage.searchInput.isVisible().catch(() => false)
|
||||
|
||||
if (!searchVisible) {
|
||||
test.skip(true, 'Search input not visible')
|
||||
return
|
||||
}
|
||||
|
||||
// Apply a filter first
|
||||
await logsPage.searchLogs('test query to clear')
|
||||
|
||||
// Clear the search
|
||||
await logsPage.clearSearch()
|
||||
|
||||
// Search should be empty
|
||||
const searchValue = await logsPage.searchInput.inputValue().catch(() => '')
|
||||
expect(searchValue).toBe('')
|
||||
})
|
||||
|
||||
test('should search within filtered results', async ({ logsPage }) => {
|
||||
const searchVisible = await logsPage.searchInput.isVisible().catch(() => false)
|
||||
const statusVisible = await logsPage.statusFilter.isVisible().catch(() => false)
|
||||
|
||||
if (!searchVisible || !statusVisible) {
|
||||
test.skip(true, 'Search input or status filter not visible')
|
||||
return
|
||||
}
|
||||
|
||||
// Apply status filter first
|
||||
await logsPage.filterByStatus('success')
|
||||
|
||||
// Then apply search
|
||||
await logsPage.searchLogs('api')
|
||||
|
||||
// Search input should contain the query
|
||||
const searchValue = await logsPage.searchInput.inputValue().catch(() => '')
|
||||
expect(searchValue).toContain('api')
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('URL State Persistence', () => {
|
||||
test('should persist filters in URL', async ({ logsPage }) => {
|
||||
const searchVisible = await logsPage.searchInput.isVisible().catch(() => false)
|
||||
if (!searchVisible) return
|
||||
|
||||
await logsPage.searchLogs('persistent-search')
|
||||
|
||||
// Search is debounced (500ms) then URL updates; wait for URL to contain the param
|
||||
await expect
|
||||
.poll(
|
||||
() => logsPage.page.url(),
|
||||
{ timeout: 8000, intervals: [300, 500, 500] }
|
||||
)
|
||||
.toContain('content_search=')
|
||||
const url = logsPage.page.url()
|
||||
// Value may be percent-encoded (e.g. persistent-search → persistent%2Dsearch)
|
||||
expect(decodeURIComponent(url)).toContain('persistent-search')
|
||||
})
|
||||
|
||||
test('should restore state from URL', async ({ logsPage, page }) => {
|
||||
// Logs page uses start_time and end_time (unix timestamps), not period
|
||||
const endTime = Math.floor(Date.now() / 1000)
|
||||
const startTime = endTime - 7 * 24 * 60 * 60 // 7 days ago
|
||||
await page.goto(`/workspace/logs?start_time=${startTime}&end_time=${endTime}`)
|
||||
|
||||
// Wait for page to load and URL to reflect state (nuqs may merge or keep params)
|
||||
await expect
|
||||
.poll(() => page.url(), { timeout: 5000, intervals: [200, 300, 500] })
|
||||
.toMatch(/start_time=\d+/)
|
||||
const url = page.url()
|
||||
expect(url).toMatch(/start_time=\d+/)
|
||||
expect(url).toMatch(/end_time=\d+/)
|
||||
})
|
||||
})
|
||||
})
|
||||
384
tests/e2e/features/logs/pages/logs.page.ts
Normal file
384
tests/e2e/features/logs/pages/logs.page.ts
Normal file
@@ -0,0 +1,384 @@
|
||||
import { Locator, Page, expect } from '@playwright/test'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
/**
|
||||
* Page object for the LLM Logs page
|
||||
*/
|
||||
export class LogsPage extends BasePage {
|
||||
// Main elements
|
||||
readonly logsTable: Locator
|
||||
readonly filtersSection: Locator
|
||||
readonly filtersButton: Locator
|
||||
readonly statsCards: Locator
|
||||
|
||||
// Filter elements
|
||||
readonly providerFilter: Locator
|
||||
readonly modelFilter: Locator
|
||||
readonly statusFilter: Locator
|
||||
readonly searchInput: Locator
|
||||
readonly dateRangePicker: Locator
|
||||
readonly liveToggle: Locator
|
||||
|
||||
// Table elements
|
||||
readonly tableRows: Locator
|
||||
readonly paginationControls: Locator
|
||||
readonly nextPageBtn: Locator
|
||||
readonly prevPageBtn: Locator
|
||||
|
||||
// Log detail sheet
|
||||
readonly logDetailSheet: Locator
|
||||
readonly closeDetailSheetBtn: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
|
||||
// Main elements
|
||||
this.logsTable = page.locator('[data-testid="logs-table"]').or(page.locator('table'))
|
||||
// The filters section is the container with search input and filters button
|
||||
this.filtersSection = page.locator('input[placeholder="Search logs"]').locator('..')
|
||||
this.filtersButton = page.getByRole('button', { name: /Filters/i })
|
||||
this.statsCards = page.locator('[data-testid="stats-cards"]').or(page.locator('text=Total Requests').locator('..').locator('..'))
|
||||
|
||||
// Filter elements - filters are inside a popover opened by the Filters button
|
||||
this.providerFilter = page.locator('[data-testid="filter-provider"]').or(
|
||||
page.locator('button').filter({ hasText: /Provider/i })
|
||||
)
|
||||
this.modelFilter = page.locator('[data-testid="filter-model"]').or(
|
||||
page.locator('button').filter({ hasText: /Model/i })
|
||||
)
|
||||
this.statusFilter = page.locator('[data-testid="filter-status"]').or(
|
||||
page.locator('button').filter({ hasText: /Status/i })
|
||||
)
|
||||
this.searchInput = page.locator('[data-testid="filter-search"]').or(
|
||||
page.getByPlaceholder('Search logs')
|
||||
)
|
||||
this.dateRangePicker = page.locator('[data-testid="filter-date-range"]').or(
|
||||
page.locator('button').filter({ hasText: /Last/i })
|
||||
)
|
||||
this.liveToggle = page.locator('[data-testid="live-toggle"]').or(
|
||||
page.getByRole('button', { name: /Live updates/i })
|
||||
)
|
||||
|
||||
// Table elements - exclude the "Listening for logs" row which is not a data row
|
||||
this.tableRows = this.logsTable.locator('tbody tr').filter({ hasNot: page.locator('text=Listening for logs') }).filter({ hasNot: page.locator('text=Live updates paused') }).filter({ hasNot: page.locator('text=Not connected') }).filter({ hasNot: page.locator('text=No results found') })
|
||||
// LLM logs pagination (data-testid added to logsTable.tsx)
|
||||
this.paginationControls = page.getByTestId('pagination')
|
||||
this.nextPageBtn = page.getByTestId('next-page')
|
||||
this.prevPageBtn = page.getByTestId('prev-page')
|
||||
|
||||
// Log detail sheet - Sheet component with role="dialog"
|
||||
this.logDetailSheet = page.locator('[role="dialog"]')
|
||||
this.closeDetailSheetBtn = page.locator('[role="dialog"]').locator('button').filter({ has: page.locator('svg.lucide-x') })
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to the logs page
|
||||
*/
|
||||
async goto(): Promise<void> {
|
||||
await this.page.goto('/workspace/logs')
|
||||
await waitForNetworkIdle(this.page)
|
||||
// Wait for table or empty state to be visible
|
||||
await this.logsTable.or(this.page.locator('text=/No logs found|No results found/i')).waitFor({ state: 'visible', timeout: 10000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to the logs page with a small page size so pagination can be tested with fewer total logs.
|
||||
*/
|
||||
async gotoWithSmallPageSize(limit = 5): Promise<void> {
|
||||
await this.page.goto(`/workspace/logs?limit=${limit}&offset=0`)
|
||||
await waitForNetworkIdle(this.page)
|
||||
await this.logsTable.or(this.page.locator('text=/No logs found|No results found/i')).waitFor({ state: 'visible', timeout: 10000 })
|
||||
// The useTablePageSize hook may override the limit from URL, causing a re-render.
|
||||
// Wait for pagination to become visible, retrying if the dynamic page size effect causes a brief re-render.
|
||||
await this.page.waitForTimeout(1500) // Allow useTablePageSize effect to settle
|
||||
await waitForNetworkIdle(this.page)
|
||||
await this.paginationControls.waitFor({ state: 'visible', timeout: 10000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter by provider
|
||||
*/
|
||||
async filterByProvider(provider: string): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.providerFilter.first().waitFor({ state: 'visible' })
|
||||
await this.providerFilter.first().click()
|
||||
await this.page.waitForSelector('[role="listbox"]', { timeout: 5000 }).catch(() => {})
|
||||
|
||||
// Try to find the provider option
|
||||
const option = this.page.getByRole('option', { name: new RegExp(provider, 'i') })
|
||||
if (await option.count() > 0) {
|
||||
await option.first().click()
|
||||
} else {
|
||||
// Close dropdown if option not found
|
||||
await this.page.keyboard.press('Escape')
|
||||
}
|
||||
|
||||
// Wait for dropdown to close and data to refresh
|
||||
await this.page.waitForSelector('[role="listbox"]', { state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter by model
|
||||
*/
|
||||
async filterByModel(model: string): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.modelFilter.first().waitFor({ state: 'visible' })
|
||||
await this.modelFilter.first().click()
|
||||
await this.page.waitForSelector('[role="listbox"]', { timeout: 5000 }).catch(() => {})
|
||||
|
||||
const option = this.page.getByRole('option', { name: new RegExp(model, 'i') })
|
||||
if (await option.count() > 0) {
|
||||
await option.first().click()
|
||||
} else {
|
||||
await this.page.keyboard.press('Escape')
|
||||
}
|
||||
|
||||
await this.page.waitForSelector('[role="listbox"]', { state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter by status. Opens the Filters popover and toggles the given status option (Status group uses lowercase: success, error, etc.).
|
||||
*/
|
||||
async filterByStatus(status: 'success' | 'error' | 'pending'): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.filtersButton.first().waitFor({ state: 'visible' })
|
||||
await this.filtersButton.first().click()
|
||||
await this.page.waitForSelector('[role="listbox"], [data-slot="command-list"]', { timeout: 5000 }).catch(() => {})
|
||||
|
||||
const option = this.page.getByRole('option', { name: new RegExp(status, 'i') })
|
||||
if (await option.count() > 0) {
|
||||
await option.first().click()
|
||||
} else {
|
||||
await this.page.keyboard.press('Escape')
|
||||
}
|
||||
|
||||
await this.page.waitForSelector('[role="listbox"]', { state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Search logs by content
|
||||
*/
|
||||
async searchLogs(query: string): Promise<void> {
|
||||
await this.searchInput.fill(query)
|
||||
// Wait for debounced search to trigger network request
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear search
|
||||
*/
|
||||
async clearSearch(): Promise<void> {
|
||||
await this.searchInput.clear()
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Select time period. Opens the date range popover, then clicks the predefined period button.
|
||||
*/
|
||||
async selectTimePeriod(period: '1h' | '6h' | '24h' | '7d' | '30d'): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
const trigger = this.dateRangePicker.first()
|
||||
await trigger.waitFor({ state: 'visible' })
|
||||
// Open the time period popover by clicking the date range trigger
|
||||
await trigger.click()
|
||||
const periodLabels: Record<string, string> = {
|
||||
'1h': 'Last hour',
|
||||
'6h': 'Last 6 hours',
|
||||
'24h': 'Last 24 hours',
|
||||
'7d': 'Last 7 days',
|
||||
'30d': 'Last 30 days',
|
||||
}
|
||||
const periodButton = this.page.getByRole('button', { name: periodLabels[period] })
|
||||
// Wait for popover to open (predefined period button becomes visible)
|
||||
await periodButton.waitFor({ state: 'visible', timeout: 5000 })
|
||||
await periodButton.click()
|
||||
// Wait for popover to close and requests to settle
|
||||
await periodButton.waitFor({ state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle live updates
|
||||
*/
|
||||
async toggleLiveUpdates(): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
await this.liveToggle.first().waitFor({ state: 'visible' })
|
||||
await this.liveToggle.first().click()
|
||||
}
|
||||
|
||||
/**
|
||||
* Click on a log row to view details
|
||||
*/
|
||||
async viewLogDetails(rowIndex: number = 0): Promise<void> {
|
||||
const rows = this.tableRows
|
||||
const count = await rows.count()
|
||||
|
||||
if (count <= rowIndex) {
|
||||
throw new Error(`Row index ${rowIndex} out of bounds (${count} rows available)`)
|
||||
}
|
||||
await rows.nth(rowIndex).click()
|
||||
// Wait for detail sheet to appear
|
||||
await expect(this.logDetailSheet).toBeVisible({ timeout: 5000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Close log detail sheet
|
||||
*/
|
||||
async closeLogDetails(): Promise<void> {
|
||||
if (await this.logDetailSheet.isVisible()) {
|
||||
await this.closeDetailSheetBtn.click().catch(async () => {
|
||||
// Try pressing Escape if close button not found
|
||||
await this.page.keyboard.press('Escape')
|
||||
})
|
||||
await expect(this.logDetailSheet).not.toBeVisible({ timeout: 5000 })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get log count from table
|
||||
*/
|
||||
async getLogCount(): Promise<number> {
|
||||
return await this.tableRows.count()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if log exists in table
|
||||
*/
|
||||
async logExists(searchText: string): Promise<boolean> {
|
||||
const row = this.tableRows.filter({ hasText: searchText })
|
||||
return await row.count() > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current 1-based page number from URL (offset/limit).
|
||||
*/
|
||||
getCurrentPageNumber(): number {
|
||||
const url = this.page.url()
|
||||
const params = new URL(url).searchParams
|
||||
const offset = Number.parseInt(params.get('offset') ?? '0', 10)
|
||||
const limit = Number.parseInt(params.get('limit') ?? '25', 10) || 25
|
||||
return Math.floor(offset / limit) + 1
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to next page (waits for URL to update)
|
||||
*/
|
||||
async goToNextPage(): Promise<void> {
|
||||
const btn = this.nextPageBtn.first()
|
||||
const isEnabled = await btn.isEnabled().catch(() => false)
|
||||
if (!isEnabled) return
|
||||
await btn.scrollIntoViewIfNeeded()
|
||||
await btn.waitFor({ state: 'visible' })
|
||||
const limit = Number.parseInt(new URL(this.page.url()).searchParams.get('limit') ?? '25', 10) || 25
|
||||
const currentOffset = Number.parseInt(new URL(this.page.url()).searchParams.get('offset') ?? '0', 10)
|
||||
const expectedOffset = currentOffset + limit
|
||||
await btn.click()
|
||||
await this.page.waitForURL(
|
||||
(url) => new URL(url).searchParams.get('offset') === String(expectedOffset),
|
||||
{ timeout: 10000 }
|
||||
)
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to previous page (waits for URL to update)
|
||||
*/
|
||||
async goToPreviousPage(): Promise<void> {
|
||||
const btn = this.prevPageBtn.first()
|
||||
const isEnabled = await btn.isEnabled().catch(() => false)
|
||||
if (!isEnabled) return
|
||||
await btn.scrollIntoViewIfNeeded()
|
||||
await btn.waitFor({ state: 'visible' })
|
||||
const limit = Number.parseInt(new URL(this.page.url()).searchParams.get('limit') ?? '25', 10) || 25
|
||||
const currentOffset = Number.parseInt(new URL(this.page.url()).searchParams.get('offset') ?? '0', 10)
|
||||
const expectedOffset = Math.max(0, currentOffset - limit)
|
||||
await btn.click()
|
||||
await this.page.waitForURL(
|
||||
(url) => {
|
||||
const offset = new URL(url).searchParams.get('offset')
|
||||
// When going back to page 1, offset param may be removed (null) or set to "0"
|
||||
if (expectedOffset === 0) return offset === null || offset === '0'
|
||||
return offset === String(expectedOffset)
|
||||
},
|
||||
{ timeout: 10000 }
|
||||
)
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort table by column - clicks the sort button in the column header
|
||||
*/
|
||||
async sortBy(column: 'timestamp' | 'latency' | 'tokens' | 'cost'): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
|
||||
// Map column names to header button text
|
||||
const columnLabels: Record<string, string> = {
|
||||
'timestamp': 'Time',
|
||||
'latency': 'Latency',
|
||||
'tokens': 'Tokens',
|
||||
'cost': 'Cost'
|
||||
}
|
||||
|
||||
const label = columnLabels[column] || column
|
||||
// The sortable column headers have a button with the column name
|
||||
const sortButton = this.logsTable.getByRole('button', { name: new RegExp(label, 'i') })
|
||||
|
||||
if (await sortButton.count() > 0) {
|
||||
await sortButton.first().waitFor({ state: 'visible' })
|
||||
await sortButton.first().click()
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if stats cards are visible
|
||||
*/
|
||||
async areStatsVisible(): Promise<boolean> {
|
||||
const statsText = this.page.locator('text=Total Requests')
|
||||
return await statsText.isVisible().catch(() => false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats value
|
||||
*/
|
||||
async getStatValue(statName: string): Promise<string | null> {
|
||||
const statCard = this.page.locator(`text=${statName}`).locator('..').locator('..')
|
||||
if (await statCard.isVisible()) {
|
||||
const value = statCard.locator('.font-mono').or(statCard.locator('text=/\\d+/'))
|
||||
return await value.textContent()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if empty state is shown (no logs, or no results for current filters)
|
||||
*/
|
||||
async isEmptyStateVisible(): Promise<boolean> {
|
||||
const emptyState = this.page
|
||||
.locator('text=/No logs found/i')
|
||||
.or(this.page.locator('text=/No data/i'))
|
||||
.or(this.page.locator('text=/No results found/i'))
|
||||
return await emptyState.isVisible().catch(() => false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sort state for a column from URL parameters
|
||||
* Returns 'asc', 'desc', or null if column is not the current sort column
|
||||
*/
|
||||
async getSortState(column: 'timestamp' | 'latency' | 'tokens' | 'cost'): Promise<string | null> {
|
||||
const url = this.page.url()
|
||||
const urlParams = new URL(url).searchParams
|
||||
const sortBy = urlParams.get('sort_by')
|
||||
const order = urlParams.get('order')
|
||||
|
||||
// Check if this column is the currently sorted column
|
||||
if (sortBy === column) {
|
||||
return order || 'desc' // default is desc
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
13
tests/e2e/features/mcp-auth-config/mcp-auth-config.spec.ts
Normal file
13
tests/e2e/features/mcp-auth-config/mcp-auth-config.spec.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
|
||||
// MCP Auth Config routes to @enterprise components not present in OSS.
|
||||
// Tests only verify URL routing; do not add UI assertions for enterprise-only content.
|
||||
test.describe('MCP Auth Config', () => {
|
||||
test.beforeEach(async ({ mcpAuthConfigPage }) => {
|
||||
await mcpAuthConfigPage.goto()
|
||||
})
|
||||
|
||||
test('should load MCP auth config page', async ({ mcpAuthConfigPage }) => {
|
||||
await expect(mcpAuthConfigPage.page).toHaveURL(/mcp-auth-config/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Page } from '@playwright/test'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
export class MCPAuthConfigPage extends BasePage {
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
}
|
||||
|
||||
async goto(): Promise<void> {
|
||||
await this.page.goto('/workspace/mcp-auth-config')
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
}
|
||||
35
tests/e2e/features/mcp-logs/mcp-logs.data.ts
Normal file
35
tests/e2e/features/mcp-logs/mcp-logs.data.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Test data factories for MCP logs tests
|
||||
*/
|
||||
|
||||
/**
|
||||
* Sample MCP log entry data for testing
|
||||
*/
|
||||
export interface SampleMCPLogData {
|
||||
mcpClient: string
|
||||
tool: string
|
||||
status: 'success' | 'error' | 'pending'
|
||||
content?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Create sample MCP log search query
|
||||
*/
|
||||
export function createMCPLogSearchQuery(overrides: Partial<{ query: string }> = {}): string {
|
||||
return overrides.query || `mcp-test-query-${Date.now()}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample MCP clients for filtering
|
||||
*/
|
||||
export const SAMPLE_MCP_CLIENTS = ['test-client-1', 'test-client-2'] as const
|
||||
|
||||
/**
|
||||
* Sample MCP tools for filtering
|
||||
*/
|
||||
export const SAMPLE_MCP_TOOLS = ['tool-1', 'tool-2', 'tool-3'] as const
|
||||
|
||||
/**
|
||||
* Sample statuses for filtering
|
||||
*/
|
||||
export const SAMPLE_STATUSES = ['success', 'error', 'pending'] as const
|
||||
275
tests/e2e/features/mcp-logs/mcp-logs.spec.ts
Normal file
275
tests/e2e/features/mcp-logs/mcp-logs.spec.ts
Normal file
@@ -0,0 +1,275 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
|
||||
test.describe('MCP Logs', () => {
|
||||
test.beforeEach(async ({ mcpLogsPage }) => {
|
||||
await mcpLogsPage.goto()
|
||||
})
|
||||
|
||||
test.describe('MCP Logs Display', () => {
|
||||
test('should display MCP logs table or getting started guide', async ({ mcpLogsPage }) => {
|
||||
// When MCP logs exist the table is visible; otherwise a "Get Started" guide is shown
|
||||
const tableExists = await mcpLogsPage.logsTable.isVisible().catch(() => false)
|
||||
const gettingStarted = await mcpLogsPage.page.getByText(/Get Started/i).isVisible().catch(() => false)
|
||||
expect(tableExists || gettingStarted).toBe(true)
|
||||
})
|
||||
|
||||
test('should display stats cards', async ({ mcpLogsPage }) => {
|
||||
// Stats cards are only visible when MCP log data exists
|
||||
const tableExists = await mcpLogsPage.logsTable.isVisible().catch(() => false)
|
||||
if (!tableExists) {
|
||||
test.skip(true, 'No MCP logs — stats cards not rendered in getting-started view')
|
||||
return
|
||||
}
|
||||
const statsVisible = await mcpLogsPage.areStatsVisible()
|
||||
expect(statsVisible).toBe(true)
|
||||
})
|
||||
|
||||
test('should display filters section', async ({ mcpLogsPage }) => {
|
||||
// Filters are only visible when MCP log data exists (not in getting-started view)
|
||||
const tableExists = await mcpLogsPage.logsTable.isVisible().catch(() => false)
|
||||
if (!tableExists) {
|
||||
test.skip(true, 'No MCP logs — filters not rendered in getting-started view')
|
||||
return
|
||||
}
|
||||
const searchVisible = await mcpLogsPage.searchInput.isVisible().catch(() => false)
|
||||
const filtersButtonVisible = await mcpLogsPage.filtersButton.isVisible().catch(() => false)
|
||||
expect(searchVisible || filtersButtonVisible).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('MCP Log Filtering', () => {
|
||||
test('should filter logs by tool name', async ({ mcpLogsPage, page }) => {
|
||||
const filtersVisible = await mcpLogsPage.filtersButton.isVisible().catch(() => false)
|
||||
if (!filtersVisible) {
|
||||
test.skip(true, 'Filters button not visible')
|
||||
return
|
||||
}
|
||||
const applied = await mcpLogsPage.filterByToolName()
|
||||
if (!applied) {
|
||||
test.skip(true, 'No tool name options in filter list')
|
||||
return
|
||||
}
|
||||
await expect
|
||||
.poll(() => page.url(), { timeout: 5000, intervals: [200, 300, 500] })
|
||||
.toMatch(/tool_names=/)
|
||||
})
|
||||
|
||||
test('should filter logs by server label', async ({ mcpLogsPage, page }) => {
|
||||
const filtersVisible = await mcpLogsPage.filtersButton.isVisible().catch(() => false)
|
||||
if (!filtersVisible) {
|
||||
test.skip(true, 'Filters button not visible')
|
||||
return
|
||||
}
|
||||
const applied = await mcpLogsPage.filterByServerLabel()
|
||||
if (!applied) {
|
||||
test.skip(true, 'No server label options in filter list')
|
||||
return
|
||||
}
|
||||
await expect
|
||||
.poll(() => page.url(), { timeout: 5000, intervals: [200, 300, 500] })
|
||||
.toMatch(/server_labels=/)
|
||||
})
|
||||
|
||||
test('should filter logs by status', async ({ mcpLogsPage, page }) => {
|
||||
const filtersVisible = await mcpLogsPage.filtersButton.isVisible().catch(() => false)
|
||||
if (!filtersVisible) {
|
||||
test.skip(true, 'Filters button not visible')
|
||||
return
|
||||
}
|
||||
|
||||
const applied = await mcpLogsPage.filterByStatus('success')
|
||||
if (!applied) {
|
||||
test.skip(true, 'No status options in filter list')
|
||||
return
|
||||
}
|
||||
|
||||
await expect
|
||||
.poll(() => page.url(), { timeout: 5000, intervals: [200, 300, 500] })
|
||||
.toMatch(/status=success/)
|
||||
})
|
||||
|
||||
test('should search logs by content', async ({ mcpLogsPage }) => {
|
||||
const searchInput = mcpLogsPage.searchInput
|
||||
const isVisible = await searchInput.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
const query = `test-query-${Date.now()}`
|
||||
await mcpLogsPage.searchLogs(query)
|
||||
|
||||
// Check that search input contains the query (DOM state)
|
||||
const inputValue = await searchInput.inputValue().catch(() => '')
|
||||
expect(inputValue).toContain(query)
|
||||
}
|
||||
})
|
||||
|
||||
test('should filter by time period', async ({ mcpLogsPage }) => {
|
||||
const datePicker = mcpLogsPage.dateRangePicker
|
||||
const isVisible = await datePicker.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
// Get initial date picker value
|
||||
const initialValue = await datePicker.textContent().catch(() => '')
|
||||
|
||||
await mcpLogsPage.selectTimePeriod('7d')
|
||||
|
||||
// Check that date picker value changed (DOM state)
|
||||
const newValue = await datePicker.textContent().catch(() => '')
|
||||
expect(newValue || initialValue).toBeTruthy()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('MCP Log Details', () => {
|
||||
test('should open log details sheet', async ({ mcpLogsPage }) => {
|
||||
const logCount = await mcpLogsPage.getLogCount()
|
||||
|
||||
if (logCount > 0) {
|
||||
await mcpLogsPage.viewLogDetails(0)
|
||||
|
||||
const sheetVisible = await mcpLogsPage.logDetailSheet.isVisible().catch(() => false)
|
||||
expect(sheetVisible).toBe(true)
|
||||
|
||||
await mcpLogsPage.closeLogDetails()
|
||||
}
|
||||
})
|
||||
|
||||
test('should close log details sheet', async ({ mcpLogsPage }) => {
|
||||
const logCount = await mcpLogsPage.getLogCount()
|
||||
|
||||
if (logCount > 0) {
|
||||
await mcpLogsPage.viewLogDetails(0)
|
||||
await mcpLogsPage.closeLogDetails()
|
||||
|
||||
const sheetVisible = await mcpLogsPage.logDetailSheet.isVisible().catch(() => false)
|
||||
expect(sheetVisible).toBe(false)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Pagination', () => {
|
||||
test('should navigate to next page', async ({ mcpLogsPage }) => {
|
||||
const paginationVisible = await mcpLogsPage.paginationControls.isVisible().catch(() => false)
|
||||
if (!paginationVisible) {
|
||||
test.skip(true, 'No MCP logs — pagination not rendered')
|
||||
return
|
||||
}
|
||||
const nextBtn = mcpLogsPage.nextPageBtn
|
||||
const isEnabled = await nextBtn.isEnabled().catch(() => false)
|
||||
|
||||
if (!isEnabled) {
|
||||
test.skip(true, 'Only one page of results; skipping pagination test')
|
||||
return
|
||||
}
|
||||
|
||||
const initialPage = mcpLogsPage.getCurrentPageNumber()
|
||||
await mcpLogsPage.goToNextPage()
|
||||
|
||||
await expect
|
||||
.poll(() => mcpLogsPage.getCurrentPageNumber(), { timeout: 5000 })
|
||||
.toBe(initialPage + 1)
|
||||
})
|
||||
|
||||
test('should navigate to previous page', async ({ mcpLogsPage }) => {
|
||||
const paginationVisible = await mcpLogsPage.paginationControls.isVisible().catch(() => false)
|
||||
if (!paginationVisible) {
|
||||
test.skip(true, 'No MCP logs — pagination not rendered')
|
||||
return
|
||||
}
|
||||
const nextBtn = mcpLogsPage.nextPageBtn
|
||||
const nextEnabled = await nextBtn.isEnabled().catch(() => false)
|
||||
|
||||
if (!nextEnabled) {
|
||||
test.skip(true, 'Only one page of results; skipping pagination test')
|
||||
return
|
||||
}
|
||||
|
||||
await mcpLogsPage.goToNextPage()
|
||||
|
||||
await expect
|
||||
.poll(() => mcpLogsPage.getCurrentPageNumber(), { timeout: 5000 })
|
||||
.toBe(2)
|
||||
|
||||
const prevBtn = mcpLogsPage.prevPageBtn
|
||||
const prevEnabled = await prevBtn.isEnabled().catch(() => false)
|
||||
|
||||
if (!prevEnabled) {
|
||||
test.skip(true, 'Only one page of results; skipping previous-page test')
|
||||
return
|
||||
}
|
||||
|
||||
await mcpLogsPage.goToPreviousPage()
|
||||
|
||||
// We were on page 2; after previous we must be on page 1 (assert concrete value to avoid race with captured page number)
|
||||
await expect
|
||||
.poll(() => mcpLogsPage.getCurrentPageNumber(), { timeout: 5000 })
|
||||
.toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Table Sorting', () => {
|
||||
test('should sort by timestamp', async ({ mcpLogsPage }) => {
|
||||
const tableExists = await mcpLogsPage.logsTable.isVisible().catch(() => false)
|
||||
if (!tableExists) {
|
||||
test.skip(true, 'No MCP logs — table not rendered')
|
||||
return
|
||||
}
|
||||
// Timestamp is the default sort column (desc), so clicking it toggles to asc
|
||||
const initialUrl = mcpLogsPage.page.url()
|
||||
|
||||
await mcpLogsPage.sortBy('timestamp')
|
||||
|
||||
// Wait for URL to actually change after sort
|
||||
await expect
|
||||
.poll(() => mcpLogsPage.page.url(), { timeout: 5000 })
|
||||
.not.toBe(initialUrl)
|
||||
})
|
||||
|
||||
test('should sort by latency', async ({ mcpLogsPage }) => {
|
||||
const tableExists = await mcpLogsPage.logsTable.isVisible().catch(() => false)
|
||||
if (!tableExists) {
|
||||
test.skip(true, 'No MCP logs — table not rendered')
|
||||
return
|
||||
}
|
||||
await mcpLogsPage.sortBy('latency')
|
||||
|
||||
// Wait for URL to update
|
||||
await mcpLogsPage.page.waitForURL(/sort_by=latency/, { timeout: 5000 })
|
||||
|
||||
// Check URL state for latency sort
|
||||
const sortState = await mcpLogsPage.getSortState('latency')
|
||||
expect(sortState).toBeTruthy()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Live Updates', () => {
|
||||
test('should toggle live updates', async ({ mcpLogsPage }) => {
|
||||
const tableExists = await mcpLogsPage.logsTable.isVisible().catch(() => false)
|
||||
if (!tableExists) {
|
||||
test.skip(true, 'No MCP logs — live toggle not rendered in getting-started view')
|
||||
return
|
||||
}
|
||||
const liveToggle = mcpLogsPage.liveToggle
|
||||
const isVisible = await liveToggle.isVisible().catch(() => false)
|
||||
|
||||
if (isVisible) {
|
||||
// Default is live_enabled=true (but URL may not have it since it's the default)
|
||||
// Check for live_enabled=false to determine if currently disabled
|
||||
const initialUrl = mcpLogsPage.page.url()
|
||||
const initialLiveDisabled = initialUrl.includes('live_enabled=false')
|
||||
|
||||
await mcpLogsPage.toggleLiveUpdates()
|
||||
|
||||
// Wait for URL to reflect live_enabled toggle
|
||||
await mcpLogsPage.page.waitForURL(/live_enabled=/, { timeout: 5000 })
|
||||
|
||||
const newUrl = mcpLogsPage.page.url()
|
||||
const newLiveDisabled = newUrl.includes('live_enabled=false')
|
||||
|
||||
// Live enabled state should have toggled
|
||||
// If initially enabled (not disabled), after toggle it should be disabled
|
||||
// If initially disabled, after toggle it should be enabled (no live_enabled=false)
|
||||
expect(newLiveDisabled).not.toBe(initialLiveDisabled)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
392
tests/e2e/features/mcp-logs/pages/mcp-logs.page.ts
Normal file
392
tests/e2e/features/mcp-logs/pages/mcp-logs.page.ts
Normal file
@@ -0,0 +1,392 @@
|
||||
import { Locator, Page, expect } from '@playwright/test'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
/**
|
||||
* Page object for the MCP Logs page
|
||||
*/
|
||||
export class MCPLogsPage extends BasePage {
|
||||
// Main elements
|
||||
readonly logsTable: Locator
|
||||
readonly filtersSection: Locator
|
||||
readonly filtersButton: Locator
|
||||
readonly statsCards: Locator
|
||||
|
||||
// Filter elements
|
||||
readonly toolNameFilter: Locator
|
||||
readonly serverLabelFilter: Locator
|
||||
readonly statusFilter: Locator
|
||||
readonly searchInput: Locator
|
||||
readonly dateRangePicker: Locator
|
||||
readonly liveToggle: Locator
|
||||
|
||||
// Table elements
|
||||
readonly tableRows: Locator
|
||||
readonly paginationControls: Locator
|
||||
readonly nextPageBtn: Locator
|
||||
readonly prevPageBtn: Locator
|
||||
|
||||
// Log detail sheet
|
||||
readonly logDetailSheet: Locator
|
||||
readonly closeDetailSheetBtn: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
|
||||
// Main elements
|
||||
this.logsTable = page.locator('[data-testid="mcp-logs-table"]').or(page.locator('table'))
|
||||
// The filters section is the container with search input and filters button
|
||||
this.filtersSection = page.locator('input[placeholder="Search MCP logs"]').locator('..')
|
||||
this.filtersButton = page.getByRole('button', { name: /Filters/i })
|
||||
this.statsCards = page.locator('[data-testid="mcp-stats-cards"]').or(
|
||||
page.locator('text=Total Executions').locator('..').locator('..')
|
||||
)
|
||||
|
||||
// Filter elements - filters are inside a popover opened by the Filters button
|
||||
this.toolNameFilter = page.locator('[data-testid="filter-tool-name"]').or(
|
||||
page.locator('button').filter({ hasText: /Tool Name/i })
|
||||
)
|
||||
this.serverLabelFilter = page.locator('[data-testid="filter-server-label"]').or(
|
||||
page.locator('button').filter({ hasText: /Server/i })
|
||||
)
|
||||
this.statusFilter = page.locator('[data-testid="filter-status"]').or(
|
||||
page.locator('button').filter({ hasText: /Status/i })
|
||||
)
|
||||
this.searchInput = page.locator('[data-testid="filter-search"]').or(
|
||||
page.getByPlaceholder('Search MCP logs')
|
||||
)
|
||||
this.dateRangePicker = page.locator('[data-testid="filter-date-range"]').or(
|
||||
page.locator('button').filter({ hasText: /Last/i })
|
||||
)
|
||||
this.liveToggle = page.locator('[data-testid="live-toggle"]').or(
|
||||
page.getByRole('button', { name: /Live updates/i })
|
||||
)
|
||||
|
||||
// Table elements - exclude status message rows
|
||||
this.tableRows = this.logsTable.locator('tbody tr').filter({ hasNot: page.locator('text=Listening for') }).filter({ hasNot: page.locator('text=Live updates paused') }).filter({ hasNot: page.locator('text=Not connected') }).filter({ hasNot: page.locator('text=No results found') })
|
||||
// Scope pagination to the MCP logs view (avoid matching other pages when navigating)
|
||||
const paginationContainer = page.getByTestId('pagination').filter({ has: page.locator('[data-testid="next-page"]') }).first()
|
||||
this.paginationControls = paginationContainer
|
||||
this.nextPageBtn = paginationContainer.getByRole('button', { name: 'Next page' }).or(
|
||||
paginationContainer.locator('[data-testid="next-page"]')
|
||||
)
|
||||
this.prevPageBtn = paginationContainer.getByRole('button', { name: 'Previous page' }).or(
|
||||
paginationContainer.locator('[data-testid="prev-page"]')
|
||||
)
|
||||
|
||||
// Log detail sheet - Sheet component with role="dialog"
|
||||
this.logDetailSheet = page.locator('[role="dialog"]')
|
||||
this.closeDetailSheetBtn = page.locator('[role="dialog"]').locator('button').filter({ has: page.locator('svg.lucide-x') })
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to the MCP logs page
|
||||
*/
|
||||
async goto(): Promise<void> {
|
||||
await this.page.goto('/workspace/mcp-logs')
|
||||
await waitForNetworkIdle(this.page)
|
||||
// Wait for table to be visible
|
||||
await this.logsTable.waitFor({ state: 'visible', timeout: 10000 }).catch(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Open Filters popover and wait for the command list. Caller can then resolve group/option locators.
|
||||
*/
|
||||
private async openFiltersPopover(): Promise<void> {
|
||||
await this.filtersButton.first().waitFor({ state: 'visible' })
|
||||
await this.filtersButton.first().click()
|
||||
await this.page.waitForSelector('[role="listbox"], [data-slot="command-list"]', { timeout: 5000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Close Filters popover (Escape) and wait for network idle.
|
||||
*/
|
||||
private async closeFiltersPopover(): Promise<void> {
|
||||
await this.page.keyboard.press('Escape')
|
||||
await this.page.waitForSelector('[role="listbox"]', { state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the first selectable option in a filter group by heading (e.g. "Tool Names", "Servers").
|
||||
* Skips "Loading..." so we only click real options.
|
||||
*/
|
||||
private async getFirstOptionInGroup(groupHeading: string): Promise<Locator | null> {
|
||||
const list = this.page.locator('[data-slot="command-list"]').or(this.page.locator('[role="listbox"]'))
|
||||
const group = list.locator('[data-slot="command-group"]').filter({
|
||||
has: this.page.getByText(groupHeading, { exact: true }),
|
||||
})
|
||||
const items = group.locator('[data-slot="command-item"]').or(group.getByRole('option'))
|
||||
const count = await items.count()
|
||||
for (let i = 0; i < count; i++) {
|
||||
const item = items.nth(i)
|
||||
const text = await item.textContent().catch(() => '')
|
||||
if (text && !/loading/i.test(text)) {
|
||||
return item
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Open Filters popover and click an option by name. Returns true if the option was found and clicked.
|
||||
*/
|
||||
private async openFiltersAndSelectOption(optionText: string | RegExp): Promise<boolean> {
|
||||
await this.openFiltersPopover()
|
||||
const re = typeof optionText === 'string' ? new RegExp(optionText, 'i') : optionText
|
||||
const option = this.page.getByRole('option', { name: re })
|
||||
const count = await option.count()
|
||||
if (count > 0) {
|
||||
await option.first().click()
|
||||
await this.closeFiltersPopover()
|
||||
return true
|
||||
}
|
||||
await this.closeFiltersPopover()
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter by tool name: open Filters and select the first available tool name option.
|
||||
* @returns true if at least one tool name option was found and selected
|
||||
*/
|
||||
async filterByToolName(): Promise<boolean> {
|
||||
await this.openFiltersPopover()
|
||||
const first = await this.getFirstOptionInGroup('Tool Names')
|
||||
if (!first) {
|
||||
await this.closeFiltersPopover()
|
||||
return false
|
||||
}
|
||||
await first.click()
|
||||
await this.closeFiltersPopover()
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter by server label: open Filters and select the first available server label option.
|
||||
* @returns true if at least one server label option was found and selected
|
||||
*/
|
||||
async filterByServerLabel(): Promise<boolean> {
|
||||
await this.openFiltersPopover()
|
||||
const first = await this.getFirstOptionInGroup('Servers')
|
||||
if (!first) {
|
||||
await this.closeFiltersPopover()
|
||||
return false
|
||||
}
|
||||
await first.click()
|
||||
await this.closeFiltersPopover()
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter by status. Opens Filters popover and toggles the given status option (e.g. success, error).
|
||||
* @returns true if the option was found and clicked
|
||||
*/
|
||||
async filterByStatus(status: 'success' | 'error' | 'pending'): Promise<boolean> {
|
||||
return this.openFiltersAndSelectOption(status)
|
||||
}
|
||||
|
||||
/**
|
||||
* Search logs by content
|
||||
*/
|
||||
async searchLogs(query: string): Promise<void> {
|
||||
await this.searchInput.fill(query)
|
||||
// Wait for debounced search to trigger network request
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear search
|
||||
*/
|
||||
async clearSearch(): Promise<void> {
|
||||
await this.searchInput.clear()
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Select time period
|
||||
*/
|
||||
async selectTimePeriod(period: '1h' | '6h' | '24h' | '7d' | '30d'): Promise<void> {
|
||||
await this.dateRangePicker.first().click()
|
||||
await this.page.waitForSelector('[role="listbox"], [role="menu"]', { timeout: 5000 }).catch(() => {})
|
||||
|
||||
const periodLabels: Record<string, string> = {
|
||||
'1h': 'Last hour',
|
||||
'6h': 'Last 6 hours',
|
||||
'24h': 'Last 24 hours',
|
||||
'7d': 'Last 7 days',
|
||||
'30d': 'Last 30 days',
|
||||
}
|
||||
|
||||
const periodButton = this.page.getByRole('button', { name: periodLabels[period] })
|
||||
if (await periodButton.count() > 0) {
|
||||
await periodButton.click()
|
||||
} else {
|
||||
await this.page.keyboard.press('Escape')
|
||||
}
|
||||
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle live updates
|
||||
*/
|
||||
async toggleLiveUpdates(): Promise<void> {
|
||||
await this.liveToggle.first().waitFor({ state: 'visible' })
|
||||
await this.liveToggle.first().click()
|
||||
}
|
||||
|
||||
/**
|
||||
* Click on a log row to view details
|
||||
*/
|
||||
async viewLogDetails(rowIndex: number = 0): Promise<void> {
|
||||
const rows = this.tableRows
|
||||
const count = await rows.count()
|
||||
|
||||
if (count <= rowIndex) {
|
||||
throw new Error(`Row index ${rowIndex} out of bounds (${count} rows available)`)
|
||||
}
|
||||
await rows.nth(rowIndex).click()
|
||||
await expect(this.logDetailSheet).toBeVisible({ timeout: 5000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Close log detail sheet
|
||||
*/
|
||||
async closeLogDetails(): Promise<void> {
|
||||
if (await this.logDetailSheet.isVisible()) {
|
||||
await this.closeDetailSheetBtn.click().catch(async () => {
|
||||
await this.page.keyboard.press('Escape')
|
||||
})
|
||||
await expect(this.logDetailSheet).not.toBeVisible({ timeout: 5000 })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get log count from table
|
||||
*/
|
||||
async getLogCount(): Promise<number> {
|
||||
return await this.tableRows.count()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if log exists in table
|
||||
*/
|
||||
async logExists(searchText: string): Promise<boolean> {
|
||||
const row = this.tableRows.filter({ hasText: searchText })
|
||||
return await row.count() > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current 1-based page number from URL (offset/limit).
|
||||
*/
|
||||
getCurrentPageNumber(): number {
|
||||
const url = this.page.url()
|
||||
const params = new URL(url).searchParams
|
||||
const offset = Number.parseInt(params.get('offset') ?? '0', 10)
|
||||
const limit = Number.parseInt(params.get('limit') ?? '50', 10) || 50
|
||||
return Math.floor(offset / limit) + 1
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to next page (waits for URL to update)
|
||||
*/
|
||||
async goToNextPage(): Promise<void> {
|
||||
const btn = this.nextPageBtn.first()
|
||||
const isEnabled = await btn.isEnabled().catch(() => false)
|
||||
if (!isEnabled) return
|
||||
await btn.scrollIntoViewIfNeeded()
|
||||
await btn.waitFor({ state: 'visible' })
|
||||
const limit = Number.parseInt(new URL(this.page.url()).searchParams.get('limit') ?? '50', 10) || 50
|
||||
const currentOffset = Number.parseInt(new URL(this.page.url()).searchParams.get('offset') ?? '0', 10)
|
||||
const expectedOffset = currentOffset + limit
|
||||
await btn.click()
|
||||
await this.page.waitForURL((url) => {
|
||||
const params = new URL(url).searchParams
|
||||
const offset = params.get('offset')
|
||||
return offset === String(expectedOffset)
|
||||
}, { timeout: 10000 })
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to previous page (waits for URL to update)
|
||||
*/
|
||||
async goToPreviousPage(): Promise<void> {
|
||||
const btn = this.prevPageBtn.first()
|
||||
const isEnabled = await btn.isEnabled().catch(() => false)
|
||||
if (!isEnabled) return
|
||||
await btn.scrollIntoViewIfNeeded()
|
||||
await btn.waitFor({ state: 'visible' })
|
||||
const limit = Number.parseInt(new URL(this.page.url()).searchParams.get('limit') ?? '50', 10) || 50
|
||||
const currentOffset = Number.parseInt(new URL(this.page.url()).searchParams.get('offset') ?? '0', 10)
|
||||
const expectedOffset = Math.max(0, currentOffset - limit)
|
||||
await btn.click()
|
||||
await this.page.waitForURL((url) => {
|
||||
const params = new URL(url).searchParams
|
||||
const offset = params.get('offset')
|
||||
// When going back to page 1, offset param may be removed (null) or set to "0"
|
||||
if (expectedOffset === 0) return offset === null || offset === '0'
|
||||
return offset === String(expectedOffset)
|
||||
}, { timeout: 10000 })
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort table by column - clicks the sort button in the column header
|
||||
*/
|
||||
async sortBy(column: 'timestamp' | 'latency'): Promise<void> {
|
||||
await this.dismissToasts()
|
||||
|
||||
// Map column names to header button text
|
||||
const columnLabels: Record<string, string> = {
|
||||
'timestamp': 'Time',
|
||||
'latency': 'Latency'
|
||||
}
|
||||
|
||||
const label = columnLabels[column] || column
|
||||
// The sortable column headers have a button with the column name
|
||||
const sortButton = this.logsTable.getByRole('button', { name: new RegExp(label, 'i') })
|
||||
|
||||
if (await sortButton.count() > 0) {
|
||||
await sortButton.first().waitFor({ state: 'visible' })
|
||||
await sortButton.first().click()
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if stats cards are visible
|
||||
*/
|
||||
async areStatsVisible(): Promise<boolean> {
|
||||
// MCP logs page shows "Total Executions" not "Total Requests"
|
||||
const statsText = this.page.locator('text=Total Executions')
|
||||
return await statsText.isVisible().catch(() => false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if empty state is shown
|
||||
*/
|
||||
async isEmptyStateVisible(): Promise<boolean> {
|
||||
const emptyState = this.page.locator('text=/No logs found/i').or(
|
||||
this.page.locator('text=/No data/i')
|
||||
)
|
||||
return await emptyState.isVisible().catch(() => false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sort state for a column from URL parameters
|
||||
* Returns 'asc', 'desc', or null if column is not the current sort column
|
||||
*/
|
||||
async getSortState(column: 'timestamp' | 'latency'): Promise<string | null> {
|
||||
const url = this.page.url()
|
||||
const urlParams = new URL(url).searchParams
|
||||
const sortBy = urlParams.get('sort_by')
|
||||
const order = urlParams.get('order')
|
||||
|
||||
// Check if this column is the currently sorted column
|
||||
if (sortBy === column) {
|
||||
return order || 'desc' // default is desc
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
167
tests/e2e/features/mcp-registry/mcp-registry.data.ts
Normal file
167
tests/e2e/features/mcp-registry/mcp-registry.data.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
import { join, resolve } from 'path'
|
||||
import { MCPClientConfig, EnvVarLike } from './pages/mcp-registry.page'
|
||||
|
||||
/** Normalize header value from env (string or EnvVarLike) to EnvVarLike */
|
||||
function toEnvVarLike(v: string | EnvVarLike): EnvVarLike {
|
||||
if (typeof v === 'object' && v !== null && 'value' in v) return v as EnvVarLike
|
||||
return { value: String(v), env_var: '', from_env: false }
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve header value: if string starts with "env.", use process.env[VAR_NAME].
|
||||
*/
|
||||
function resolveHeaderValue(v: EnvVarLike): EnvVarLike {
|
||||
if (v.value.startsWith('env.')) {
|
||||
const envVar = v.value.slice(4)
|
||||
const resolved = process.env[envVar]
|
||||
if (resolved !== undefined) {
|
||||
return { value: resolved, env_var: envVar, from_env: true }
|
||||
}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse MCP_SSE_HEADERS: supports single object, array of objects, or concatenated objects.
|
||||
* e.g. {"Authorization":"Bearer ..."},{"ENV_EXA_API_KEY":"..."} → merged into one record
|
||||
*/
|
||||
function parseSSEHeadersRaw(raw: string): Record<string, string | EnvVarLike> {
|
||||
const trimmed = raw.trim()
|
||||
if (!trimmed) return {}
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed)
|
||||
if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
|
||||
return parsed
|
||||
}
|
||||
} catch {
|
||||
// Fallback: concatenated objects {"a":1},{"b":2} → wrap in [ ] and merge
|
||||
}
|
||||
try {
|
||||
const asArray = JSON.parse('[' + trimmed + ']')
|
||||
if (Array.isArray(asArray) && asArray.every((o) => typeof o === 'object' && o !== null)) {
|
||||
return Object.assign({}, ...asArray)
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create basic MCP client data
|
||||
*/
|
||||
export function createMCPClientData(overrides: Partial<MCPClientConfig> = {}): MCPClientConfig {
|
||||
return {
|
||||
name: `test_client_${Date.now()}`,
|
||||
connectionType: 'http',
|
||||
connectionUrl: 'http://localhost:3001/',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create HTTP MCP client data
|
||||
* Uses http-no-ping-server from examples/mcps
|
||||
*/
|
||||
export function createHTTPClientData(overrides: Partial<MCPClientConfig> = {}): MCPClientConfig {
|
||||
return createMCPClientData({
|
||||
connectionType: 'http',
|
||||
connectionUrl: 'http://localhost:3001/', // http-no-ping-server
|
||||
authType: 'none',
|
||||
isPingAvailable: false, // http-no-ping-server doesn't support ping
|
||||
...overrides,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize parsed headers to EnvVarLike format (handles both plain values and nested EnvVar objects).
|
||||
*/
|
||||
function normalizeHeaders(parsed: Record<string, string | EnvVarLike>): Record<string, EnvVarLike> {
|
||||
const out: Record<string, EnvVarLike> = {}
|
||||
for (const [k, v] of Object.entries(parsed)) {
|
||||
if (v !== undefined && v !== null) {
|
||||
out[k] = resolveHeaderValue(toEnvVarLike(v))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Create SSE MCP client data.
|
||||
* URL and headers are injected via workflow env: MCP_SSE_URL, MCP_SSE_HEADERS (JSON).
|
||||
* Supports: {"Authorization":"Bearer ...","K":"V"} or {"Authorization":{"value":"...","env_var":"","from_env":false}}.
|
||||
*/
|
||||
export function createSSEClientData(overrides: Partial<MCPClientConfig> = {}): MCPClientConfig {
|
||||
const connectionUrl = "https://ts-mcp-sse-proxy.fly.dev/npx%20-y%20exa-mcp-server/sse"
|
||||
const raw = process.env.MCP_SSE_HEADERS
|
||||
const parsed = raw ? parseSSEHeadersRaw(raw) : {}
|
||||
const headers = normalizeHeaders(parsed)
|
||||
return createMCPClientData({
|
||||
connectionType: 'sse',
|
||||
connectionUrl,
|
||||
authType: 'headers',
|
||||
headers: headers,
|
||||
isPingAvailable: false,
|
||||
...overrides,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Create STDIO MCP client data
|
||||
* Uses test-tools-server from examples/mcps
|
||||
*/
|
||||
export function createSTDIOClientData(overrides: Partial<MCPClientConfig> = {}): MCPClientConfig {
|
||||
// Use the built test-tools-server
|
||||
const REPO_ROOT = resolve(__dirname, '..', '..', '..', '..')
|
||||
|
||||
// Then for the stdio server:
|
||||
const serverPath = join(REPO_ROOT, 'examples', 'mcps', 'test-tools-server', 'dist', 'index.js')
|
||||
return createMCPClientData({
|
||||
name: `stdio_client_${Date.now()}`,
|
||||
connectionType: 'stdio',
|
||||
command: 'node',
|
||||
args: serverPath, // Run the actual MCP server
|
||||
...overrides,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Create HTTP client with headers auth
|
||||
*/
|
||||
export function createHTTPClientWithHeaders(overrides: Partial<MCPClientConfig> = {}): MCPClientConfig {
|
||||
return createMCPClientData({
|
||||
connectionType: 'http',
|
||||
connectionUrl: 'http://localhost:3001/', // http-no-ping-server
|
||||
authType: 'headers',
|
||||
isPingAvailable: false,
|
||||
...overrides,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Create HTTP client with OAuth auth (minimal config for testing)
|
||||
* Note: http-no-ping-server doesn't support OAuth, so this is for UI testing only
|
||||
*/
|
||||
export function createHTTPClientWithOAuth(overrides: Partial<MCPClientConfig> = {}): MCPClientConfig {
|
||||
return createMCPClientData({
|
||||
name: `oauth_client_${Date.now()}`,
|
||||
connectionType: 'http',
|
||||
connectionUrl: 'http://localhost:3001/', // http-no-ping-server
|
||||
authType: 'oauth',
|
||||
oauthClientId: 'test-client-id',
|
||||
oauthAuthorizeUrl: 'http://localhost:3001/oauth/authorize',
|
||||
oauthTokenUrl: 'http://localhost:3001/oauth/token',
|
||||
isPingAvailable: false,
|
||||
...overrides,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Create client with code mode enabled
|
||||
*/
|
||||
export function createCodeModeClientData(overrides: Partial<MCPClientConfig> = {}): MCPClientConfig {
|
||||
return createMCPClientData({
|
||||
isCodeMode: true,
|
||||
...overrides,
|
||||
})
|
||||
}
|
||||
357
tests/e2e/features/mcp-registry/mcp-registry.spec.ts
Normal file
357
tests/e2e/features/mcp-registry/mcp-registry.spec.ts
Normal file
@@ -0,0 +1,357 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
import {
|
||||
createCodeModeClientData,
|
||||
createHTTPClientData,
|
||||
createSSEClientData,
|
||||
createSTDIOClientData
|
||||
} from './mcp-registry.data'
|
||||
|
||||
const hasSSEHeaders = Boolean(process.env.MCP_SSE_HEADERS)
|
||||
|
||||
// Track created clients for cleanup
|
||||
const createdClients: string[] = []
|
||||
|
||||
test.describe('MCP Registry', () => {
|
||||
// MCP client creation can be slow (backend connects to MCP server); give tests room to complete
|
||||
test.setTimeout(120000)
|
||||
|
||||
test.beforeEach(async ({ mcpRegistryPage }) => {
|
||||
await mcpRegistryPage.goto()
|
||||
})
|
||||
|
||||
test.afterEach(async ({ mcpRegistryPage }) => {
|
||||
const toClean = [...createdClients]
|
||||
createdClients.length = 0
|
||||
if (toClean.length > 0) {
|
||||
await mcpRegistryPage.cleanupMCPClients(toClean)
|
||||
}
|
||||
})
|
||||
|
||||
test.describe('MCP Client Display', () => {
|
||||
test('should display MCP clients table', async ({ mcpRegistryPage }) => {
|
||||
await expect(mcpRegistryPage.table).toBeVisible()
|
||||
})
|
||||
|
||||
test('should display create button', async ({ mcpRegistryPage }) => {
|
||||
await expect(mcpRegistryPage.createBtn).toBeVisible()
|
||||
})
|
||||
|
||||
test('should show empty state or client list', async ({ mcpRegistryPage }) => {
|
||||
const count = await mcpRegistryPage.getClientCount()
|
||||
const isEmptyStateVisible = await mcpRegistryPage.isEmptyStateVisible()
|
||||
|
||||
if (count === 0) {
|
||||
expect(isEmptyStateVisible).toBe(true)
|
||||
} else {
|
||||
expect(count).toBeGreaterThan(0)
|
||||
expect(isEmptyStateVisible).toBe(false)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('MCP Client Creation', () => {
|
||||
test('should open client creation sheet', async ({ mcpRegistryPage }) => {
|
||||
await mcpRegistryPage.createBtn.click()
|
||||
await expect(mcpRegistryPage.sheet).toBeVisible()
|
||||
await expect(mcpRegistryPage.nameInput).toBeVisible()
|
||||
|
||||
// Cancel to clean up
|
||||
await mcpRegistryPage.cancelCreation()
|
||||
})
|
||||
|
||||
test('should create basic HTTP client', async ({ mcpRegistryPage }) => {
|
||||
const clientData = createHTTPClientData()
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed
|
||||
|
||||
createdClients.push(clientData.name)
|
||||
const exists = await mcpRegistryPage.clientExists(clientData.name)
|
||||
expect(exists).toBe(true)
|
||||
|
||||
// Verify connection type displayed correctly
|
||||
const connectionType = await mcpRegistryPage.getClientConnectionType(clientData.name)
|
||||
expect(connectionType).toBe('HTTP')
|
||||
})
|
||||
|
||||
test('should create SSE client', async ({ mcpRegistryPage }) => {
|
||||
test.skip(!hasSSEHeaders, 'Requires MCP_SSE_HEADERS for authenticated SSE MCP endpoint')
|
||||
const clientData = createSSEClientData({
|
||||
name: `sse_test_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed
|
||||
|
||||
createdClients.push(clientData.name)
|
||||
const exists = await mcpRegistryPage.clientExists(clientData.name)
|
||||
expect(exists).toBe(true)
|
||||
|
||||
// Verify connection type displayed correctly
|
||||
const connectionType = await mcpRegistryPage.getClientConnectionType(clientData.name)
|
||||
expect(connectionType).toBe('SSE')
|
||||
})
|
||||
|
||||
test('should create STDIO client with command', async ({ mcpRegistryPage }) => {
|
||||
const clientData = createSTDIOClientData()
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed
|
||||
|
||||
createdClients.push(clientData.name)
|
||||
const exists = await mcpRegistryPage.clientExists(clientData.name)
|
||||
expect(exists).toBe(true)
|
||||
})
|
||||
|
||||
test('should create client with code mode enabled', async ({ mcpRegistryPage }) => {
|
||||
const clientData = createCodeModeClientData({
|
||||
name: `codemode_test_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed
|
||||
|
||||
createdClients.push(clientData.name)
|
||||
const exists = await mcpRegistryPage.clientExists(clientData.name)
|
||||
expect(exists).toBe(true)
|
||||
})
|
||||
|
||||
test('should cancel client creation', async ({ mcpRegistryPage }) => {
|
||||
await mcpRegistryPage.createBtn.click()
|
||||
await expect(mcpRegistryPage.sheet).toBeVisible()
|
||||
|
||||
const testName = `cancelled_client_${Date.now()}`
|
||||
await mcpRegistryPage.nameInput.fill(testName)
|
||||
|
||||
await mcpRegistryPage.cancelCreation()
|
||||
|
||||
// Sheet should be closed
|
||||
await expect(mcpRegistryPage.sheet).not.toBeVisible()
|
||||
|
||||
// Client should not exist
|
||||
const exists = await mcpRegistryPage.clientExists(testName)
|
||||
expect(exists).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('MCP Server Connection Validation', () => {
|
||||
test('should connect to HTTP server and list tools', async ({ mcpRegistryPage }) => {
|
||||
const clientData = createHTTPClientData({
|
||||
name: `http_validation_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true)
|
||||
createdClients.push(clientData.name)
|
||||
|
||||
// Wait a moment for connection to establish
|
||||
await mcpRegistryPage.page.waitForTimeout(2000)
|
||||
|
||||
// Verify client shows connection status
|
||||
const status = await mcpRegistryPage.getClientStatus(clientData.name)
|
||||
expect(status).toBeTruthy()
|
||||
// Status could be connecting, connected, or disconnected depending on timing
|
||||
expect(['connected', 'disconnected', 'connecting', 'error']).toContain(status.toLowerCase())
|
||||
|
||||
// Verify tools are loaded (http-no-ping-server has: echo, add, greet)
|
||||
await mcpRegistryPage.viewClientDetails(clientData.name)
|
||||
const toolsCount = await mcpRegistryPage.getToolsCount()
|
||||
expect(toolsCount).toBeGreaterThanOrEqual(3)
|
||||
|
||||
await mcpRegistryPage.closeDetailSheet()
|
||||
})
|
||||
|
||||
test('should connect to SSE server and list tools', async ({ mcpRegistryPage }) => {
|
||||
test.skip(!hasSSEHeaders, 'Requires MCP_SSE_HEADERS for authenticated SSE MCP endpoint')
|
||||
const clientData = createSSEClientData({
|
||||
name: `sse_validation_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true)
|
||||
createdClients.push(clientData.name)
|
||||
|
||||
// Wait a moment for connection to establish
|
||||
await mcpRegistryPage.page.waitForTimeout(2000)
|
||||
|
||||
// Verify tools are loaded
|
||||
await mcpRegistryPage.viewClientDetails(clientData.name)
|
||||
const toolsCount = await mcpRegistryPage.getToolsCount()
|
||||
expect(toolsCount).toBeGreaterThanOrEqual(3)
|
||||
|
||||
await mcpRegistryPage.closeDetailSheet()
|
||||
})
|
||||
|
||||
test('should connect to STDIO server and list tools', async ({ mcpRegistryPage }) => {
|
||||
const clientData = createSTDIOClientData({
|
||||
name: `stdio_validation_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true)
|
||||
createdClients.push(clientData.name)
|
||||
|
||||
// Wait a moment for connection to establish
|
||||
await mcpRegistryPage.page.waitForTimeout(2000)
|
||||
|
||||
// Verify tools from test-tools-server (echo, calculator, get_weather, delay, throw_error)
|
||||
await mcpRegistryPage.viewClientDetails(clientData.name)
|
||||
const toolsCount = await mcpRegistryPage.getToolsCount()
|
||||
expect(toolsCount).toBeGreaterThanOrEqual(5)
|
||||
|
||||
await mcpRegistryPage.closeDetailSheet()
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('MCP Client Management', () => {
|
||||
test('should delete MCP client', async ({ mcpRegistryPage }) => {
|
||||
// Create a client first using HTTP (most reliable)
|
||||
const clientData = createHTTPClientData({
|
||||
name: `delete_test_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed for this test
|
||||
|
||||
// Verify it exists
|
||||
let exists = await mcpRegistryPage.clientExists(clientData.name)
|
||||
expect(exists).toBe(true)
|
||||
|
||||
// Delete it
|
||||
await mcpRegistryPage.deleteClient(clientData.name)
|
||||
|
||||
// Verify it's gone
|
||||
exists = await mcpRegistryPage.clientExists(clientData.name)
|
||||
expect(exists).toBe(false)
|
||||
})
|
||||
|
||||
test('should view client details', async ({ mcpRegistryPage }) => {
|
||||
// Create a client first
|
||||
const clientData = createHTTPClientData({
|
||||
name: `view_test_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed for this test
|
||||
createdClients.push(clientData.name)
|
||||
|
||||
// View details
|
||||
await mcpRegistryPage.viewClientDetails(clientData.name)
|
||||
|
||||
// Detail sheet should be visible
|
||||
await expect(mcpRegistryPage.detailSheet).toBeVisible()
|
||||
|
||||
// Close the sheet
|
||||
await mcpRegistryPage.closeDetailSheet()
|
||||
})
|
||||
|
||||
test('should close client details sheet', async ({ mcpRegistryPage }) => {
|
||||
// Create a client first
|
||||
const clientData = createHTTPClientData({
|
||||
name: `close_sheet_test_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed for this test
|
||||
createdClients.push(clientData.name)
|
||||
|
||||
// Open details
|
||||
await mcpRegistryPage.viewClientDetails(clientData.name)
|
||||
await expect(mcpRegistryPage.detailSheet).toBeVisible()
|
||||
|
||||
// Close it
|
||||
await mcpRegistryPage.closeDetailSheet()
|
||||
|
||||
// Should be closed
|
||||
await expect(mcpRegistryPage.detailSheet).not.toBeVisible()
|
||||
})
|
||||
|
||||
test('should reconnect MCP client', async ({ mcpRegistryPage }) => {
|
||||
// Create a client first
|
||||
const clientData = createHTTPClientData({
|
||||
name: `reconnect_test_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed for this test
|
||||
createdClients.push(clientData.name)
|
||||
|
||||
// Reconnect - method waits for success toast
|
||||
await mcpRegistryPage.reconnectClient(clientData.name)
|
||||
|
||||
// Verify client still exists and has a status (reconnect completed)
|
||||
const exists = await mcpRegistryPage.clientExists(clientData.name)
|
||||
expect(exists).toBe(true)
|
||||
const status = await mcpRegistryPage.getClientStatus(clientData.name)
|
||||
expect(status).toBeTruthy()
|
||||
expect(['connected', 'disconnected', 'connecting']).toContain(status.toLowerCase())
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Client Status Display', () => {
|
||||
test('should display client connection status', async ({ mcpRegistryPage }) => {
|
||||
// Create a client first
|
||||
const clientData = createHTTPClientData({
|
||||
name: `status_test_${Date.now()}`,
|
||||
})
|
||||
|
||||
const created = await mcpRegistryPage.createClient(clientData)
|
||||
expect(created).toBe(true) // Client creation must succeed for this test
|
||||
createdClients.push(clientData.name)
|
||||
|
||||
// Get status
|
||||
const status = await mcpRegistryPage.getClientStatus(clientData.name)
|
||||
|
||||
// Status should be one of the expected values
|
||||
expect(status).toBeTruthy()
|
||||
expect(['connected', 'disconnected', 'connecting', 'error']).toContain(status?.toLowerCase())
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Form Validation', () => {
|
||||
test('should require name for client', async ({ mcpRegistryPage }) => {
|
||||
await mcpRegistryPage.createBtn.click()
|
||||
await expect(mcpRegistryPage.sheet).toBeVisible()
|
||||
|
||||
// Clear name field (should be empty by default)
|
||||
await mcpRegistryPage.nameInput.clear()
|
||||
|
||||
// Save button should be disabled when name is empty
|
||||
const saveBtn = mcpRegistryPage.saveBtn
|
||||
await expect(saveBtn).toBeDisabled()
|
||||
|
||||
await mcpRegistryPage.cancelCreation()
|
||||
})
|
||||
|
||||
test('should validate name format', async ({ mcpRegistryPage }) => {
|
||||
await mcpRegistryPage.createBtn.click()
|
||||
await expect(mcpRegistryPage.sheet).toBeVisible()
|
||||
|
||||
// Try invalid name with hyphens (not allowed)
|
||||
await mcpRegistryPage.nameInput.fill('invalid-name-with-hyphens')
|
||||
|
||||
// Fill connection URL to satisfy other validation
|
||||
await mcpRegistryPage.connectionUrlInput.fill('http://localhost:3001')
|
||||
|
||||
// Save button should be disabled due to validation error
|
||||
const saveBtn = mcpRegistryPage.saveBtn
|
||||
await expect(saveBtn).toBeDisabled()
|
||||
|
||||
await mcpRegistryPage.cancelCreation()
|
||||
})
|
||||
|
||||
test('should require connection URL for HTTP clients', async ({ mcpRegistryPage }) => {
|
||||
await mcpRegistryPage.createBtn.click()
|
||||
await expect(mcpRegistryPage.sheet).toBeVisible()
|
||||
|
||||
// Fill valid name
|
||||
await mcpRegistryPage.nameInput.fill(`valid_name_${Date.now()}`)
|
||||
|
||||
// Leave connection URL empty - save should be disabled
|
||||
const saveBtn = mcpRegistryPage.saveBtn
|
||||
await expect(saveBtn).toBeDisabled()
|
||||
|
||||
await mcpRegistryPage.cancelCreation()
|
||||
})
|
||||
})
|
||||
})
|
||||
703
tests/e2e/features/mcp-registry/pages/mcp-registry.page.ts
Normal file
703
tests/e2e/features/mcp-registry/pages/mcp-registry.page.ts
Normal file
@@ -0,0 +1,703 @@
|
||||
import { Page, Locator, expect } from '@playwright/test'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
/**
|
||||
* Connection types supported by MCP clients
|
||||
*/
|
||||
export type MCPConnectionType = 'http' | 'sse' | 'stdio'
|
||||
|
||||
/**
|
||||
* Authentication types for HTTP/SSE connections
|
||||
*/
|
||||
export type MCPAuthType = 'none' | 'headers' | 'oauth'
|
||||
|
||||
/** Header value shape used by API (value / env_var / from_env) */
|
||||
export type EnvVarLike = { value: string; env_var?: string; from_env?: boolean }
|
||||
|
||||
/**
|
||||
* MCP Client configuration
|
||||
*/
|
||||
export interface MCPClientConfig {
|
||||
name: string
|
||||
connectionType?: MCPConnectionType
|
||||
connectionUrl?: string
|
||||
authType?: MCPAuthType
|
||||
/** Headers for auth_type 'headers'. API shape: Record<string, EnvVarLike> */
|
||||
headers?: Record<string, EnvVarLike | string>
|
||||
isCodeMode?: boolean
|
||||
isPingAvailable?: boolean
|
||||
// STDIO specific
|
||||
command?: string
|
||||
args?: string
|
||||
envs?: string
|
||||
// OAuth specific
|
||||
oauthClientId?: string
|
||||
oauthClientSecret?: string
|
||||
oauthAuthorizeUrl?: string
|
||||
oauthTokenUrl?: string
|
||||
oauthScopes?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Page object for the MCP Registry page
|
||||
*/
|
||||
export class MCPRegistryPage extends BasePage {
|
||||
readonly table: Locator
|
||||
readonly createBtn: Locator
|
||||
readonly sheet: Locator
|
||||
readonly detailSheet: Locator
|
||||
readonly nameInput: Locator
|
||||
readonly saveBtn: Locator
|
||||
readonly cancelBtn: Locator
|
||||
readonly connectionTypeSelect: Locator
|
||||
readonly authTypeSelect: Locator
|
||||
readonly connectionUrlInput: Locator
|
||||
readonly codeModeSwitch: Locator
|
||||
readonly pingAvailableSwitch: Locator
|
||||
// STDIO inputs
|
||||
readonly commandInput: Locator
|
||||
readonly argsInput: Locator
|
||||
readonly envsInput: Locator
|
||||
// OAuth inputs
|
||||
readonly oauthClientIdInput: Locator
|
||||
readonly oauthClientSecretInput: Locator
|
||||
readonly oauthAuthorizeUrlInput: Locator
|
||||
readonly oauthTokenUrlInput: Locator
|
||||
readonly oauthScopesInput: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
this.table = page.locator('[data-testid="mcp-clients-table"]').or(page.locator('table'))
|
||||
this.createBtn = page.locator('[data-testid="create-mcp-client-btn"]').or(
|
||||
page.getByRole('button', { name: /New MCP Server/i }).or(page.getByRole('button', { name: /Add/i }))
|
||||
)
|
||||
this.sheet = page.locator('[role="dialog"]')
|
||||
this.detailSheet = page.locator('[role="dialog"]')
|
||||
this.nameInput = page.locator('[data-testid="client-name-input"]').or(
|
||||
this.sheet.getByLabel(/Name/i).first()
|
||||
)
|
||||
this.saveBtn = page.locator('[data-testid="save-client-btn"]').or(
|
||||
this.sheet.getByRole('button', { name: /Create/i }).or(
|
||||
this.sheet.getByRole('button', { name: /Save/i })
|
||||
)
|
||||
)
|
||||
this.cancelBtn = page.locator('[data-testid="cancel-client-btn"]').or(
|
||||
this.sheet.getByRole('button', { name: /Cancel/i })
|
||||
)
|
||||
|
||||
// Connection type and auth
|
||||
this.connectionTypeSelect = page.locator('[data-testid="connection-type-select"]')
|
||||
this.authTypeSelect = page.locator('[data-testid="auth-type-select"]')
|
||||
// Use placeholder as primary selector for EnvVarInput (more reliable)
|
||||
this.connectionUrlInput = this.sheet.getByPlaceholder(/http:\/\/your-mcp-server/i).or(
|
||||
page.locator('[data-testid="connection-url-input"]')
|
||||
)
|
||||
|
||||
// Switches (Radix UI switches)
|
||||
this.codeModeSwitch = page.locator('[data-testid="code-mode-switch"]')
|
||||
this.pingAvailableSwitch = this.sheet.locator('#ping-available')
|
||||
|
||||
// STDIO inputs
|
||||
this.commandInput = page.locator('[data-testid="stdio-command-input"]')
|
||||
this.argsInput = page.locator('[data-testid="stdio-args-input"]')
|
||||
this.envsInput = page.locator('[data-testid="stdio-envs-input"]')
|
||||
|
||||
// OAuth inputs
|
||||
this.oauthClientIdInput = this.sheet.getByPlaceholder(/your-client-id/i)
|
||||
this.oauthClientSecretInput = this.sheet.getByPlaceholder(/your-client-secret/i)
|
||||
this.oauthAuthorizeUrlInput = this.sheet.getByPlaceholder(/oauth\/authorize/i)
|
||||
this.oauthTokenUrlInput = this.sheet.getByPlaceholder(/oauth\/token/i)
|
||||
this.oauthScopesInput = this.sheet.getByPlaceholder(/read, write, admin/i)
|
||||
}
|
||||
|
||||
async goto(): Promise<void> {
|
||||
await this.page.goto('/workspace/mcp-registry')
|
||||
await waitForNetworkIdle(this.page)
|
||||
// Wait for table to be visible
|
||||
await this.table.waitFor({ state: 'visible', timeout: 10000 }).catch(() => {})
|
||||
}
|
||||
|
||||
/** Get the table row for a client by name. Scoped to tbody so the header row is never matched; first() for stable single-row target. */
|
||||
getClientRow(name: string): Locator {
|
||||
return this.table.locator('tbody tr').filter({ hasText: name }).first()
|
||||
}
|
||||
|
||||
async clientExists(name: string): Promise<boolean> {
|
||||
await this.page.waitForTimeout(500) // Brief wait for UI update
|
||||
return (await this.getClientRow(name).count()) > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll until the client row appears in the table or timeout.
|
||||
* Used as a fallback success signal when the create form doesn't close (e.g. SSE/stdio).
|
||||
*/
|
||||
async waitForClientInTable(name: string, timeoutMs: number): Promise<boolean> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
if ((await this.getClientRow(name).count()) > 0) return true
|
||||
await this.page.waitForTimeout(500)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
async getClientCount(): Promise<number> {
|
||||
// Exclude header row
|
||||
const rows = this.table.locator('tbody tr')
|
||||
return await rows.count()
|
||||
}
|
||||
|
||||
/**
|
||||
* Select connection type from dropdown
|
||||
*/
|
||||
async selectConnectionType(type: MCPConnectionType): Promise<void> {
|
||||
// Click the connection type select trigger
|
||||
const selectTrigger = this.page.locator('[data-testid="connection-type-select"]')
|
||||
await expect(selectTrigger).toBeVisible({ timeout: 5000 })
|
||||
await selectTrigger.click()
|
||||
|
||||
// Wait for dropdown to open and select the option by data-testid
|
||||
const optionTestId = `connection-type-${type}`
|
||||
const option = this.page.locator(`[data-testid="${optionTestId}"]`)
|
||||
await expect(option).toBeVisible({ timeout: 5000 })
|
||||
await option.click()
|
||||
|
||||
// Wait for dropdown to close
|
||||
await expect(option).not.toBeVisible({ timeout: 3000 }).catch(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Select authentication type from dropdown
|
||||
*/
|
||||
async selectAuthType(type: MCPAuthType): Promise<void> {
|
||||
const selectTrigger = this.page.locator('[data-testid="auth-type-select"]')
|
||||
await expect(selectTrigger).toBeVisible({ timeout: 5000 })
|
||||
await selectTrigger.click()
|
||||
|
||||
// Select the option by data-testid
|
||||
const optionTestId = `auth-type-${type}`
|
||||
const option = this.page.locator(`[data-testid="${optionTestId}"]`)
|
||||
await expect(option).toBeVisible({ timeout: 5000 })
|
||||
await option.click()
|
||||
|
||||
// Wait for dropdown to close
|
||||
await expect(option).not.toBeVisible({ timeout: 3000 }).catch(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill the MCP client form with configuration (doesn't submit)
|
||||
*/
|
||||
async fillClientForm(config: MCPClientConfig): Promise<void> {
|
||||
// Fill name
|
||||
await this.nameInput.fill(config.name)
|
||||
|
||||
// Select connection type if specified
|
||||
if (config.connectionType) {
|
||||
await this.selectConnectionType(config.connectionType)
|
||||
// Wait for the form to update after connection type change
|
||||
await this.page.waitForTimeout(500)
|
||||
}
|
||||
|
||||
// Toggle code mode if specified (Radix Switch uses data-state="checked"/"unchecked")
|
||||
if (config.isCodeMode !== undefined) {
|
||||
await expect(this.codeModeSwitch).toBeVisible({ timeout: 5000 })
|
||||
const dataState = await this.codeModeSwitch.getAttribute('data-state')
|
||||
const currentState = dataState === 'checked'
|
||||
if (currentState !== config.isCodeMode) {
|
||||
await this.codeModeSwitch.click()
|
||||
// Wait for state to change
|
||||
const expectedState = config.isCodeMode ? 'checked' : 'unchecked'
|
||||
await expect(this.codeModeSwitch).toHaveAttribute('data-state', expectedState, { timeout: 3000 })
|
||||
}
|
||||
}
|
||||
|
||||
// Toggle ping available if specified (Radix Switch)
|
||||
if (config.isPingAvailable !== undefined) {
|
||||
const dataState = await this.pingAvailableSwitch.getAttribute('data-state')
|
||||
const currentState = dataState === 'checked'
|
||||
if (currentState !== config.isPingAvailable) {
|
||||
await this.pingAvailableSwitch.click()
|
||||
}
|
||||
}
|
||||
|
||||
// Handle connection-type specific fields
|
||||
if (config.connectionType === 'http' || config.connectionType === 'sse' || !config.connectionType) {
|
||||
// Wait for auth type field to be visible (only shows for HTTP/SSE)
|
||||
await expect(this.authTypeSelect).toBeVisible({ timeout: 5000 })
|
||||
|
||||
// Select auth type if specified
|
||||
if (config.authType) {
|
||||
await this.selectAuthType(config.authType)
|
||||
await this.page.waitForTimeout(500)
|
||||
}
|
||||
|
||||
// Fill connection URL
|
||||
if (config.connectionUrl) {
|
||||
await expect(this.connectionUrlInput).toBeVisible({ timeout: 5000 })
|
||||
await this.connectionUrlInput.fill(config.connectionUrl)
|
||||
// Wait for React to process the input
|
||||
await this.page.waitForTimeout(500)
|
||||
}
|
||||
|
||||
// Fill headers when auth_type is 'headers' (required for SSE test; export MCP_SSE_HEADERS in your environment)
|
||||
if (config.authType === 'headers' && config.headers && Object.keys(config.headers).length > 0) {
|
||||
const headersTable = this.sheet.locator('[data-testid="mcp-headers-table"]')
|
||||
await expect(headersTable).toBeVisible({ timeout: 5000 })
|
||||
const entries = Object.entries(config.headers)
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const [key, val] = entries[i]
|
||||
const valueStr = typeof val === 'object' && val !== null && 'value' in val ? (val as EnvVarLike).value : String(val)
|
||||
const keyInput = headersTable.locator(`input[data-row="${i}"][data-column="key"]`)
|
||||
const valueInput = headersTable.locator(`input[data-row="${i}"][data-column="value"]`).or(
|
||||
headersTable.locator(`[data-row="${i}"][data-column="value"] input`)
|
||||
)
|
||||
await keyInput.waitFor({ state: 'visible', timeout: 8000 })
|
||||
await keyInput.scrollIntoViewIfNeeded()
|
||||
await keyInput.click()
|
||||
await keyInput.fill(key)
|
||||
await this.page.waitForTimeout(400)
|
||||
const valueEl = valueInput.first()
|
||||
await valueEl.waitFor({ state: 'visible', timeout: 3000 })
|
||||
await valueEl.scrollIntoViewIfNeeded()
|
||||
await valueEl.click()
|
||||
await valueEl.fill(valueStr)
|
||||
await this.page.waitForTimeout(500)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle OAuth config
|
||||
if (config.authType === 'oauth') {
|
||||
if (config.oauthClientId) {
|
||||
await this.oauthClientIdInput.fill(config.oauthClientId)
|
||||
}
|
||||
if (config.oauthClientSecret) {
|
||||
await this.oauthClientSecretInput.fill(config.oauthClientSecret)
|
||||
}
|
||||
if (config.oauthAuthorizeUrl) {
|
||||
await this.oauthAuthorizeUrlInput.fill(config.oauthAuthorizeUrl)
|
||||
}
|
||||
if (config.oauthTokenUrl) {
|
||||
await this.oauthTokenUrlInput.fill(config.oauthTokenUrl)
|
||||
}
|
||||
if (config.oauthScopes) {
|
||||
await this.oauthScopesInput.fill(config.oauthScopes)
|
||||
}
|
||||
}
|
||||
} else if (config.connectionType === 'stdio') {
|
||||
// Fill STDIO specific fields - wait for them to be visible after type change
|
||||
if (config.command) {
|
||||
await expect(this.commandInput).toBeVisible({ timeout: 5000 })
|
||||
await this.commandInput.fill(config.command)
|
||||
// Wait for React to process the input
|
||||
await this.page.waitForTimeout(500)
|
||||
}
|
||||
if (config.args) {
|
||||
await expect(this.argsInput).toBeVisible({ timeout: 5000 })
|
||||
await this.argsInput.fill(config.args)
|
||||
}
|
||||
if (config.envs) {
|
||||
await expect(this.envsInput).toBeVisible({ timeout: 5000 })
|
||||
await this.envsInput.fill(config.envs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an MCP client with full configuration
|
||||
*/
|
||||
async createClient(config: MCPClientConfig): Promise<boolean> {
|
||||
await this.dismissToasts()
|
||||
await this.createBtn.click()
|
||||
await expect(this.sheet).toBeVisible({ timeout: 5000 })
|
||||
|
||||
// Fill the form
|
||||
await this.fillClientForm(config)
|
||||
|
||||
// Wait for form validation to complete
|
||||
await this.page.waitForTimeout(1500)
|
||||
|
||||
// Wait for save button to be enabled (validation passed)
|
||||
await expect(this.saveBtn).toBeEnabled({ timeout: 10000 })
|
||||
|
||||
// Verify button is visible and contains expected text
|
||||
await expect(this.saveBtn).toBeVisible()
|
||||
await expect(this.saveBtn).toContainText(/Create|Save/i)
|
||||
|
||||
// Wait for create-client API response then click save (backend may be slow connecting to MCP server)
|
||||
// Create is POST to /mcp/client (singular); do not match GET /mcp/clients
|
||||
const responsePromise = this.page.waitForResponse(
|
||||
(response) => {
|
||||
const url = response.url()
|
||||
const method = response.request().method()
|
||||
return (
|
||||
(url.includes('/mcp/client') && !url.endsWith('/mcp/clients')) &&
|
||||
(method === 'POST' || method === 'PUT')
|
||||
)
|
||||
},
|
||||
{ timeout: 60000 }
|
||||
)
|
||||
await this.saveBtn.click({ force: true })
|
||||
const response = await responsePromise.catch(() => null)
|
||||
const ok = response && response.ok()
|
||||
if (response && !ok) {
|
||||
const body = await response.text().catch(() => '')
|
||||
await this.page.keyboard.press('Escape')
|
||||
await this.sheet.waitFor({ state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
throw new Error(`Create MCP client failed: ${response.status()} ${body}`)
|
||||
}
|
||||
|
||||
// Success: backend returned 2xx. Wait for create form to close (short timeout; UI usually updates quickly).
|
||||
const createFormHeading = this.page.getByRole('heading', { name: 'New MCP Server' })
|
||||
await createFormHeading.waitFor({ state: 'hidden', timeout: 15000 }).catch(() => null)
|
||||
const createFormClosed = !(await createFormHeading.isVisible().catch(() => false))
|
||||
if (createFormClosed) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Backend succeeded but form may not close quickly (e.g. SSE/stdio). If client appears in table, treat as success.
|
||||
const inTable = await this.waitForClientInTable(config.name, 10000)
|
||||
if (inTable) {
|
||||
await this.page.keyboard.press('Escape')
|
||||
await this.sheet.waitFor({ state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
return true
|
||||
}
|
||||
|
||||
// Fallback: wait for toast or heading (e.g. slow UI)
|
||||
const toast = this.getToast()
|
||||
await Promise.race([
|
||||
createFormHeading.waitFor({ state: 'hidden', timeout: 10000 }).catch(() => null),
|
||||
toast.waitFor({ state: 'visible', timeout: 10000 }).catch(() => null),
|
||||
])
|
||||
if (!(await createFormHeading.isVisible().catch(() => true))) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Sheet still open - check for toast (success or error)
|
||||
let toastText = ''
|
||||
let toastVisible = false
|
||||
try {
|
||||
toastVisible = await toast.isVisible()
|
||||
if (toastVisible) toastText = (await toast.textContent()) || ''
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (toastVisible && toastText) {
|
||||
const isSuccess =
|
||||
toastText.toLowerCase().includes('success') ||
|
||||
toastText.toLowerCase().includes('created') ||
|
||||
toastText.toLowerCase().includes('server created')
|
||||
if (isSuccess) {
|
||||
await createFormHeading.waitFor({ state: 'hidden', timeout: 10000 }).catch(() => null)
|
||||
return true
|
||||
}
|
||||
await this.page.keyboard.press('Escape')
|
||||
await this.sheet.waitFor({ state: 'hidden', timeout: 5000 }).catch(() => {})
|
||||
throw new Error(`Client creation failed with error: ${toastText}`)
|
||||
}
|
||||
|
||||
// No toast, sheet still open - validation or unknown failure
|
||||
const errorMessages = (await this.page.locator('[role="alert"]').allTextContents().catch(() => []))
|
||||
.map((t) => t.trim())
|
||||
.filter(Boolean)
|
||||
if (errorMessages.length > 0) {
|
||||
throw new Error(`Form validation errors: ${errorMessages.join(', ')}`)
|
||||
}
|
||||
const isDisabled = await this.saveBtn.isDisabled().catch(() => false)
|
||||
if (isDisabled) {
|
||||
throw new Error('Save button is disabled - form validation failed')
|
||||
}
|
||||
throw new Error('No toast appeared and sheet did not close - form submission may have failed')
|
||||
}
|
||||
|
||||
/**
|
||||
* View client details by clicking on the row
|
||||
*/
|
||||
async viewClientDetails(name: string): Promise<void> {
|
||||
const row = this.getClientRow(name)
|
||||
await row.click()
|
||||
await expect(this.detailSheet).toBeVisible({ timeout: 5000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the detail sheet
|
||||
*/
|
||||
async closeDetailSheet(): Promise<void> {
|
||||
// Press Escape or click the X button
|
||||
await this.page.keyboard.press('Escape')
|
||||
await expect(this.detailSheet).not.toBeVisible({ timeout: 5000 }).catch(async () => {
|
||||
// If still visible, try clicking X button
|
||||
const closeBtn = this.detailSheet.locator('button').filter({ has: this.page.locator('svg.lucide-x') })
|
||||
if (await closeBtn.isVisible()) {
|
||||
await closeBtn.click()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Close any open sheet/dialog (create or detail) so the table is visible
|
||||
*/
|
||||
async closeSheet(): Promise<void> {
|
||||
const isVisible = await this.sheet.isVisible().catch(() => false)
|
||||
if (isVisible) {
|
||||
await this.page.keyboard.press('Escape')
|
||||
await expect(this.sheet).not.toBeVisible({ timeout: 5000 }).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up MCP clients by name. Ensures we're on the page and any sheet is closed before deleting.
|
||||
*/
|
||||
async cleanupMCPClients(names: string[]): Promise<void> {
|
||||
if (names.length === 0) return
|
||||
|
||||
await this.goto()
|
||||
await this.closeSheet()
|
||||
await this.dismissToasts()
|
||||
await this.table.waitFor({ state: 'visible', timeout: 10000 }).catch(() => {})
|
||||
await this.page.waitForTimeout(500)
|
||||
|
||||
for (const name of names) {
|
||||
const tryDelete = async (): Promise<void> => {
|
||||
const exists = await this.clientExists(name)
|
||||
if (!exists) return
|
||||
await this.closeSheet()
|
||||
await this.deleteClient(name, { requireToast: false })
|
||||
}
|
||||
|
||||
try {
|
||||
await tryDelete()
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error)
|
||||
console.error(`[CLEANUP ERROR] Failed to delete MCP client: ${name} - ${errorMsg}`)
|
||||
await this.closeSheet()
|
||||
await this.page.waitForTimeout(1000)
|
||||
try {
|
||||
await tryDelete()
|
||||
} catch (retryErr) {
|
||||
const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr)
|
||||
console.error(`[CLEANUP ERROR] Retry failed for MCP client: ${name} - ${retryMsg}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Edit an existing client
|
||||
*/
|
||||
async editClient(name: string, updates: Partial<MCPClientConfig>): Promise<void> {
|
||||
await this.viewClientDetails(name)
|
||||
|
||||
// Update name if provided
|
||||
if (updates.name) {
|
||||
const nameInput = this.detailSheet.getByLabel(/Name/i).first()
|
||||
await nameInput.clear()
|
||||
await nameInput.fill(updates.name)
|
||||
}
|
||||
|
||||
// Toggle code mode if specified
|
||||
if (updates.isCodeMode !== undefined) {
|
||||
const codeModeSwitch = this.detailSheet
|
||||
.locator('input[type="checkbox"]')
|
||||
.filter({ has: this.page.locator('#code-mode') })
|
||||
.or(this.detailSheet.getByRole('switch', { name: /Code Mode/i }))
|
||||
const isVisible = await codeModeSwitch.isVisible().catch(() => false)
|
||||
if (isVisible) {
|
||||
const currentState = await codeModeSwitch.isChecked()
|
||||
if (currentState !== updates.isCodeMode) {
|
||||
await codeModeSwitch.click()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save changes
|
||||
const saveBtn = this.detailSheet.getByRole('button', { name: /Save/i })
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.detailSheet).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconnect an MCP client
|
||||
*/
|
||||
async reconnectClient(name: string): Promise<void> {
|
||||
const row = this.getClientRow(name)
|
||||
// Stop propagation by clicking the reconnect button directly
|
||||
const reconnectBtn = row.locator('button').filter({ has: this.page.locator('svg.lucide-refresh-ccw') })
|
||||
await reconnectBtn.click()
|
||||
await this.waitForSuccessToast('Reconnected')
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle tool enabled state in the detail sheet
|
||||
*/
|
||||
async toggleToolEnabled(clientName: string, toolName: string): Promise<void> {
|
||||
await this.viewClientDetails(clientName)
|
||||
|
||||
// Find the tool row and toggle its enabled switch
|
||||
const toolRow = this.detailSheet.locator('tr').filter({ hasText: toolName })
|
||||
const enabledSwitch = toolRow.locator('button[role="switch"]').first()
|
||||
await enabledSwitch.click()
|
||||
|
||||
// Save
|
||||
const saveBtn = this.detailSheet.getByRole('button', { name: /Save/i })
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.detailSheet).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle auto-execute for a tool in the detail sheet
|
||||
*/
|
||||
async toggleAutoExecute(clientName: string, toolName: string): Promise<void> {
|
||||
await this.viewClientDetails(clientName)
|
||||
|
||||
// Find the tool row and toggle its auto-execute switch (second switch)
|
||||
const toolRow = this.detailSheet.locator('tr').filter({ hasText: toolName })
|
||||
const autoExecuteSwitch = toolRow.locator('button[role="switch"]').nth(1)
|
||||
await autoExecuteSwitch.click()
|
||||
|
||||
// Save
|
||||
const saveBtn = this.detailSheet.getByRole('button', { name: /Save/i })
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.detailSheet).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle code mode for a client
|
||||
*/
|
||||
async toggleCodeMode(clientName: string): Promise<void> {
|
||||
await this.viewClientDetails(clientName)
|
||||
|
||||
// Find and toggle the code mode switch
|
||||
const codeModeSwitch = this.detailSheet
|
||||
.getByRole('switch', { name: /Code Mode/i })
|
||||
.or(this.detailSheet.locator('#code-mode'))
|
||||
await codeModeSwitch.click()
|
||||
|
||||
// Save
|
||||
const saveBtn = this.detailSheet.getByRole('button', { name: /Save/i })
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.detailSheet).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Get client status from the table
|
||||
*/
|
||||
async getClientStatus(name: string): Promise<string> {
|
||||
const row = this.getClientRow(name)
|
||||
const statusBadge = row
|
||||
.locator('[class*="badge"]')
|
||||
.or(row.locator('span').filter({ hasText: /connected|disconnected|connecting|error/i }))
|
||||
.last()
|
||||
const statusText = await statusBadge.textContent()
|
||||
return statusText?.toLowerCase().trim() || ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Get connection type displayed in the table (HTTP, SSE, STDIO)
|
||||
*/
|
||||
async getClientConnectionType(name: string): Promise<string> {
|
||||
const row = this.getClientRow(name)
|
||||
const typeCell = row.getByTestId('mcp-client-connection-type')
|
||||
if ((await typeCell.count()) > 0) {
|
||||
return (await typeCell.first().textContent())?.trim() ?? ''
|
||||
}
|
||||
const cells = row.locator('td')
|
||||
if ((await cells.count()) >= 2) {
|
||||
return (await cells.nth(1).textContent())?.trim() ?? ''
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Get tools count from the client details sheet
|
||||
* Assumes the detail sheet is already open
|
||||
*/
|
||||
async getToolsCount(): Promise<number> {
|
||||
// Tools are displayed in a table in the detail sheet
|
||||
const toolRows = this.detailSheet.locator('table tbody tr')
|
||||
const count = await toolRows.count()
|
||||
return count
|
||||
}
|
||||
|
||||
/**
|
||||
* Get enabled tools count from table
|
||||
*/
|
||||
async getEnabledToolsCount(name: string): Promise<string | null> {
|
||||
const row = this.getClientRow(name)
|
||||
// Enabled tools is typically shown as "X/Y" format
|
||||
const cells = row.locator('td')
|
||||
const count = await cells.count()
|
||||
if (count >= 5) {
|
||||
return await cells.nth(4).textContent()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel client creation
|
||||
*/
|
||||
async cancelCreation(): Promise<void> {
|
||||
await this.cancelBtn.click()
|
||||
await expect(this.sheet).not.toBeVisible({ timeout: 5000 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the client row to disappear from the table (e.g. after delete or refetch).
|
||||
* Polls so we don't rely on a stale locator.
|
||||
*/
|
||||
async waitForClientGone(name: string, timeoutMs: number): Promise<boolean> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
if ((await this.getClientRow(name).count()) === 0) return true
|
||||
await this.page.waitForTimeout(500)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an MCP client. Success is determined by the DELETE API completing and the row
|
||||
* disappearing from the table after the list refetches.
|
||||
*/
|
||||
async deleteClient(name: string, options?: { requireToast?: boolean }): Promise<void> {
|
||||
const row = this.getClientRow(name)
|
||||
const deleteBtn = row
|
||||
.locator('button')
|
||||
.filter({ has: this.page.locator('svg.lucide-trash-2') })
|
||||
.or(row.locator('button').filter({ has: this.page.locator('svg.lucide-trash') }))
|
||||
await deleteBtn.click()
|
||||
|
||||
const confirmDialog = this.page.locator('[role="alertdialog"]')
|
||||
await expect(confirmDialog).toBeVisible({ timeout: 5000 })
|
||||
const deleteResponsePromise = this.page.waitForResponse(
|
||||
(response) => {
|
||||
const url = response.url()
|
||||
return url.includes('/mcp/client/') && response.request().method() === 'DELETE'
|
||||
},
|
||||
{ timeout: 15000 }
|
||||
)
|
||||
await confirmDialog.getByRole('button', { name: /Delete/i }).click()
|
||||
|
||||
await deleteResponsePromise.catch(() => null)
|
||||
|
||||
// Wait for table to refetch and row to disappear (poll fresh locator; avoid stale row reference)
|
||||
const gone = await this.waitForClientGone(name, 20000)
|
||||
if (!gone) {
|
||||
throw new Error(`Client "${name}" still visible after delete`)
|
||||
}
|
||||
|
||||
if (options?.requireToast !== false) {
|
||||
await this.getToast().waitFor({ state: 'visible', timeout: 5000 }).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if empty state is visible
|
||||
*/
|
||||
async isEmptyStateVisible(): Promise<boolean> {
|
||||
const emptyMessage = this.page.getByText(/No clients found/i)
|
||||
return await emptyMessage.isVisible().catch(() => false)
|
||||
}
|
||||
}
|
||||
21
tests/e2e/features/mcp-settings/mcp-settings.spec.ts
Normal file
21
tests/e2e/features/mcp-settings/mcp-settings.spec.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
|
||||
test.describe('MCP Settings', () => {
|
||||
test.beforeEach(async ({ mcpSettingsPage }) => {
|
||||
await mcpSettingsPage.goto()
|
||||
})
|
||||
|
||||
test('should display MCP settings page', async ({ mcpSettingsPage }) => {
|
||||
await expect(mcpSettingsPage.mcpSettingsView).toBeVisible()
|
||||
})
|
||||
|
||||
test('should display MCP settings form fields', async ({ mcpSettingsPage }) => {
|
||||
await expect(mcpSettingsPage.page.getByLabel('Max Agent Depth')).toBeVisible()
|
||||
await expect(mcpSettingsPage.page.getByLabel('Tool Execution Timeout (seconds)')).toBeVisible()
|
||||
})
|
||||
|
||||
test('should have save button disabled when no changes', async ({ mcpSettingsPage }) => {
|
||||
await expect(mcpSettingsPage.saveBtn).toBeVisible()
|
||||
await expect(mcpSettingsPage.saveBtn).toBeDisabled()
|
||||
})
|
||||
})
|
||||
24
tests/e2e/features/mcp-settings/pages/mcp-settings.page.ts
Normal file
24
tests/e2e/features/mcp-settings/pages/mcp-settings.page.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
import { Locator, Page } from '@playwright/test'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
export class MCPSettingsPage extends BasePage {
|
||||
readonly mcpSettingsView: Locator
|
||||
readonly saveBtn: Locator
|
||||
readonly maxAgentDepthInput: Locator
|
||||
readonly toolTimeoutInput: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
|
||||
this.mcpSettingsView = page.getByTestId('mcp-settings-view')
|
||||
this.saveBtn = page.getByTestId('mcp-settings-save-btn')
|
||||
this.maxAgentDepthInput = page.getByTestId('mcp-agent-depth-input').or(page.locator('#mcp-agent-depth'))
|
||||
this.toolTimeoutInput = page.getByTestId('mcp-tool-timeout-input').or(page.locator('#mcp-tool-execution-timeout'))
|
||||
}
|
||||
|
||||
async goto(): Promise<void> {
|
||||
await this.page.goto('/workspace/mcp-settings')
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
}
|
||||
13
tests/e2e/features/mcp-tool-groups/mcp-tool-groups.spec.ts
Normal file
13
tests/e2e/features/mcp-tool-groups/mcp-tool-groups.spec.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
|
||||
// MCP Tool Groups routes to @enterprise components not present in OSS.
|
||||
// Tests only verify URL routing; do not add UI assertions for enterprise-only content.
|
||||
test.describe('MCP Tool Groups', () => {
|
||||
test.beforeEach(async ({ mcpToolGroupsPage }) => {
|
||||
await mcpToolGroupsPage.goto()
|
||||
})
|
||||
|
||||
test('should load MCP tool groups page', async ({ mcpToolGroupsPage }) => {
|
||||
await expect(mcpToolGroupsPage.page).toHaveURL(/mcp-tool-groups/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Page } from '@playwright/test'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
export class MCPToolGroupsPage extends BasePage {
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
}
|
||||
|
||||
async goto(): Promise<void> {
|
||||
await this.page.goto('/workspace/mcp-tool-groups')
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
}
|
||||
11
tests/e2e/features/model-limits/model-limits.data.ts
Normal file
11
tests/e2e/features/model-limits/model-limits.data.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import { ModelLimitConfig } from './pages/model-limits.page'
|
||||
|
||||
export function createModelLimitData(overrides: Partial<ModelLimitConfig> = {}): ModelLimitConfig {
|
||||
return {
|
||||
provider: 'openai',
|
||||
modelName: 'gpt-4o-mini',
|
||||
budget: { maxLimit: 10, resetDuration: '1M' },
|
||||
rateLimit: { tokenMaxLimit: 1000, requestMaxLimit: 50 },
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
81
tests/e2e/features/model-limits/model-limits.spec.ts
Normal file
81
tests/e2e/features/model-limits/model-limits.spec.ts
Normal file
@@ -0,0 +1,81 @@
|
||||
import { expect, test } from '../../core/fixtures/base.fixture'
|
||||
import { createModelLimitData } from './model-limits.data'
|
||||
|
||||
const createdLimits: { modelName: string; provider: string }[] = []
|
||||
|
||||
test.describe('Model Limits', () => {
|
||||
test.beforeEach(async ({ modelLimitsPage }) => {
|
||||
await modelLimitsPage.goto()
|
||||
})
|
||||
|
||||
test.afterEach(async ({ modelLimitsPage }) => {
|
||||
await modelLimitsPage.closeSheet()
|
||||
for (const { modelName, provider } of [...createdLimits]) {
|
||||
try {
|
||||
const exists = await modelLimitsPage.modelLimitExists(modelName, provider)
|
||||
if (exists) {
|
||||
await modelLimitsPage.deleteModelLimit(modelName, provider)
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[CLEANUP] Failed to delete model limit ${modelName}:`, e)
|
||||
}
|
||||
}
|
||||
createdLimits.length = 0
|
||||
})
|
||||
|
||||
test('should display create button or empty state', async ({ modelLimitsPage }) => {
|
||||
const createVisible = await modelLimitsPage.createBtn.isVisible().catch(() => false)
|
||||
expect(createVisible).toBe(true)
|
||||
})
|
||||
|
||||
test('should create a model limit with budget and rate limit', async ({ modelLimitsPage }) => {
|
||||
const limitData = createModelLimitData({
|
||||
provider: 'openai',
|
||||
budget: { maxLimit: 5, resetDuration: '1M' },
|
||||
rateLimit: { tokenMaxLimit: 500, requestMaxLimit: 20 },
|
||||
})
|
||||
|
||||
const modelName = await modelLimitsPage.createModelLimit(limitData)
|
||||
createdLimits.push({ modelName, provider: limitData.provider })
|
||||
|
||||
const exists = await modelLimitsPage.modelLimitExists(modelName, limitData.provider)
|
||||
expect(exists).toBe(true)
|
||||
})
|
||||
|
||||
test('should edit a model limit budget and rate limit', async ({ modelLimitsPage }) => {
|
||||
const limitData = createModelLimitData({ provider: 'openai' })
|
||||
|
||||
const modelName = await modelLimitsPage.createModelLimit(limitData)
|
||||
createdLimits.push({ modelName, provider: limitData.provider })
|
||||
|
||||
await modelLimitsPage.editModelLimit(modelName, limitData.provider, {
|
||||
budget: { maxLimit: 20 },
|
||||
rateLimit: { tokenMaxLimit: 2000, requestMaxLimit: 100 },
|
||||
})
|
||||
|
||||
const exists = await modelLimitsPage.modelLimitExists(modelName, limitData.provider)
|
||||
expect(exists).toBe(true)
|
||||
})
|
||||
|
||||
test('should delete a model limit', async ({ modelLimitsPage }) => {
|
||||
const limitData = createModelLimitData({
|
||||
provider: 'openai',
|
||||
budget: { maxLimit: 5 },
|
||||
})
|
||||
|
||||
const modelName = await modelLimitsPage.createModelLimit(limitData)
|
||||
createdLimits.push({ modelName, provider: limitData.provider })
|
||||
|
||||
let exists = await modelLimitsPage.modelLimitExists(modelName, limitData.provider)
|
||||
expect(exists).toBe(true)
|
||||
|
||||
await modelLimitsPage.deleteModelLimit(modelName, limitData.provider)
|
||||
const idx = createdLimits.findIndex(
|
||||
(limit) => limit.modelName === modelName && limit.provider === limitData.provider
|
||||
)
|
||||
if (idx >= 0) createdLimits.splice(idx, 1)
|
||||
|
||||
exists = await modelLimitsPage.modelLimitExists(modelName, limitData.provider)
|
||||
expect(exists).toBe(false)
|
||||
})
|
||||
})
|
||||
138
tests/e2e/features/model-limits/pages/model-limits.page.ts
Normal file
138
tests/e2e/features/model-limits/pages/model-limits.page.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import type { Locator, Page } from '@playwright/test'
|
||||
import { expect } from '../../../core/fixtures/base.fixture'
|
||||
import { BasePage } from '../../../core/pages/base.page'
|
||||
import { fillSelect, waitForNetworkIdle } from '../../../core/utils/test-helpers'
|
||||
|
||||
export interface ModelLimitConfig {
|
||||
provider: string
|
||||
modelName: string
|
||||
budget?: { maxLimit: number; resetDuration?: string }
|
||||
rateLimit?: {
|
||||
tokenMaxLimit?: number
|
||||
requestMaxLimit?: number
|
||||
}
|
||||
}
|
||||
|
||||
function toTestIdPart(value: string): string {
|
||||
return value.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
|
||||
}
|
||||
|
||||
export class ModelLimitsPage extends BasePage {
|
||||
readonly createBtn: Locator
|
||||
readonly table: Locator
|
||||
readonly sheet: Locator
|
||||
|
||||
constructor(page: Page) {
|
||||
super(page)
|
||||
|
||||
this.createBtn = page.getByTestId('model-limits-button-create')
|
||||
this.table = page.getByTestId('model-limits-table')
|
||||
this.sheet = page.getByTestId('model-limit-sheet')
|
||||
}
|
||||
|
||||
async goto(): Promise<void> {
|
||||
await this.page.goto('/workspace/model-limits')
|
||||
await waitForNetworkIdle(this.page)
|
||||
}
|
||||
|
||||
getModelLimitRow(modelName: string, provider: string = 'all'): Locator {
|
||||
return this.page.getByTestId(`model-limit-row-${toTestIdPart(modelName)}-${toTestIdPart(provider)}`)
|
||||
}
|
||||
|
||||
async modelLimitExists(modelName: string, provider: string = 'all'): Promise<boolean> {
|
||||
const row = this.getModelLimitRow(modelName, provider)
|
||||
return (await row.count()) > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a model limit via the sheet: selects provider, selects the requested
|
||||
* model (config.modelName) in the search dropdown, fills budget and rate
|
||||
* limit, then saves. Returns the selected model name for use in exists/edit/delete.
|
||||
*/
|
||||
async createModelLimit(config: ModelLimitConfig): Promise<string> {
|
||||
await this.createBtn.click()
|
||||
await expect(this.sheet).toBeVisible({ timeout: 5000 })
|
||||
await this.waitForSheetAnimation()
|
||||
|
||||
// Select provider (stable testid; sheet is the only one open)
|
||||
await fillSelect(
|
||||
this.page,
|
||||
'[data-testid="model-limit-provider-select"]',
|
||||
config.provider === 'all' ? 'All Providers' : config.provider
|
||||
)
|
||||
|
||||
// Model multiselect - search and select requested model deterministically
|
||||
const modelSelectContainer = this.sheet.getByTestId('model-limit-model-select')
|
||||
const modelInput = modelSelectContainer.locator('input')
|
||||
await modelInput.fill(config.modelName)
|
||||
await this.page.waitForSelector('[role="option"]', { timeout: 10000 })
|
||||
const targetOption = this.page.getByRole('option', { name: config.modelName, exact: true })
|
||||
await expect(targetOption).toBeVisible({ timeout: 10000 })
|
||||
await targetOption.click()
|
||||
const selectedModelName = config.modelName
|
||||
|
||||
if (config.budget?.maxLimit !== undefined) {
|
||||
const budgetInput = this.page.locator('#modelBudgetMaxLimit')
|
||||
await budgetInput.fill(String(config.budget.maxLimit))
|
||||
}
|
||||
|
||||
if (config.rateLimit?.tokenMaxLimit !== undefined) {
|
||||
await this.page.locator('#modelTokenMaxLimit').fill(String(config.rateLimit.tokenMaxLimit))
|
||||
}
|
||||
if (config.rateLimit?.requestMaxLimit !== undefined) {
|
||||
await this.page.locator('#modelRequestMaxLimit').fill(String(config.rateLimit.requestMaxLimit))
|
||||
}
|
||||
|
||||
const saveBtn = this.page.getByRole('button', { name: /Create Limit/i })
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.sheet).not.toBeVisible({ timeout: 10000 })
|
||||
return selectedModelName
|
||||
}
|
||||
|
||||
async editModelLimit(modelName: string, provider: string, updates: Partial<ModelLimitConfig>): Promise<void> {
|
||||
const editBtn = this.page.getByTestId(`model-limit-button-edit-${toTestIdPart(modelName)}-${toTestIdPart(provider)}`)
|
||||
await editBtn.click()
|
||||
await expect(this.sheet).toBeVisible({ timeout: 5000 })
|
||||
await this.waitForSheetAnimation()
|
||||
|
||||
if (updates.budget?.maxLimit !== undefined) {
|
||||
const budgetInput = this.page.locator('#modelBudgetMaxLimit')
|
||||
await budgetInput.clear()
|
||||
await budgetInput.fill(String(updates.budget.maxLimit))
|
||||
}
|
||||
|
||||
if (updates.rateLimit?.tokenMaxLimit !== undefined) {
|
||||
const tokenInput = this.page.locator('#modelTokenMaxLimit')
|
||||
await tokenInput.clear()
|
||||
await tokenInput.fill(String(updates.rateLimit.tokenMaxLimit))
|
||||
}
|
||||
if (updates.rateLimit?.requestMaxLimit !== undefined) {
|
||||
const requestInput = this.page.locator('#modelRequestMaxLimit')
|
||||
await requestInput.clear()
|
||||
await requestInput.fill(String(updates.rateLimit.requestMaxLimit))
|
||||
}
|
||||
|
||||
const saveBtn = this.page.getByRole('button', { name: /Save Changes|Create Limit/i })
|
||||
await saveBtn.click()
|
||||
await this.waitForSuccessToast()
|
||||
await expect(this.sheet).not.toBeVisible({ timeout: 10000 })
|
||||
}
|
||||
|
||||
async deleteModelLimit(modelName: string, provider: string = 'all'): Promise<void> {
|
||||
const deleteBtn = this.page.getByTestId(`model-limit-button-delete-${toTestIdPart(modelName)}-${toTestIdPart(provider)}`)
|
||||
await deleteBtn.click()
|
||||
|
||||
const confirmDialog = this.page.locator('[role="alertdialog"]')
|
||||
await confirmDialog.getByRole('button', { name: /Delete/i }).click()
|
||||
await this.waitForSuccessToast()
|
||||
await this.page.waitForTimeout(1000)
|
||||
}
|
||||
|
||||
async closeSheet(): Promise<void> {
|
||||
if (await this.sheet.isVisible().catch(() => false)) {
|
||||
await this.page.keyboard.press('Escape')
|
||||
await expect(this.sheet).not.toBeVisible({ timeout: 5000 }).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
38
tests/e2e/features/observability/observability.data.ts
Normal file
38
tests/e2e/features/observability/observability.data.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Test data factories for observability connector tests
|
||||
*/
|
||||
|
||||
/**
|
||||
* Observability connector configuration
|
||||
*/
|
||||
export interface ObservabilityConnectorConfig {
|
||||
type: 'otel' | 'maxim'
|
||||
enabled: boolean
|
||||
endpoint?: string
|
||||
apiKey?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Create OTEL connector configuration data
|
||||
*/
|
||||
export function createOtelConnectorData(overrides: Partial<ObservabilityConnectorConfig> = {}): ObservabilityConnectorConfig {
|
||||
return {
|
||||
type: 'otel',
|
||||
enabled: true,
|
||||
endpoint: 'http://localhost:4318',
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create Maxim connector configuration data
|
||||
*/
|
||||
export function createMaximConnectorData(overrides: Partial<ObservabilityConnectorConfig> = {}): ObservabilityConnectorConfig {
|
||||
return {
|
||||
type: 'maxim',
|
||||
enabled: true,
|
||||
endpoint: 'http://localhost:8080',
|
||||
apiKey: 'test-api-key',
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user