943 lines
26 KiB
YAML
943 lines
26 KiB
YAML
# Bifrost Integration Tests Configuration
|
|
# This file centralizes all configuration for AI integration clients and test settings
|
|
|
|
# Bifrost Gateway Configuration
|
|
# All integrations route through Bifrost as a proxy/gateway
|
|
bifrost:
|
|
base_url: "${BIFROST_BASE_URL:-http://localhost:8080}"
|
|
|
|
# Integration-specific endpoints (suffixes appended to base_url)
|
|
endpoints:
|
|
openai: "openai"
|
|
anthropic: "anthropic"
|
|
cohere: "cohere"
|
|
google: "genai"
|
|
"gemini_passthrough": "genai_passthrough"
|
|
"anthropic_passthrough": "anthropic_passthrough"
|
|
litellm: "litellm"
|
|
langchain: "langchain"
|
|
pydanticai: "pydanticai"
|
|
bedrock: "bedrock"
|
|
azure: "openai" # Azure uses OpenAI routes with /openai/deployments/{id} paths
|
|
|
|
# Full URLs constructed as: {base_url.rstrip('/')}/{endpoints[integration]}
|
|
# Examples:
|
|
# - OpenAI: http://localhost:8080/openai
|
|
# - Anthropic: http://localhost:8080/anthropic
|
|
# - Google: http://localhost:8080/genai
|
|
# - LiteLLM: http://localhost:8080/litellm
|
|
# - LangChain: http://localhost:8080/langchain
|
|
|
|
# API Configuration
|
|
api:
|
|
timeout: 30 # seconds
|
|
max_retries: 3
|
|
retry_delay: 1 # seconds
|
|
|
|
# Provider model configurations
|
|
# Integrations (openai, anthropic, google, litellm, langchain) map to these providers
|
|
providers:
|
|
openai:
|
|
chat: "gpt-4o"
|
|
vision: "gpt-4o"
|
|
file: "gpt-4o"
|
|
tools: "gpt-4o-mini"
|
|
speech: "tts-1"
|
|
transcription: "whisper-1"
|
|
embeddings: "text-embedding-3-small"
|
|
image_generation: "gpt-image-1"
|
|
image_edit: "gpt-image-1"
|
|
streaming: "gpt-4o-mini"
|
|
thinking: "gpt-5.1"
|
|
batch_file_upload: "gpt-4o-mini"
|
|
batch_list: "gpt-4o"
|
|
batch_retrieve: "gpt-4o"
|
|
batch_cancel: "gpt-4o"
|
|
batch_inline: "gpt-4o"
|
|
file_upload: "gpt-4o-mini"
|
|
file_list: "gpt-4o-mini"
|
|
file_retrieve: "gpt-4o-mini"
|
|
file_delete: "gpt-4o-mini"
|
|
file_content: "gpt-4o-mini"
|
|
count_tokens: "gpt-4o-mini"
|
|
video: "sora-2"
|
|
alternatives:
|
|
- "gpt-4"
|
|
- "gpt-4-turbo-preview"
|
|
- "gpt-4o"
|
|
- "gpt-3.5-turbo"
|
|
|
|
azure:
|
|
chat: "gpt-4o"
|
|
vision: "gpt-4o"
|
|
tools: "gpt-4o-mini"
|
|
streaming: "gpt-4o-mini"
|
|
speech: "gpt-4o-mini-tts"
|
|
transcription: "whisper"
|
|
embeddings: "text-embedding-3-small"
|
|
image_generation: "gpt-image-1"
|
|
thinking: "o1"
|
|
batch_file_upload: "gpt-4o-batch"
|
|
batch_list: "gpt-4o-batch"
|
|
batch_retrieve: "gpt-4o-batch"
|
|
batch_cancel: "gpt-4o-batch"
|
|
file_upload: "gpt-4o"
|
|
file_list: "gpt-4o"
|
|
file_retrieve: "gpt-4o"
|
|
file_delete: "gpt-4o"
|
|
file_content: "gpt-4o"
|
|
|
|
xai:
|
|
chat: "grok-4-0709"
|
|
vision: "grok-2-vision-1212"
|
|
tools: "grok-4-0709"
|
|
streaming: "grok-4-0709"
|
|
thinking: "grok-3-mini"
|
|
|
|
anthropic:
|
|
chat: "claude-sonnet-4-5-20250929"
|
|
vision: "claude-sonnet-4-5-20250929"
|
|
file: "claude-sonnet-4-5-20250929"
|
|
tools: "claude-sonnet-4-5-20250929"
|
|
streaming: "claude-sonnet-4-5-20250929"
|
|
thinking: "claude-opus-4-5"
|
|
batch_file_upload: "claude-sonnet-4-20250514"
|
|
batch_inline: "claude-sonnet-4-20250514"
|
|
batch_list: "claude-sonnet-4-20250514"
|
|
batch_retrieve: "claude-sonnet-4-20250514"
|
|
batch_cancel: "claude-sonnet-4-20250514"
|
|
file_upload: "claude-sonnet-4-20250514"
|
|
file_list: "claude-sonnet-4-20250514"
|
|
file_retrieve: "claude-sonnet-4-20250514"
|
|
file_delete: "claude-sonnet-4-20250514"
|
|
file_content: "claude-sonnet-4-20250514"
|
|
count_tokens: "claude-sonnet-4-5-20250929"
|
|
alternatives:
|
|
- "claude-3-sonnet-20240229"
|
|
- "claude-3-opus-20240229"
|
|
- "claude-3-5-sonnet-20241022"
|
|
- "claude-3-haiku-20240307"
|
|
|
|
gemini:
|
|
chat: "gemini-3-flash-preview"
|
|
vision: "gemini-3-flash-preview"
|
|
tools: "gemini-3-flash-preview"
|
|
file: "gemini-2.5-flash"
|
|
thinking: "gemini-3-pro-preview"
|
|
speech: "gemini-2.5-flash-preview-tts"
|
|
transcription: "gemini-2.5-flash"
|
|
embeddings: "gemini-embedding-001"
|
|
image_generation: "gemini-2.5-flash-image"
|
|
image_edit: "gemini-3-pro-image-preview"
|
|
imagen: "imagen-4.0-generate-001"
|
|
video: "veo-3.1-generate-preview"
|
|
streaming: "gemini-3-flash-preview"
|
|
batch_create: "gemini-2.5-flash"
|
|
batch_inline: "gemini-2.5-flash"
|
|
batch_file_upload: "gemini-2.5-flash"
|
|
batch_list: "gemini-2.5-flash"
|
|
batch_retrieve: "gemini-2.5-flash"
|
|
batch_cancel: "gemini-2.5-flash"
|
|
batch_s3: "gemini-2.5-flash"
|
|
file_upload: "gemini-2.0-flash"
|
|
file_list: "gemini-2.0-flash"
|
|
file_content: "gemini-2.0-flash"
|
|
file_download: "gemini-2.0-flash"
|
|
file_retrieve: "gemini-2.0-flash"
|
|
file_delete: "gemini-2.0-flash"
|
|
count_tokens: "gemini-2.5-flash"
|
|
alternatives:
|
|
- "gemini-1.5-pro"
|
|
- "gemini-1.5-flash"
|
|
- "gemini-1.0-pro"
|
|
- "gemini-2.0-flash-001"
|
|
|
|
vertex:
|
|
chat: "gemini-2.5-flash"
|
|
vision: "claude-sonnet-4-5"
|
|
tools: "gemini-2.5-flash"
|
|
file: "claude-sonnet-4-5"
|
|
thinking: "gemini-2.5-pro"
|
|
embeddings: "gemini-embedding-001"
|
|
image_generation: "imagen-4.0-generate-001"
|
|
image_edit: "imagen-3.0-capability-001"
|
|
imagen: "imagen-4.0-generate-001"
|
|
streaming: "gemini-2.5-flash"
|
|
count_tokens: "claude-sonnet-4-5"
|
|
video: "veo-3.1-generate-preview"
|
|
bedrock:
|
|
chat: "global.anthropic.claude-sonnet-4-20250514-v1:0"
|
|
vision: "global.anthropic.claude-sonnet-4-20250514-v1:0"
|
|
file: "global.anthropic.claude-sonnet-4-20250514-v1:0"
|
|
tools: "global.anthropic.claude-sonnet-4-20250514-v1:0"
|
|
streaming: "global.anthropic.claude-sonnet-4-20250514-v1:0"
|
|
thinking: "us.anthropic.claude-opus-4-5-20251101-v1:0"
|
|
text_completion: "mistral.mistral-7b-instruct-v0:2"
|
|
embeddings: "global.cohere.embed-v4:0"
|
|
image_generation: "amazon.titan-image-generator-v2:0"
|
|
image_variation: "amazon.titan-image-generator-v2:0"
|
|
batch_inline: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
image_edit: "amazon.nova-canvas-v1:0"
|
|
batch_list: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
batch_retrieve: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
batch_cancel: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
batch_file_upload: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
batch_s3: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
file_upload: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
file_list: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
file_delete: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
file_content: "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
count_tokens: "us.anthropic.claude-3-7-sonnet-20250219-v1:0"
|
|
alternatives:
|
|
- "anthropic.claude-3-opus-20240229-v1:0"
|
|
|
|
cohere:
|
|
chat: "command-a-03-2025"
|
|
vision: "command-a-vision-07-2025"
|
|
tools: "command-a-03-2025"
|
|
embeddings: "embed-v4.0"
|
|
streaming: "command-a-03-2025"
|
|
count_tokens: "command-a-03-2025"
|
|
alternatives:
|
|
- "command-r-plus"
|
|
|
|
huggingface:
|
|
image_generation: "fal-ai/fal-ai/flux/dev"
|
|
image_edit: "fal-ai/fal-ai/flux-2/edit"
|
|
|
|
nebius:
|
|
image_generation: "black-forest-labs/flux-schnell"
|
|
|
|
replicate:
|
|
video: "openai/sora-2-pro"
|
|
|
|
runway:
|
|
video: "gen4.5"
|
|
|
|
|
|
|
|
# Provider availability configuration
|
|
# Maps provider names to their API key environment variables
|
|
provider_api_keys:
|
|
openai: "OPENAI_API_KEY"
|
|
anthropic: "ANTHROPIC_API_KEY"
|
|
gemini: "GEMINI_API_KEY"
|
|
vertex: "VERTEX_API_KEY"
|
|
bedrock: "AWS_ACCESS_KEY_ID"
|
|
cohere: "COHERE_API_KEY"
|
|
xai: "XAI_API_KEY"
|
|
huggingface: "HUGGING_FACE_API_KEY"
|
|
nebius: "NEBIUS_API_KEY"
|
|
azure: "AZURE_API_KEY"
|
|
replicate: "REPLICATE_API_KEY"
|
|
runway: "RUNWAY_API_KEY"
|
|
|
|
# Provider test scenarios - which tests each provider supports
|
|
provider_scenarios:
|
|
openai:
|
|
simple_chat: true
|
|
multi_turn_conversation: true
|
|
streaming: true
|
|
tool_calls: true
|
|
multiple_tool_calls: true
|
|
end2end_tool_calling: true
|
|
automatic_function_calling: true
|
|
"web_search": true
|
|
image_url: true
|
|
image_base64: true
|
|
file_input: true
|
|
multiple_images: true
|
|
speech_synthesis: true
|
|
speech_synthesis_streaming: true
|
|
transcription: true
|
|
transcription_streaming: true
|
|
embeddings: true
|
|
image_generation: true
|
|
image_edit: true
|
|
thinking: true
|
|
prompt_caching: false
|
|
citations: false
|
|
list_models: true
|
|
responses: true
|
|
responses_image: true
|
|
text_completion: false
|
|
langchain_structured_output: true
|
|
pydantic_structured_output: true # PydanticAI structured output works reliably with OpenAI
|
|
pydanticai_streaming: true # PydanticAI streaming works with OpenAI
|
|
batch_file_upload: true
|
|
batch_create: true
|
|
batch_list: true
|
|
batch_retrieve: true
|
|
batch_cancel: true
|
|
batch_inline: true # OpenAI supports inline requests for batch
|
|
batch_s3: false # OpenAI does not use S3 for batch
|
|
file_upload: true
|
|
file_list: true
|
|
file_retrieve: true
|
|
file_delete: true
|
|
file_content: true
|
|
count_tokens: true
|
|
video_generation: false # disabled for now because of long running operations
|
|
|
|
azure:
|
|
simple_chat: true
|
|
multi_turn_conversation: true
|
|
streaming: true
|
|
tool_calls: true
|
|
multiple_tool_calls: true
|
|
end2end_tool_calling: true
|
|
automatic_function_calling: true
|
|
web_search: false
|
|
image_url: true
|
|
image_base64: true
|
|
file_input: false
|
|
multiple_images: true
|
|
speech_synthesis: true
|
|
speech_synthesis_streaming: true
|
|
transcription: true
|
|
transcription_streaming: true
|
|
embeddings: true
|
|
image_generation: true
|
|
image_edit: false
|
|
thinking: true
|
|
prompt_caching: false
|
|
citations: false
|
|
list_models: true
|
|
responses: true
|
|
responses_image: true
|
|
text_completion: false
|
|
langchain_structured_output: false
|
|
pydantic_structured_output: false
|
|
pydanticai_streaming: false
|
|
batch_file_upload: true
|
|
batch_create: false
|
|
batch_list: true
|
|
batch_retrieve: true
|
|
batch_cancel: true
|
|
batch_inline: false
|
|
batch_s3: false
|
|
file_upload: true
|
|
file_list: true
|
|
file_retrieve: true
|
|
file_delete: true
|
|
file_content: true
|
|
count_tokens: false
|
|
|
|
xai:
|
|
simple_chat: true
|
|
multi_turn_conversation: true
|
|
streaming: true
|
|
tool_calls: true
|
|
multiple_tool_calls: true
|
|
end2end_tool_calling: true
|
|
automatic_function_calling: true
|
|
image_url: true
|
|
image_base64: false
|
|
file_input: false
|
|
multiple_images: false
|
|
thinking: true
|
|
list_models: true
|
|
responses: true
|
|
responses_image: true
|
|
text_completion: false
|
|
langchain_structured_output: true
|
|
pydantic_structured_output: true
|
|
pydanticai_streaming: true
|
|
|
|
anthropic:
|
|
simple_chat: true
|
|
multi_turn_conversation: true
|
|
streaming: true
|
|
tool_calls: true
|
|
multiple_tool_calls: true
|
|
end2end_tool_calling: true
|
|
automatic_function_calling: true
|
|
web_search: true
|
|
image_url: true
|
|
image_base64: true
|
|
file_input: true
|
|
file_input_text: true
|
|
multiple_images: true
|
|
speech_synthesis: false
|
|
speech_synthesis_streaming: false
|
|
transcription: false
|
|
transcription_streaming: false
|
|
embeddings: false
|
|
thinking: true
|
|
prompt_caching: true
|
|
citations: true
|
|
list_models: true
|
|
responses: true
|
|
responses_image: true
|
|
text_completion: false
|
|
langchain_structured_output: false
|
|
pydantic_structured_output: true # PydanticAI structured output works with Anthropic
|
|
pydanticai_streaming: true # PydanticAI streaming works with Anthropic
|
|
batch_file_upload: true # Anthropic batch API uses inline requests, not files
|
|
batch_create: true
|
|
batch_list: true
|
|
batch_retrieve: true
|
|
batch_cancel: true
|
|
batch_inline: true # Anthropic uses inline requests for batch
|
|
batch_s3: false # Anthropic does not use S3 for batch
|
|
file_upload: true
|
|
file_list: true
|
|
file_retrieve: true
|
|
file_delete: true
|
|
file_content: true
|
|
count_tokens: true
|
|
|
|
gemini:
|
|
simple_chat: true
|
|
multi_turn_conversation: true
|
|
streaming: true
|
|
tool_calls: true
|
|
multiple_tool_calls: true
|
|
end2end_tool_calling: true
|
|
automatic_function_calling: true
|
|
image_url: false # Gemini requires base64 or file upload
|
|
image_base64: true
|
|
file_input: true
|
|
multiple_images: false
|
|
speech_synthesis: true
|
|
speech_synthesis_streaming: true
|
|
transcription: true
|
|
transcription_streaming: true
|
|
embeddings: true
|
|
image_generation: true # Gemini image generation via responseModalities
|
|
image_edit: true # Gemini image editing
|
|
imagen: true # Imagen via :predict endpoint
|
|
imagen_edit: true # Imagen editing via image_edit model
|
|
thinking: true
|
|
video_generation: false # disabled for now because of long running operations
|
|
prompt_caching: false
|
|
citations: false
|
|
list_models: true
|
|
responses: true
|
|
responses_image: true
|
|
text_completion: false
|
|
langchain_structured_output: true
|
|
pydantic_structured_output: false # PydanticAI structured output unreliable via Bifrost for Gemini
|
|
pydanticai_streaming: false # PydanticAI GoogleModel streaming has asyncio issues
|
|
batch_file_upload: true # Gemini supports file upload via Files API
|
|
batch_create: true
|
|
batch_list: true
|
|
batch_retrieve: true
|
|
batch_cancel: true
|
|
batch_inline: true # Gemini uses inline requests for batch (synchronous)
|
|
batch_s3: false # Gemini does not use S3 for batch
|
|
file_upload: true
|
|
file_list: true
|
|
file_retrieve: true
|
|
file_delete: true
|
|
file_content: false # Gemini doesn't support direct file download
|
|
count_tokens: true
|
|
context_caching: true # Gemini context caching (Caches API) via Bifrost passthrough
|
|
|
|
vertex:
|
|
simple_chat: true
|
|
multi_turn_conversation: true
|
|
streaming: true
|
|
tool_calls: true
|
|
multiple_tool_calls: true
|
|
end2end_tool_calling: true
|
|
automatic_function_calling: true
|
|
image_url: false # Gemini requires base64 or file upload
|
|
image_base64: true
|
|
file_input: true
|
|
multiple_images: false
|
|
speech_synthesis: false
|
|
speech_synthesis_streaming: false
|
|
transcription: false
|
|
transcription_streaming: false
|
|
embeddings: true
|
|
image_generation: true
|
|
image_edit: true
|
|
imagen: true # Imagen via :predict endpoint
|
|
imagen_edit: true # Imagen editing via image_edit model
|
|
thinking: true
|
|
prompt_caching: false
|
|
list_models: true
|
|
video_generation: false # disabled for now because of long running operations
|
|
responses: true
|
|
responses_image: true
|
|
text_completion: false
|
|
langchain_structured_output: true
|
|
pydantic_structured_output: false # PydanticAI structured output unreliable via Bifrost for Gemini
|
|
pydanticai_streaming: false # PydanticAI GoogleModel streaming has asyncio issues
|
|
batch_file_upload: false # Gemini supports file upload via Files API
|
|
batch_create: false
|
|
batch_list: false
|
|
batch_retrieve: false
|
|
batch_cancel: false
|
|
batch_inline: false # Gemini uses inline requests for batch (synchronous)
|
|
batch_s3: false # Gemini does not use S3 for batch
|
|
file_upload: false
|
|
file_list: false
|
|
file_retrieve: false
|
|
file_delete: false
|
|
file_content: false # Gemini doesn't support direct file download
|
|
count_tokens: false
|
|
|
|
bedrock:
|
|
simple_chat: true
|
|
multi_turn_conversation: true
|
|
streaming: true
|
|
tool_calls: true
|
|
multiple_tool_calls: true
|
|
end2end_tool_calling: true
|
|
automatic_function_calling: true
|
|
image_url: false
|
|
image_base64: true
|
|
file_input: true
|
|
file_input_text: true
|
|
multiple_images: false
|
|
speech_synthesis: false
|
|
speech_synthesis_streaming: false
|
|
transcription: false
|
|
transcription_streaming: false
|
|
embeddings: true
|
|
thinking: true
|
|
prompt_caching: true
|
|
citations: false
|
|
list_models: true
|
|
responses: true
|
|
responses_image: true
|
|
text_completion: false
|
|
langchain_structured_output: true
|
|
pydantic_structured_output: false # Bedrock not supported in PydanticAI tests
|
|
pydanticai_streaming: false # Bedrock not supported in PydanticAI tests
|
|
batch_file_upload: true # Bedrock uses S3 wrapper for file uploads
|
|
batch_create: true
|
|
batch_list: true
|
|
batch_retrieve: true
|
|
batch_cancel: true
|
|
batch_inline: false # Bedrock batch uses S3, not inline requests via API
|
|
batch_s3: true # Bedrock uses S3 for batch input/output
|
|
file_upload: true # Bedrock uses S3 wrapper for file storage
|
|
file_list: true # Bedrock lists files in S3 bucket
|
|
file_retrieve: true # Bedrock retrieves S3 object metadata
|
|
file_delete: true # Bedrock deletes S3 objects
|
|
file_content: true # Bedrock downloads S3 object content
|
|
image_generation: true # Bedrock supports image generation via invoke (Titan, SA, cross-provider)
|
|
image_edit: true # Bedrock supports image editing via invoke (Titan, SA)
|
|
image_variation: true # Bedrock supports image variation via invoke (Titan IMAGE_VARIATION)
|
|
count_tokens: true # Bedrock supports token counting via CountTokens API
|
|
|
|
cohere:
|
|
simple_chat: true
|
|
multi_turn_conversation: true
|
|
streaming: true
|
|
tool_calls: true
|
|
multiple_tool_calls: true
|
|
end2end_tool_calling: true
|
|
automatic_function_calling: false
|
|
image_url: true
|
|
image_base64: true
|
|
multiple_images: true
|
|
speech_synthesis: false
|
|
speech_synthesis_streaming: false
|
|
transcription: false
|
|
transcription_streaming: false
|
|
embeddings: true
|
|
thinking: false
|
|
prompt_caching: false
|
|
citations: false
|
|
list_models: false
|
|
responses: true
|
|
responses_image: true
|
|
text_completion: false
|
|
langchain_structured_output: true
|
|
pydantic_structured_output: false # PydanticAI CohereModel doesn't reliably support structured output
|
|
pydanticai_streaming: false # PydanticAI CohereModel doesn't implement streaming
|
|
batch_file_upload: false
|
|
batch_create: false
|
|
batch_list: false
|
|
batch_retrieve: false
|
|
batch_cancel: false
|
|
batch_inline: false # Cohere does not support batch API
|
|
batch_s3: false # Cohere does not support batch API
|
|
file_upload: false # Cohere does not support Files API
|
|
file_list: false
|
|
file_retrieve: false
|
|
file_delete: false
|
|
file_content: false
|
|
count_tokens: true
|
|
|
|
huggingface:
|
|
image_generation: true
|
|
image_edit: true
|
|
|
|
nebius:
|
|
image_generation: true
|
|
|
|
replicate:
|
|
video_generation: false # disabled for now because of long running operations
|
|
|
|
runway:
|
|
video_generation: false # disabled for now because of long running operations
|
|
|
|
# Scenario to capability mapping
|
|
# Maps test scenario names to their corresponding capability types
|
|
scenario_capabilities:
|
|
simple_chat: "chat"
|
|
multi_turn_conversation: "chat"
|
|
responses: "chat"
|
|
responses_image: "vision"
|
|
text_completion: "chat"
|
|
streaming: "streaming"
|
|
tool_calls: "tools"
|
|
multiple_tool_calls: "tools"
|
|
end2end_tool_calling: "tools"
|
|
automatic_function_calling: "tools"
|
|
web_search: "chat"
|
|
image_url: "vision"
|
|
image_base64: "vision"
|
|
file_input: "file"
|
|
file_input_text: "file"
|
|
multiple_images: "vision"
|
|
speech_synthesis: "speech"
|
|
speech_synthesis_streaming: "speech"
|
|
transcription: "transcription"
|
|
transcription_streaming: "transcription"
|
|
embeddings: "embeddings"
|
|
image_generation: "image_generation" # Uses image_generation model
|
|
image_edit: "image_edit" # Uses image_edit model
|
|
imagen: "imagen" # Uses imagen model (Gemini/Vertex)
|
|
imagen_edit: "image_edit" # Uses image_edit model for Imagen editing
|
|
thinking: "thinking"
|
|
prompt_caching: "chat"
|
|
citations: "chat"
|
|
list_models: "chat"
|
|
langchain_structured_output: "chat" # LangChain structured output uses chat capability
|
|
count_tokens: "count_tokens" # Token counting capability
|
|
pydantic_structured_output: "chat" # Structured output uses chat capability
|
|
pydanticai_streaming: "streaming" # PydanticAI streaming uses streaming capability
|
|
batch_file_upload: "batch_file_upload" # Uses batch_file_upload model directly
|
|
batch_create: "batch_create"
|
|
batch_list: "batch_list"
|
|
batch_retrieve: "batch_retrieve"
|
|
batch_cancel: "batch_cancel"
|
|
batch_inline: "batch_inline" # Uses batch_inline model directly
|
|
batch_s3: "batch_s3" # Uses batch_s3 model directly
|
|
file_upload: "file_upload" # Uses file_upload model directly
|
|
file_list: "file_list" # Uses file_list model directly
|
|
file_retrieve: "file_retrieve" # Uses file_retrieve model directly
|
|
file_delete: "file_delete" # Uses file_delete model directly
|
|
file_content: "file_content" # Uses file_content model directly
|
|
count_tokens: "chat"
|
|
video_generation: "video"
|
|
context_caching: "chat" # Gemini Caches API (passthrough)
|
|
|
|
# Model capabilities matrix
|
|
model_capabilities:
|
|
# OpenAI Models
|
|
"gpt-3.5-turbo":
|
|
chat: true
|
|
tools: true
|
|
vision: false
|
|
streaming: true
|
|
max_tokens: 4096
|
|
context_window: 4096
|
|
|
|
"gpt-4":
|
|
chat: true
|
|
tools: true
|
|
vision: false
|
|
streaming: true
|
|
max_tokens: 8192
|
|
context_window: 8192
|
|
|
|
"gpt-4o":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
max_tokens: 4096
|
|
context_window: 128000
|
|
|
|
"gpt-4o-mini":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
speech: false
|
|
transcription: false
|
|
max_tokens: 4096
|
|
context_window: 128000
|
|
|
|
# OpenAI Speech Models
|
|
"tts-1":
|
|
chat: false
|
|
tools: false
|
|
vision: false
|
|
streaming: false
|
|
speech: true
|
|
transcription: false
|
|
max_tokens: null
|
|
context_window: null
|
|
|
|
"tts-1-hd":
|
|
chat: false
|
|
tools: false
|
|
vision: false
|
|
streaming: false
|
|
speech: true
|
|
transcription: false
|
|
max_tokens: null
|
|
context_window: null
|
|
|
|
# OpenAI Transcription Models
|
|
"whisper-1":
|
|
chat: false
|
|
tools: false
|
|
vision: false
|
|
streaming: false
|
|
speech: false
|
|
transcription: true
|
|
embeddings: false
|
|
max_tokens: null
|
|
context_window: null
|
|
|
|
# OpenAI Embedding Models
|
|
"text-embedding-3-small":
|
|
chat: false
|
|
tools: false
|
|
vision: false
|
|
streaming: false
|
|
speech: false
|
|
transcription: false
|
|
embeddings: true
|
|
max_tokens: null
|
|
context_window: 8191
|
|
dimensions: 1536
|
|
|
|
"text-embedding-3-large":
|
|
chat: false
|
|
tools: false
|
|
vision: false
|
|
streaming: false
|
|
speech: false
|
|
transcription: false
|
|
embeddings: true
|
|
max_tokens: null
|
|
context_window: 8191
|
|
dimensions: 3072
|
|
|
|
"text-embedding-ada-002":
|
|
chat: false
|
|
tools: false
|
|
vision: false
|
|
streaming: false
|
|
speech: false
|
|
transcription: false
|
|
embeddings: true
|
|
max_tokens: null
|
|
context_window: 8191
|
|
dimensions: 1536
|
|
|
|
# Anthropic Models
|
|
"claude-3-haiku-20240307":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
max_tokens: 4096
|
|
context_window: 200000
|
|
|
|
"claude-3-sonnet-20240229":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
max_tokens: 4096
|
|
context_window: 200000
|
|
|
|
"claude-3-opus-20240229":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
max_tokens: 4096
|
|
context_window: 200000
|
|
|
|
# Google Models
|
|
"gemini-pro":
|
|
chat: true
|
|
tools: true
|
|
vision: false
|
|
streaming: true
|
|
max_tokens: 8192
|
|
context_window: 32768
|
|
|
|
"gemini-2.0-flash-001":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
max_tokens: 8192
|
|
context_window: 32768
|
|
|
|
"gemini-1.5-pro":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
max_tokens: 8192
|
|
context_window: 1000000
|
|
|
|
# Gemini Transcription Models
|
|
"gemini-2.5-flash":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
speech: false
|
|
transcription: true
|
|
embeddings: false
|
|
max_tokens: 8192
|
|
context_window: 1000000
|
|
audio_max_duration: 34200 # 9.5 hours in seconds
|
|
|
|
"gemini-2.5-pro":
|
|
chat: true
|
|
tools: true
|
|
vision: true
|
|
streaming: true
|
|
speech: false
|
|
transcription: true
|
|
embeddings: false
|
|
max_tokens: 8192
|
|
context_window: 2000000
|
|
audio_max_duration: 34200 # 9.5 hours in seconds
|
|
|
|
# Gemini TTS Models
|
|
"gemini-2.5-flash-preview-tts":
|
|
chat: false
|
|
tools: false
|
|
vision: false
|
|
streaming: false
|
|
speech: true
|
|
transcription: false
|
|
embeddings: false
|
|
max_tokens: 32000 # 32k token context window for TTS
|
|
context_window: 32000
|
|
audio_format: "pcm"
|
|
sample_rate: 24000
|
|
channels: 1
|
|
|
|
"gemini-2.5-pro-preview-tts":
|
|
chat: false
|
|
tools: false
|
|
vision: false
|
|
streaming: false
|
|
speech: true
|
|
transcription: false
|
|
embeddings: false
|
|
max_tokens: 32000 # 32k token context window for TTS
|
|
context_window: 32000
|
|
audio_format: "pcm"
|
|
sample_rate: 24000
|
|
channels: 1
|
|
|
|
# Mistral Models
|
|
"mistral-7b-instruct":
|
|
chat: true
|
|
tools: false
|
|
vision: false
|
|
streaming: true
|
|
max_tokens: 4096
|
|
context_window: 32768
|
|
|
|
"mistral-8x7b-instruct":
|
|
chat: true
|
|
tools: true
|
|
vision: false
|
|
streaming: true
|
|
max_tokens: 4096
|
|
context_window: 32768
|
|
|
|
# Test configuration
|
|
test_settings:
|
|
# Maximum tokens for test responses
|
|
max_tokens:
|
|
chat: 100
|
|
vision: 200
|
|
tools: 100
|
|
complex: 300
|
|
speech: null # Speech doesn't use token limits
|
|
transcription: null # Transcription doesn't use token limits
|
|
embeddings: null # Embeddings don't use token limits (text is the input)
|
|
|
|
# Timeout settings for tests
|
|
timeouts:
|
|
simple: 30 # seconds
|
|
complex: 60 # seconds
|
|
|
|
# Retry settings for flaky tests
|
|
retries:
|
|
max_attempts: 3
|
|
delay: 2 # seconds
|
|
|
|
# Integration-specific settings
|
|
integration_settings:
|
|
openai:
|
|
organization: "${OPENAI_ORG_ID:-}"
|
|
project: "${OPENAI_PROJECT_ID:-}"
|
|
|
|
anthropic:
|
|
version: "2023-06-01"
|
|
|
|
google:
|
|
project_id: "${GOOGLE_PROJECT_ID:-}"
|
|
location: "${GOOGLE_LOCATION:-us-central1}"
|
|
|
|
litellm:
|
|
drop_params: true
|
|
debug: false
|
|
|
|
langchain:
|
|
debug: false
|
|
streaming: true
|
|
|
|
bedrock:
|
|
region: "${AWS_REGION:-us-west-2}"
|
|
s3_bucket: "${AWS_S3_BUCKET:-}"
|
|
batch_role_arn: "${AWS_ARN:-}"
|
|
output_s3_prefix: "${AWS_OUTPUT_S3_PREFIX:-bifrost-batch-output/}"
|
|
|
|
azure:
|
|
api_version: "${AZURE_API_VERSION:-2024-10-21}"
|
|
|
|
# Environment-specific overrides
|
|
environments:
|
|
development:
|
|
api:
|
|
timeout: 60
|
|
max_retries: 5
|
|
test_settings:
|
|
timeouts:
|
|
simple: 60
|
|
complex: 120
|
|
|
|
production:
|
|
api:
|
|
timeout: 15
|
|
max_retries: 2
|
|
test_settings:
|
|
timeouts:
|
|
simple: 20
|
|
complex: 40
|
|
|
|
# Virtual key testing configuration
|
|
# When enabled, cross-provider tests will run twice: with and without the x-bf-vk header
|
|
virtual_key:
|
|
enabled: true
|
|
value: "sk-bf-test-key"
|
|
|
|
# Logging configuration
|
|
logging:
|
|
level: "INFO"
|
|
format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
file: "tests.log"
|