// Package llmtests provides comprehensive test account and configuration management for the Bifrost system. // It implements account functionality for testing purposes, supporting multiple AI providers // and comprehensive test scenarios. package llmtests import ( "context" "fmt" "os" "time" bifrost "github.com/maximhq/bifrost/core" "github.com/maximhq/bifrost/core/schemas" ) const Concurrency = 4 // Replicate test key names (see replicateProviderTestKeys). // ListModels uses the deployments API via the list-models key; all other operations use the inference key. const ( ReplicateKeyNameListModels = "replicate-list-models-deployments" ReplicateKeyNameInference = "replicate-inference" ) // ProviderOpenAICustom represents the custom OpenAI provider for testing const ProviderOpenAICustom = schemas.ModelProvider("openai-custom") // TestScenarios defines the comprehensive test scenarios type TestScenarios struct { TextCompletion bool TextCompletionStream bool SimpleChat bool CompletionStream bool MultiTurnConversation bool ToolCalls bool ToolCallsStreaming bool // Streaming tool calls functionality MultipleToolCalls bool MultipleToolCallsStreaming bool // Streaming multiple tool calls (some providers only return 1 tool call in streaming) End2EndToolCalling bool AutomaticFunctionCall bool ImageURL bool ImageBase64 bool MultipleImages bool FileBase64 bool FileURL bool CompleteEnd2End bool SpeechSynthesis bool // Text-to-speech functionality SpeechSynthesisStream bool // Streaming text-to-speech functionality Transcription bool // Speech-to-text functionality TranscriptionStream bool // Streaming speech-to-text functionality Embedding bool // Embedding functionality Reasoning bool // Reasoning/thinking functionality via Responses API PromptCaching bool // Prompt caching functionality ListModels bool // List available models functionality ImageGeneration bool // Image generation functionality ImageGenerationStream bool // Streaming image generation functionality ImageEdit bool // Image edit functionality ImageEditStream bool // Streaming image edit functionality ImageVariation bool // Image variation functionality ImageVariationStream bool // Streaming image variation functionality (if supported) VideoGeneration bool // Video generation functionality VideoRetrieve bool // Video retrieve functionality VideoRemix bool // Video remix functionality (OpenAI only) VideoDownload bool // Video download functionality VideoList bool // Video list functionality VideoDelete bool // Video delete functionality BatchCreate bool // Batch API create functionality BatchList bool // Batch API list functionality BatchRetrieve bool // Batch API retrieve functionality BatchCancel bool // Batch API cancel functionality BatchResults bool // Batch API results functionality FileUpload bool // File API upload functionality FileList bool // File API list functionality FileRetrieve bool // File API retrieve functionality FileDelete bool // File API delete functionality FileContent bool // File API content download functionality FileBatchInput bool // Whether batch create supports file-based input (InputFileID) CountTokens bool // Count tokens functionality ChatAudio bool // Chat completion with audio input/output functionality StructuredOutputs bool // Structured outputs (JSON schema) functionality WebSearchTool bool // Web search tool functionality ContainerCreate bool // Container API create functionality ContainerList bool // Container API list functionality ContainerRetrieve bool // Container API retrieve functionality ContainerDelete bool // Container API delete functionality ContainerFileCreate bool // Container File API create functionality ContainerFileList bool // Container File API list functionality ContainerFileRetrieve bool // Container File API retrieve functionality ContainerFileContent bool // Container File API content functionality ContainerFileDelete bool // Container File API delete functionality PassThroughExtraParams bool // Pass through extra params functionality Rerank bool // Rerank functionality PassthroughAPI bool // Raw HTTP passthrough API (Passthrough + PassthroughStream) WebSocketResponses bool // WebSocket Responses API mode Realtime bool // Realtime API (bidirectional audio/text) Compaction bool // Server-side compaction (context management) InterleavedThinking bool // Interleaved thinking between tool calls (beta) FastMode bool // Fast mode for Opus 4.6 (beta: research preview) EagerInputStreaming bool // Fine-grained tool input streaming (Anthropic fine-grained-tool-streaming-2025-05-14) ServerToolsViaOpenAIEndpoint bool // Anthropic server-tool shapes in tools[] via /v1/chat/completions (web_search / web_fetch / code_execution) } // ComprehensiveTestConfig extends TestConfig with additional scenarios type ComprehensiveTestConfig struct { Provider schemas.ModelProvider TextModel string ChatModel string PromptCachingModel string VisionModel string ReasoningModel string EmbeddingModel string RerankModel string TranscriptionModel string SpeechSynthesisModel string ChatAudioModel string Scenarios TestScenarios Fallbacks []schemas.Fallback // for chat, responses, image and reasoning tests TextCompletionFallbacks []schemas.Fallback // for text completion tests TranscriptionFallbacks []schemas.Fallback // for transcription tests SpeechSynthesisFallbacks []schemas.Fallback // for speech synthesis tests EmbeddingFallbacks []schemas.Fallback // for embedding tests RerankFallbacks []schemas.Fallback // for rerank tests SkipReason string // Reason to skip certain tests ImageGenerationModel string // Model for image generation ImageGenerationFallbacks []schemas.Fallback // Fallbacks for image generation ImageEditModel string // Model for image editing ImageEditFallbacks []schemas.Fallback // Fallbacks for image editing ImageVariationModel string // Model for image variation ImageVariationFallbacks []schemas.Fallback // Fallbacks for image variation VideoGenerationModel string // Model for video generation ExternalTTSProvider schemas.ModelProvider // External TTS provider to use for testing ExternalTTSModel string // External TTS model to use for testing BatchExtraParams map[string]interface{} // Extra params for batch operations (e.g., role_arn, output_s3_uri for Bedrock) FileExtraParams map[string]interface{} // Extra params for file operations (e.g., s3_bucket for Bedrock) DisableParallelFor []string // Test scenarios to disable parallel execution for (e.g., "Transcription" for rate-limited APIs) ExpectRawRequestResponse bool // When true, validate rawRequest/rawResponse in ExtraFields PassthroughModel string // Model for passthrough API tests; defaults to ChatModel when empty CompactionModel string // Model for compaction tests; defaults to claude-sonnet-4-6 InterleavedThinkingModel string // Model for interleaved thinking tests; defaults to claude-opus-4-5 FastModeModel string // Model for fast mode tests; defaults to claude-opus-4-6 RealtimeModel string // Model for Realtime API (e.g., "gpt-4o-realtime-preview") } // ComprehensiveTestAccount provides a test implementation of the Account interface for comprehensive testing. type ComprehensiveTestAccount struct{} // getEnvWithDefault returns the value of the environment variable if set, otherwise returns the default value func getEnvWithDefault(envVar, defaultValue string) string { if value := os.Getenv(envVar); value != "" { return value } return defaultValue } // GetConfiguredProviders returns the list of initially supported providers. func (account *ComprehensiveTestAccount) GetConfiguredProviders() ([]schemas.ModelProvider, error) { return []schemas.ModelProvider{ schemas.OpenAI, schemas.Anthropic, schemas.Bedrock, schemas.Cohere, schemas.Azure, schemas.Vertex, schemas.Ollama, schemas.Mistral, schemas.Groq, schemas.SGL, schemas.Parasail, schemas.Elevenlabs, schemas.Perplexity, schemas.Cerebras, schemas.Gemini, schemas.OpenRouter, schemas.HuggingFace, schemas.Nebius, schemas.XAI, schemas.Replicate, schemas.VLLM, schemas.Runway, schemas.Fireworks, ProviderOpenAICustom, }, nil } // replicateProviderTestKeys returns the two Replicate keys used by comprehensive tests: func replicateProviderTestKeys() []schemas.Key { return []schemas.Key{ { Name: ReplicateKeyNameListModels, Value: *schemas.NewEnvVar("env.REPLICATE_API_KEY"), Models: []string{"*"}, Weight: 0, UseForBatchAPI: bifrost.Ptr(false), ReplicateKeyConfig: &schemas.ReplicateKeyConfig{UseDeploymentsEndpoint: true}, }, { Name: ReplicateKeyNameInference, Value: *schemas.NewEnvVar("env.REPLICATE_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), ReplicateKeyConfig: nil, }, } } // ReplicateDirectKeyForListModels returns the key used for Replicate ListModels (deployments endpoint). // List-models tests set it on the context as schemas.BifrostContextKeyDirectKey so Bifrost passes only this key. func ReplicateDirectKeyForListModels() schemas.Key { return replicateProviderTestKeys()[0] } // GetKeysForProvider returns the API keys and associated models for a given provider. func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx context.Context, providerKey schemas.ModelProvider) ([]schemas.Key, error) { switch providerKey { case schemas.OpenAI: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.OPENAI_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case ProviderOpenAICustom: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.OPENAI_API_KEY"), // Use GROQ API key for OpenAI-compatible endpoint Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Anthropic: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.ANTHROPIC_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Bedrock: return []schemas.Key{ { Models: []string{"*"}, Weight: 1.0, Aliases: map[string]string{ "claude-3.7-sonnet": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", "claude-4-sonnet": "global.anthropic.claude-sonnet-4-20250514-v1:0", "claude-4.5-sonnet": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", "claude-4.5-haiku": "global.anthropic.claude-haiku-4-5-20251001-v1:0", }, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("env.AWS_ACCESS_KEY_ID"), SecretKey: *schemas.NewEnvVar("env.AWS_SECRET_ACCESS_KEY"), SessionToken: schemas.NewEnvVar("env.AWS_SESSION_TOKEN"), Region: schemas.NewEnvVar(getEnvWithDefault("AWS_REGION", "us-east-1")), ARN: schemas.NewEnvVar("env.AWS_ARN"), }, }, { Models: []string{"*"}, Weight: 1.0, Aliases: map[string]string{ "claude-3.5-sonnet": "anthropic.claude-3-5-sonnet-20240620-v1:0", "claude-3.7-sonnet": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", "claude-4-sonnet": "global.anthropic.claude-sonnet-4-20250514-v1:0", "claude-4.5-sonnet": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", "claude-4.5-haiku": "global.anthropic.claude-haiku-4-5-20251001-v1:0", }, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("env.AWS_ACCESS_KEY_ID"), SecretKey: *schemas.NewEnvVar("env.AWS_SECRET_ACCESS_KEY"), SessionToken: schemas.NewEnvVar("env.AWS_SESSION_TOKEN"), Region: schemas.NewEnvVar(getEnvWithDefault("AWS_REGION", "us-east-1")), ARN: schemas.NewEnvVar("env.AWS_BEDROCK_ARN"), }, UseForBatchAPI: bifrost.Ptr(true), }, { Models: []string{"cohere.embed-v4:0", "amazon.nova-canvas-v1:0", "anthropic.claude-sonnet-4-20250514-v1:0"}, Weight: 1.0, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("env.AWS_ACCESS_KEY_ID"), SecretKey: *schemas.NewEnvVar("env.AWS_SECRET_ACCESS_KEY"), SessionToken: schemas.NewEnvVar("env.AWS_SESSION_TOKEN"), Region: schemas.NewEnvVar(getEnvWithDefault("AWS_REGION", "us-east-1")), }, }, }, nil case schemas.Cohere: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.COHERE_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Azure: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.AZURE_API_KEY"), Models: []string{"*"}, Weight: 1.0, Aliases: schemas.KeyAliases{ "gpt-4o": "gpt-4o", "gpt-4o-backup": "gpt-4o-3", "claude-opus-4-5": "claude-opus-4-5", "o1": "o1", "gpt-image-1": "gpt-image-1", "text-embedding-ada-002": "text-embedding-ada-002", "sora-2": "sora-2", }, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("env.AZURE_ENDPOINT"), APIVersion: schemas.NewEnvVar("env.AZURE_API_VERSION"), ClientID: schemas.NewEnvVar("env.AZURE_CLIENT_ID"), ClientSecret: schemas.NewEnvVar("env.AZURE_CLIENT_SECRET"), TenantID: schemas.NewEnvVar("env.AZURE_TENANT_ID"), }, UseForBatchAPI: bifrost.Ptr(true), }, { Value: *schemas.NewEnvVar("env.AZURE_API_KEY"), Models: []string{"*"}, Weight: 1.0, Aliases: schemas.KeyAliases{ "whisper": "whisper", "whisper-1": "whisper", "gpt-4o-mini-tts": "gpt-4o-mini-tts", "gpt-4o-mini-audio-preview": "gpt-4o-mini-audio-preview", }, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("env.AZURE_ENDPOINT"), APIVersion: schemas.NewEnvVar("env.AZURE_API_VERSION"), }, }, }, nil case schemas.Vertex: //https://aiplatform.googleapis.com/v1/projects/maxim-development-433105/locations/global/publishers/google/models/veo-3.1-generate-preview:fetchPredictOperation return []schemas.Key{ { Value: *schemas.NewEnvVar("env.VERTEX_API_KEY"), Models: []string{"text-multilingual-embedding-002", "gemini-2.5-pro", "gemini-2.5-flash-image", "imagen-4.0-generate-001", "imagen-3.0-capability-001", "semantic-ranker-default@latest", "semantic-ranker-default-004"}, Weight: 1.0, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("env.VERTEX_PROJECT_ID"), Region: *schemas.NewEnvVar(getEnvWithDefault("VERTEX_REGION", "us-central1")), AuthCredentials: *schemas.NewEnvVar("env.VERTEX_CREDENTIALS"), }, UseForBatchAPI: bifrost.Ptr(true), }, { Value: *schemas.NewEnvVar("env.VERTEX_API_KEY"), Models: []string{"veo-3.1-generate-preview"}, Weight: 1.0, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("env.VERTEX_PROJECT_ID"), Region: *schemas.NewEnvVar("global"), AuthCredentials: *schemas.NewEnvVar("env.VERTEX_CREDENTIALS"), }, UseForBatchAPI: bifrost.Ptr(true), }, { Value: *schemas.NewEnvVar("env.VERTEX_API_KEY"), Models: []string{"claude-sonnet-4-5", "claude-4.5-haiku", "claude-opus-4-5"}, Weight: 1.0, Aliases: schemas.KeyAliases{ "claude-sonnet-4-5": "claude-sonnet-4-5", "claude-4.5-haiku": "claude-haiku-4-5@20251001", "claude-opus-4-5": "claude-opus-4-5", }, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("env.VERTEX_PROJECT_ID"), Region: *schemas.NewEnvVar(getEnvWithDefault("VERTEX_REGION_ANTHROPIC", "us-east5")), AuthCredentials: *schemas.NewEnvVar("env.VERTEX_CREDENTIALS"), }, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Mistral: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.MISTRAL_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Groq: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.GROQ_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Parasail: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.PARASAIL_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Elevenlabs: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.ELEVENLABS_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Perplexity: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.PERPLEXITY_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Cerebras: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.CEREBRAS_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Gemini: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.GEMINI_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.OpenRouter: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.OPENROUTER_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.HuggingFace: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.HUGGING_FACE_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Nebius: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.NEBIUS_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.XAI: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.XAI_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Replicate: return replicateProviderTestKeys(), nil case schemas.Runway: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.RUNWAY_API_KEY"), Models: []string{"*"}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil case schemas.Fireworks: return []schemas.Key{ { Value: *schemas.NewEnvVar("env.FIREWORKS_API_KEY"), Models: []string{}, Weight: 1.0, UseForBatchAPI: bifrost.Ptr(true), }, }, nil default: return nil, fmt.Errorf("unsupported provider: %s", providerKey) } } // GetConfigForProvider returns the configuration settings for a given provider. func (account *ComprehensiveTestAccount) GetConfigForProvider(providerKey schemas.ModelProvider) (*schemas.ProviderConfig, error) { switch providerKey { case schemas.OpenAI: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Higher retries for production-grade provider RetryBackoffInitial: 500 * time.Millisecond, RetryBackoffMax: 8 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: 10, BufferSize: 10, }, }, nil case ProviderOpenAICustom: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ BaseURL: "https://api.openai.com", DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Higher retries for Groq (can be flaky) RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 10 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, CustomProviderConfig: &schemas.CustomProviderConfig{ BaseProviderType: schemas.OpenAI, AllowedRequests: &schemas.AllowedRequests{ TextCompletion: false, ChatCompletion: true, ChatCompletionStream: true, Embedding: false, Speech: false, SpeechStream: false, Transcription: false, TranscriptionStream: false, }, }, }, nil case schemas.Anthropic: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Claude is generally reliable RetryBackoffInitial: 500 * time.Millisecond, RetryBackoffMax: 8 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Bedrock: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // AWS services can have occasional issues RetryBackoffInitial: 5 * time.Second, RetryBackoffMax: 40 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Cohere: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Cohere can be variable RetryBackoffInitial: 5 * time.Second, RetryBackoffMax: 40 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Azure: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 600, MaxRetries: 10, RetryBackoffInitial: 20 * time.Second, RetryBackoffMax: 3 * time.Minute, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Vertex: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Google Cloud is generally reliable RetryBackoffInitial: 500 * time.Millisecond, RetryBackoffMax: 8 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Ollama: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 8, // Local service, fewer retries needed RetryBackoffInitial: 250 * time.Millisecond, RetryBackoffMax: 4 * time.Second, BaseURL: os.Getenv("OLLAMA_BASE_URL"), }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Mistral: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Mistral can be variable RetryBackoffInitial: 5 * time.Second, RetryBackoffMax: 5 * time.Minute, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Groq: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Groq can be flaky at times RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 15 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.SGL: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ BaseURL: os.Getenv("SGL_BASE_URL"), DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // SGL (self-hosted) can be variable RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 15 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Parasail: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Parasail can be variable RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Elevenlabs: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Elevenlabs can be variable RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Perplexity: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Perplexity can be variable RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Cerebras: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Cerebras is reasonably stable RetryBackoffInitial: 5 * time.Second, RetryBackoffMax: 3 * time.Minute, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.VLLM: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ BaseURL: os.Getenv("VLLM_BASE_URL"), DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // vllm is stable RetryBackoffInitial: 5 * time.Second, RetryBackoffMax: 3 * time.Minute, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Gemini: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // Gemini can be variable RetryBackoffInitial: 750 * time.Millisecond, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 20, }, }, nil case schemas.OpenRouter: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, // OpenRouter can be variable (proxy service) RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.HuggingFace: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 300, MaxRetries: 10, // HuggingFace can be variable RetryBackoffInitial: 2 * time.Second, RetryBackoffMax: 30 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Nebius: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.XAI: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Replicate: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 300, MaxRetries: 10, RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Runway: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 300, MaxRetries: 10, RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil case schemas.Fireworks: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ DefaultRequestTimeoutInSeconds: 120, MaxRetries: 10, RetryBackoffInitial: 1 * time.Second, RetryBackoffMax: 12 * time.Second, }, ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ Concurrency: Concurrency, BufferSize: 10, }, }, nil default: return nil, fmt.Errorf("unsupported provider: %s", providerKey) } } // AllProviderConfigs contains test configurations for all providers var AllProviderConfigs = []ComprehensiveTestConfig{ { Provider: schemas.OpenAI, ChatModel: "gpt-4o-mini", TextModel: "", // OpenAI doesn't support text completion in newer models ReasoningModel: "o1-mini", // OpenAI reasoning model PromptCachingModel: "gpt-4.1", TranscriptionModel: "whisper-1", SpeechSynthesisModel: "tts-1", ImageGenerationModel: "gpt-image-1", ImageEditModel: "dall-e-2", ImageVariationModel: "dall-e-2", ChatAudioModel: "gpt-4o-mini-audio-preview", Scenarios: TestScenarios{ TextCompletion: false, // Not supported TextCompletionStream: false, // Not supported SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, SpeechSynthesis: true, // OpenAI supports TTS SpeechSynthesisStream: true, // OpenAI supports streaming TTS Transcription: true, // OpenAI supports STT with Whisper TranscriptionStream: true, // OpenAI supports streaming STT ImageGeneration: true, // OpenAI supports image generation with DALL-E ImageGenerationStream: true, // OpenAI supports streaming image generation ImageEdit: true, // OpenAI supports image editing ImageEditStream: true, // OpenAI supports streaming image editing ImageVariation: true, // OpenAI supports image variation ImageVariationStream: false, // OpenAI does not support streaming image variation Embedding: true, Reasoning: true, // OpenAI supports reasoning via o1 models ListModels: true, BatchCreate: true, // OpenAI supports batch API BatchList: true, // OpenAI supports batch API BatchRetrieve: true, // OpenAI supports batch API BatchCancel: true, // OpenAI supports batch API BatchResults: true, // OpenAI supports batch API FileUpload: true, // OpenAI supports file API FileList: true, // OpenAI supports file API FileRetrieve: true, // OpenAI supports file API FileDelete: true, // OpenAI supports file API FileContent: true, // OpenAI supports file API ChatAudio: true, // OpenAI supports chat audio ContainerCreate: true, // OpenAI supports container API ContainerList: true, // OpenAI supports container API ContainerRetrieve: true, // OpenAI supports container API ContainerDelete: true, // OpenAI supports container API ContainerFileCreate: true, // OpenAI supports container file API ContainerFileList: true, // OpenAI supports container file API ContainerFileRetrieve: true, // OpenAI supports container file API ContainerFileContent: true, // OpenAI supports container file API ContainerFileDelete: true, // OpenAI supports container file API }, Fallbacks: []schemas.Fallback{ {Provider: schemas.Anthropic, Model: "claude-3-7-sonnet-20250219"}, }, }, { Provider: schemas.Anthropic, ChatModel: "claude-3-7-sonnet-20250219", TextModel: "", // Anthropic doesn't support text completion Scenarios: TestScenarios{ TextCompletion: false, // Not supported SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, PromptCaching: true, SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: false, ImageGeneration: false, ImageGenerationStream: false, ImageEdit: false, // Anthropic does not support image editing ImageEditStream: false, // Anthropic does not support streaming image editing ImageVariation: false, // Anthropic does not support image variation ImageVariationStream: false, // Anthropic does not support streaming image variation ListModels: true, BatchCreate: true, // Anthropic supports batch API BatchList: true, // Anthropic supports batch API BatchRetrieve: true, // Anthropic supports batch API BatchCancel: true, // Anthropic supports batch API BatchResults: true, // Anthropic supports batch API }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.Bedrock, ChatModel: "anthropic.claude-3-sonnet-20240229-v1:0", TextModel: "", // Bedrock Claude doesn't support text completion ImageEditModel: "amazon.titan-image-generator-v1", ImageVariationModel: "amazon.titan-image-generator-v1", Scenarios: TestScenarios{ TextCompletion: false, // Not supported for Claude SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, PromptCaching: true, SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: true, ImageGeneration: false, ImageGenerationStream: false, ImageEdit: true, // Bedrock supports image editing ImageEditStream: false, // Bedrock does not support streaming image editing ImageVariation: true, // Bedrock supports image variation ImageVariationStream: false, // Bedrock does not support streaming image variation ListModels: true, BatchCreate: true, // Bedrock supports batch via Model Invocation Jobs (requires S3 config) BatchList: true, // Bedrock supports listing batch jobs BatchRetrieve: true, // Bedrock supports retrieving batch jobs BatchCancel: true, // Bedrock supports stopping batch jobs BatchResults: true, // Bedrock batch results via S3 FileUpload: true, // Bedrock file upload to S3 (requires S3 config) FileList: true, // Bedrock file list from S3 (requires S3 config) FileRetrieve: true, // Bedrock file retrieve from S3 (requires S3 config) FileDelete: true, // Bedrock file delete from S3 (requires S3 config) FileContent: true, // Bedrock file content from S3 (requires S3 config) }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.Cohere, ChatModel: "command-a-03-2025", TextModel: "", // Cohere focuses on chat Scenarios: TestScenarios{ TextCompletion: false, // Not typical for Cohere SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: false, // May not support automatic ImageURL: false, // Check if supported ImageBase64: false, // Check if supported MultipleImages: false, // Check if supported CompleteEnd2End: true, ImageGeneration: false, ImageGenerationStream: false, ImageEdit: false, // Cohere does not support image editing ImageEditStream: false, // Cohere does not support streaming image editing ImageVariation: false, // Cohere does not support image variation ImageVariationStream: false, // Cohere does not support streaming image variation SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: true, ListModels: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.Azure, ChatModel: "gpt-4o", TextModel: "", // Azure doesn't support text completion in newer models ChatAudioModel: "gpt-4o-mini-audio-preview", TranscriptionModel: "whisper-1", SpeechSynthesisModel: "gpt-4o-mini-tts", ImageGenerationModel: "gpt-image-1", ImageEditModel: "dall-e-2", Scenarios: TestScenarios{ TextCompletion: false, // Not supported SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, SpeechSynthesis: true, // Supported via gpt-4o-mini-tts SpeechSynthesisStream: true, // Supported via gpt-4o-mini-tts Transcription: true, // Supported via whisper-1 TranscriptionStream: false, // Not properly supported yet by Azure Embedding: true, ImageGeneration: false, // Skipped for Azure ImageGenerationStream: false, // Skipped for Azure ImageEdit: true, // Azure supports image editing ImageEditStream: true, // Azure supports streaming image editing ImageVariation: false, // Azure does not support image variation ImageVariationStream: false, // Azure does not support streaming image variation ListModels: true, BatchCreate: true, // Azure supports batch API BatchList: true, // Azure supports batch API BatchRetrieve: true, // Azure supports batch API BatchCancel: true, // Azure supports batch API BatchResults: true, // Azure supports batch API FileUpload: true, // Azure supports file API FileList: true, // Azure supports file API FileRetrieve: true, // Azure supports file API FileDelete: true, // Azure supports file API FileContent: true, // Azure supports file API ChatAudio: true, // Azure supports chat audio }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, DisableParallelFor: []string{"Transcription"}, // Azure Whisper has 3 calls/minute quota }, { Provider: schemas.Vertex, ChatModel: "gemini-pro", TextModel: "", // Vertex focuses on chat ImageGenerationModel: "imagen-4.0-generate-001", ImageEditModel: "imagen-4.0-generate-001", Scenarios: TestScenarios{ TextCompletion: false, // Not typical SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, ImageGeneration: true, ImageGenerationStream: false, ImageEdit: true, // Vertex supports image editing ImageEditStream: false, // Vertex does not support streaming image editing ImageVariation: false, // Vertex does not support image variation ImageVariationStream: false, // Vertex does not support streaming image variation SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: true, ListModels: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.Mistral, ChatModel: "mistral-large-2411", TextModel: "", // Mistral focuses on chat TranscriptionModel: "voxtral-mini-latest", Scenarios: TestScenarios{ TextCompletion: false, // Not typical SimpleChat: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: true, // Supported via voxtral-mini-latest TranscriptionStream: true, // Supported via voxtral-mini-latest Embedding: true, ImageGeneration: false, ImageGenerationStream: false, ImageEdit: false, // Mistral does not support image editing ImageEditStream: false, // Mistral does not support streaming image editing ImageVariation: false, // Mistral does not support image variation ImageVariationStream: false, // Mistral does not support streaming image variation ListModels: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.Ollama, ChatModel: "llama3.2", TextModel: "", // Ollama focuses on chat Scenarios: TestScenarios{ TextCompletion: false, // Not typical SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: false, ImageGeneration: false, ImageGenerationStream: false, ImageEdit: false, // Ollama does not support image editing ImageEditStream: false, // Ollama does not support streaming image editing ImageVariation: false, // Ollama does not support image variation ImageVariationStream: false, // Ollama does not support streaming image variation ListModels: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.Groq, ChatModel: "llama-3.3-70b-versatile", TextModel: "", // Groq doesn't support text completion Scenarios: TestScenarios{ TextCompletion: false, // Not supported SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: false, ImageGeneration: false, ImageGenerationStream: false, ImageEdit: false, // Groq does not support image editing ImageEditStream: false, // Groq does not support streaming image editing ImageVariation: false, // Groq does not support image variation ImageVariationStream: false, // Groq does not support streaming image variation ListModels: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.Fireworks, ChatModel: "accounts/fireworks/models/deepseek-v3p2", TextModel: "accounts/fireworks/models/deepseek-v3p2", EmbeddingModel: "nomic-ai/nomic-embed-text-v1.5", Scenarios: TestScenarios{ TextCompletion: true, TextCompletionStream: true, SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: false, End2EndToolCalling: false, AutomaticFunctionCall: false, ImageURL: false, ImageBase64: false, MultipleImages: false, FileBase64: false, FileURL: false, CompleteEnd2End: true, Embedding: true, ListModels: true, Reasoning: false, Transcription: false, SpeechSynthesis: false, PromptCaching: false, }, }, { Provider: ProviderOpenAICustom, ChatModel: "llama-3.3-70b-versatile", TextModel: "", // Custom OpenAI instance doesn't support text completion Scenarios: TestScenarios{ TextCompletion: false, SimpleChat: true, // Enable simple chat for testing CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: false, ImageBase64: false, MultipleImages: false, CompleteEnd2End: true, SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: false, ImageGeneration: false, // ProviderOpenAICustom does not support image generation ImageGenerationStream: false, // ProviderOpenAICustom does not support streaming image generation ImageEdit: false, // ProviderOpenAICustom does not support image editing ImageEditStream: false, // ProviderOpenAICustom does not support streaming image editing ImageVariation: false, // ProviderOpenAICustom does not support image variation ImageVariationStream: false, // ProviderOpenAICustom does not support streaming image variation ListModels: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.Gemini, ChatModel: "gemini-2.0-flash", TextModel: "", // GenAI doesn't support text completion in newer models TranscriptionModel: "gemini-2.5-flash", SpeechSynthesisModel: "gemini-2.5-flash-preview-tts", EmbeddingModel: "gemini-embedding-001", ImageGenerationModel: "imagen-4.0-generate-001", ImageEditModel: "imagen-4.0-generate-001", Scenarios: TestScenarios{ TextCompletion: false, // Not supported SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, SpeechSynthesis: true, SpeechSynthesisStream: true, Transcription: true, TranscriptionStream: true, Embedding: true, ImageGeneration: true, ImageGenerationStream: false, ImageEdit: true, // Gemini supports image editing ImageEditStream: false, // Gemini does not support streaming image editing ImageVariation: false, // Gemini does not support image variation ImageVariationStream: false, // Gemini does not support streaming image variation ListModels: true, BatchCreate: true, BatchList: true, BatchRetrieve: true, BatchCancel: true, BatchResults: true, FileUpload: true, FileList: true, FileRetrieve: true, FileDelete: true, FileContent: false, // Gemini doesn't support direct content download }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.OpenRouter, ChatModel: "openai/gpt-4o", TextModel: "google/gemini-2.5-flash", Scenarios: TestScenarios{ TextCompletion: true, SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, ImageGeneration: false, ImageGenerationStream: false, ImageEdit: false, // OpenRouter does not support image editing ImageEditStream: false, // OpenRouter does not support streaming image editing ImageVariation: false, // OpenRouter does not support image variation ImageVariationStream: false, // OpenRouter does not support streaming image variation SpeechSynthesis: false, SpeechSynthesisStream: false, Transcription: false, TranscriptionStream: false, Embedding: false, ListModels: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.HuggingFace, ChatModel: "groq/openai/gpt-oss-120b", VisionModel: "novita/zai-org/GLM-4.6V-Flash", EmbeddingModel: "sambanova/intfloat/e5-mistral-7b-instruct", TranscriptionModel: "fal-ai/openai/whisper-large-v3", SpeechSynthesisModel: "fal-ai/hexgrad/Kokoro-82M", ImageGenerationModel: "fal-ai/fal-ai/flux-2", ImageEditModel: "fal-ai/fal-ai/flux-2", Scenarios: TestScenarios{ TextCompletion: false, TextCompletionStream: false, SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: false, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, Embedding: true, ImageGeneration: true, ImageGenerationStream: true, ImageEdit: true, // HuggingFace (fal-ai) supports image editing ImageEditStream: true, // HuggingFace (fal-ai) supports streaming image editing ImageVariation: false, // HuggingFace does not support image variation ImageVariationStream: false, // HuggingFace does not support streaming image variation Transcription: true, TranscriptionStream: false, SpeechSynthesis: true, SpeechSynthesisStream: false, Reasoning: false, ListModels: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, { Provider: schemas.XAI, ChatModel: "grok-4-0709", TextModel: "", // XAI focuses on chat ImageGenerationModel: "grok-2-image", Scenarios: TestScenarios{ TextCompletion: false, // Not typical SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: false, // Not supported ImageGeneration: true, ImageGenerationStream: false, ImageEdit: false, // XAI does not support image editing ImageEditStream: false, // XAI does not support streaming image editing ImageVariation: false, // XAI does not support image variation ImageVariationStream: false, // XAI does not support streaming image variation ListModels: true, }, }, { Provider: schemas.Replicate, ChatModel: "openai/gpt-4.1-mini", TextModel: "openai/gpt-4.1-mini", ImageGenerationModel: "black-forest-labs/flux-dev", Scenarios: TestScenarios{ TextCompletion: false, // Not typical SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, AutomaticFunctionCall: true, ImageURL: true, ImageBase64: true, MultipleImages: true, CompleteEnd2End: true, SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: false, // Not supported TranscriptionStream: false, // Not supported Embedding: false, // Not supported ListModels: true, ImageGeneration: true, ImageGenerationStream: false, }, }, { Provider: schemas.VLLM, ChatModel: "Qwen/Qwen3-0.6B", TextModel: "Qwen/Qwen3-0.6B", EmbeddingModel: "Qwen/Qwen3-Embedding-0.6B", TranscriptionModel: "openai/whisper-small", Scenarios: TestScenarios{ SpeechSynthesis: false, // Not supported SpeechSynthesisStream: false, // Not supported Transcription: true, // VLLM supports transcription TranscriptionStream: true, // VLLM supports transcription streaming Embedding: true, // VLLM supports embedding ImageGeneration: false, ImageGenerationStream: false, ImageEdit: false, // VLLM does not support image editing ImageEditStream: false, // VLLM does not support streaming image editing ImageVariation: false, // VLLM does not support image variation ImageVariationStream: false, // VLLM does not support streaming image variation ListModels: true, TextCompletion: true, TextCompletionStream: true, SimpleChat: true, CompletionStream: true, MultiTurnConversation: true, ToolCalls: true, ToolCallsStreaming: true, MultipleToolCalls: true, MultipleToolCallsStreaming: true, End2EndToolCalling: true, }, Fallbacks: []schemas.Fallback{ {Provider: schemas.OpenAI, Model: "gpt-4o-mini"}, }, }, }