273 lines
12 KiB
Go
273 lines
12 KiB
Go
package llmtests
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
|
|
bifrost "github.com/maximhq/bifrost/core"
|
|
"github.com/maximhq/bifrost/core/schemas"
|
|
)
|
|
|
|
// TestScenarioFunc defines the function signature for test scenario functions
|
|
type TestScenarioFunc func(*testing.T, *bifrost.Bifrost, context.Context, ComprehensiveTestConfig)
|
|
|
|
// RunAllComprehensiveTests executes all comprehensive test scenarios for a given configuration
|
|
func RunAllComprehensiveTests(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
|
if testConfig.SkipReason != "" {
|
|
t.Skipf("Skipping %s: %s", testConfig.Provider, testConfig.SkipReason)
|
|
return
|
|
}
|
|
|
|
t.Logf("🚀 Running comprehensive tests for provider: %s", testConfig.Provider)
|
|
|
|
// Define all test scenario functions in a slice
|
|
testScenarios := []TestScenarioFunc{
|
|
RunTextCompletionTest,
|
|
RunTextCompletionStreamTest,
|
|
RunSimpleChatTest,
|
|
RunChatCompletionStreamTest,
|
|
RunResponsesStreamTest,
|
|
RunMultiTurnConversationTest,
|
|
RunToolCallsTest,
|
|
RunToolCallsWithEmptyPropertiesTest,
|
|
RunToolCallsWithNilPropertiesTest,
|
|
RunToolCallsStreamingTest,
|
|
RunMultipleToolCallsTest,
|
|
RunEnd2EndToolCallingTest,
|
|
RunAutomaticFunctionCallingTest,
|
|
RunWebSearchToolTest,
|
|
RunWebSearchToolStreamTest,
|
|
RunWebSearchToolWithDomainsTest,
|
|
RunWebSearchToolContextSizesTest,
|
|
RunWebSearchToolMultiTurnTest,
|
|
RunWebSearchToolMaxUsesTest,
|
|
RunImageURLTest,
|
|
RunImageBase64Test,
|
|
RunMultipleImagesTest,
|
|
RunFileBase64Test,
|
|
RunFileURLTest,
|
|
RunCompleteEnd2EndTest,
|
|
RunSpeechSynthesisTest,
|
|
RunSpeechSynthesisAdvancedTest,
|
|
RunSpeechSynthesisStreamTest,
|
|
RunSpeechSynthesisStreamAdvancedTest,
|
|
RunTranscriptionTest,
|
|
RunTranscriptionAdvancedTest,
|
|
RunTranscriptionStreamTest,
|
|
RunTranscriptionStreamAdvancedTest,
|
|
RunEmbeddingTest,
|
|
RunRerankTest,
|
|
RunChatCompletionReasoningTest,
|
|
RunMultiTurnReasoningTest,
|
|
RunResponsesReasoningTest,
|
|
RunListModelsTest,
|
|
RunListModelsResponseMarshalTest,
|
|
RunListModelsErrorMarshalTest,
|
|
RunListModelsPaginationTest,
|
|
RunPromptCachingTest,
|
|
RunPromptCachingToolBlocksTest,
|
|
RunPromptCachingMultipleToolCallsTest,
|
|
RunPromptCachingMultiTurnTest,
|
|
RunImageGenerationTest,
|
|
RunImageGenerationStreamTest,
|
|
RunImageEditTest,
|
|
RunImageEditStreamTest,
|
|
RunImageVariationTest,
|
|
RunImageVariationStreamTest,
|
|
RunVideoGenerationTest,
|
|
RunVideoRetrieveTest,
|
|
RunVideoRemixTest,
|
|
RunVideoDownloadTest,
|
|
RunVideoListTest,
|
|
RunVideoDeleteTest,
|
|
RunBatchCreateTest,
|
|
RunBatchListTest,
|
|
RunBatchRetrieveTest,
|
|
RunBatchCancelTest,
|
|
RunBatchResultsTest,
|
|
RunBatchUnsupportedTest,
|
|
RunFileUploadTest,
|
|
RunFileListTest,
|
|
RunFileRetrieveTest,
|
|
RunFileDeleteTest,
|
|
RunFileContentTest,
|
|
RunFileUnsupportedTest,
|
|
RunFileAndBatchIntegrationTest,
|
|
RunCountTokenTest,
|
|
RunChatAudioTest,
|
|
RunChatAudioStreamTest,
|
|
RunStructuredOutputChatTest,
|
|
RunStructuredOutputChatStreamTest,
|
|
RunStructuredOutputResponsesTest,
|
|
RunStructuredOutputResponsesStreamTest,
|
|
RunContainerCreateTest,
|
|
RunContainerListTest,
|
|
RunContainerRetrieveTest,
|
|
RunContainerDeleteTest,
|
|
RunContainerUnsupportedTest,
|
|
RunContainerFileCreateTest,
|
|
RunContainerFileListTest,
|
|
RunContainerFileRetrieveTest,
|
|
RunContainerFileContentTest,
|
|
RunContainerFileDeleteTest,
|
|
RunContainerFileUnsupportedTest,
|
|
RunPassthroughExtraParamsTest,
|
|
RunStreamErrorStatusCodeTest,
|
|
RunPassthroughAPITest,
|
|
RunWebSocketResponsesTest,
|
|
RunRealtimeTest,
|
|
RunCompactionTest,
|
|
RunInterleavedThinkingTest,
|
|
RunFastModeTest,
|
|
RunEagerInputStreamingTest,
|
|
RunServerToolsViaOpenAIEndpointTest,
|
|
}
|
|
|
|
// Execute all test scenarios without raw request/response (default behavior)
|
|
for _, scenarioFunc := range testScenarios {
|
|
scenarioFunc(t, client, ctx, testConfig)
|
|
}
|
|
|
|
// Execute all test scenarios WITH raw request/response enabled
|
|
t.Run("WithRawRequestResponse", func(t *testing.T) {
|
|
rawCtx := context.WithValue(ctx, schemas.BifrostContextKeySendBackRawRequest, true)
|
|
rawCtx = context.WithValue(rawCtx, schemas.BifrostContextKeySendBackRawResponse, true)
|
|
rawConfig := testConfig
|
|
rawConfig.ExpectRawRequestResponse = true
|
|
for _, scenarioFunc := range testScenarios {
|
|
scenarioFunc(t, client, rawCtx, rawConfig)
|
|
}
|
|
})
|
|
|
|
// Print comprehensive summary based on configuration
|
|
printTestSummary(t, testConfig)
|
|
}
|
|
|
|
// printTestSummary prints a detailed summary of all test scenarios
|
|
func printTestSummary(t *testing.T, testConfig ComprehensiveTestConfig) {
|
|
testScenarios := []struct {
|
|
name string
|
|
supported bool
|
|
}{
|
|
{"TextCompletion", testConfig.Scenarios.TextCompletion && testConfig.TextModel != ""},
|
|
{"SimpleChat", testConfig.Scenarios.SimpleChat},
|
|
{"CompletionStream", testConfig.Scenarios.CompletionStream},
|
|
{"MultiTurnConversation", testConfig.Scenarios.MultiTurnConversation},
|
|
{"ToolCalls", testConfig.Scenarios.ToolCalls},
|
|
{"ToolCallsWithEmptyProperties", testConfig.Scenarios.ToolCalls},
|
|
{"ToolCallsWithNilProperties", testConfig.Scenarios.ToolCalls},
|
|
{"ToolCallsStreaming", testConfig.Scenarios.ToolCallsStreaming},
|
|
{"MultipleToolCalls", testConfig.Scenarios.MultipleToolCalls},
|
|
{"End2EndToolCalling", testConfig.Scenarios.End2EndToolCalling},
|
|
{"AutomaticFunctionCall", testConfig.Scenarios.AutomaticFunctionCall},
|
|
{"ImageURL", testConfig.Scenarios.ImageURL},
|
|
{"ImageBase64", testConfig.Scenarios.ImageBase64},
|
|
{"MultipleImages", testConfig.Scenarios.MultipleImages},
|
|
{"FileBase64", testConfig.Scenarios.FileBase64},
|
|
{"FileURL", testConfig.Scenarios.FileURL},
|
|
{"CompleteEnd2End", testConfig.Scenarios.CompleteEnd2End},
|
|
{"WebSearchTool", testConfig.Scenarios.WebSearchTool},
|
|
{"SpeechSynthesis", testConfig.Scenarios.SpeechSynthesis},
|
|
{"SpeechSynthesisStream", testConfig.Scenarios.SpeechSynthesisStream},
|
|
{"Transcription", testConfig.Scenarios.Transcription},
|
|
{"TranscriptionStream", testConfig.Scenarios.TranscriptionStream},
|
|
{"Embedding", testConfig.Scenarios.Embedding && testConfig.EmbeddingModel != ""},
|
|
{"Rerank", testConfig.Scenarios.Rerank && testConfig.RerankModel != ""},
|
|
{"ChatCompletionReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""},
|
|
{"MultiTurnReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""},
|
|
{"ResponsesReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""},
|
|
{"ListModels", testConfig.Scenarios.ListModels},
|
|
{"ListModelsResponseMarshal", testConfig.Scenarios.ListModels},
|
|
{"ListModelsErrorMarshal", testConfig.Scenarios.ListModels},
|
|
{"PromptCaching", testConfig.Scenarios.SimpleChat && testConfig.PromptCachingModel != ""},
|
|
{"PromptCachingToolBlocks", testConfig.Scenarios.PromptCaching && testConfig.PromptCachingModel != ""},
|
|
{"PromptCachingMultipleToolCalls", testConfig.Scenarios.PromptCaching && testConfig.PromptCachingModel != ""},
|
|
{"PromptCachingMultiTurn", testConfig.Scenarios.PromptCaching && testConfig.PromptCachingModel != ""},
|
|
{"ImageGeneration", testConfig.Scenarios.ImageGeneration && testConfig.ImageGenerationModel != ""},
|
|
{"ImageGenerationStream", testConfig.Scenarios.ImageGenerationStream && testConfig.ImageGenerationModel != ""},
|
|
{"ImageEdit", testConfig.Scenarios.ImageEdit && testConfig.ImageEditModel != ""},
|
|
{"ImageEditStream", testConfig.Scenarios.ImageEditStream && testConfig.ImageEditModel != ""},
|
|
{"ImageVariation", testConfig.Scenarios.ImageVariation && testConfig.ImageVariationModel != ""},
|
|
{"ImageVariationStream", testConfig.Scenarios.ImageVariationStream && testConfig.ImageVariationModel != ""},
|
|
{"VideoGeneration", testConfig.Scenarios.VideoGeneration && testConfig.VideoGenerationModel != ""},
|
|
{"VideoRetrieve", testConfig.Scenarios.VideoRetrieve && testConfig.VideoGenerationModel != ""},
|
|
{"VideoRemix", testConfig.Scenarios.VideoRemix && testConfig.VideoGenerationModel != ""},
|
|
{"VideoDownload", testConfig.Scenarios.VideoDownload && testConfig.VideoGenerationModel != ""},
|
|
{"VideoList", testConfig.Scenarios.VideoList},
|
|
{"VideoDelete", testConfig.Scenarios.VideoDelete},
|
|
{"VideoUnsupported", !testConfig.Scenarios.VideoGeneration &&
|
|
!testConfig.Scenarios.VideoRetrieve &&
|
|
!testConfig.Scenarios.VideoRemix &&
|
|
!testConfig.Scenarios.VideoDownload &&
|
|
!testConfig.Scenarios.VideoList &&
|
|
!testConfig.Scenarios.VideoDelete},
|
|
{"BatchCreate", testConfig.Scenarios.BatchCreate},
|
|
{"BatchList", testConfig.Scenarios.BatchList},
|
|
{"BatchRetrieve", testConfig.Scenarios.BatchRetrieve},
|
|
{"BatchCancel", testConfig.Scenarios.BatchCancel},
|
|
{"BatchResults", testConfig.Scenarios.BatchResults},
|
|
{"BatchUnsupported", !testConfig.Scenarios.BatchCreate && !testConfig.Scenarios.BatchList && !testConfig.Scenarios.BatchRetrieve && !testConfig.Scenarios.BatchCancel && !testConfig.Scenarios.BatchResults},
|
|
{"FileUpload", testConfig.Scenarios.FileUpload},
|
|
{"FileList", testConfig.Scenarios.FileList},
|
|
{"FileRetrieve", testConfig.Scenarios.FileRetrieve},
|
|
{"FileDelete", testConfig.Scenarios.FileDelete},
|
|
{"FileContent", testConfig.Scenarios.FileContent},
|
|
{"FileUnsupported", !testConfig.Scenarios.FileUpload && !testConfig.Scenarios.FileList && !testConfig.Scenarios.FileRetrieve && !testConfig.Scenarios.FileDelete && !testConfig.Scenarios.FileContent},
|
|
{"FileAndBatchIntegration", testConfig.Scenarios.FileBatchInput},
|
|
{"CountTokens", testConfig.Scenarios.CountTokens},
|
|
{"ChatAudio", testConfig.Scenarios.ChatAudio && testConfig.ChatAudioModel != ""},
|
|
{"ChatAudioStream", testConfig.Scenarios.ChatAudio && testConfig.ChatAudioModel != ""},
|
|
{"StructuredOutputChat", testConfig.Scenarios.StructuredOutputs},
|
|
{"StructuredOutputChatStream", testConfig.Scenarios.StructuredOutputs && testConfig.Scenarios.CompletionStream},
|
|
{"StructuredOutputResponses", testConfig.Scenarios.StructuredOutputs},
|
|
{"StructuredOutputResponsesStream", testConfig.Scenarios.StructuredOutputs && testConfig.Scenarios.CompletionStream},
|
|
{"ContainerCreate", testConfig.Scenarios.ContainerCreate},
|
|
{"ContainerList", testConfig.Scenarios.ContainerList},
|
|
{"ContainerRetrieve", testConfig.Scenarios.ContainerRetrieve},
|
|
{"ContainerDelete", testConfig.Scenarios.ContainerDelete},
|
|
{"ContainerUnsupported", !testConfig.Scenarios.ContainerCreate && !testConfig.Scenarios.ContainerList && !testConfig.Scenarios.ContainerRetrieve && !testConfig.Scenarios.ContainerDelete},
|
|
{"ContainerFileCreate", testConfig.Scenarios.ContainerFileCreate},
|
|
{"ContainerFileList", testConfig.Scenarios.ContainerFileList},
|
|
{"ContainerFileRetrieve", testConfig.Scenarios.ContainerFileRetrieve},
|
|
{"ContainerFileContent", testConfig.Scenarios.ContainerFileContent},
|
|
{"ContainerFileDelete", testConfig.Scenarios.ContainerFileDelete},
|
|
{"ContainerFileUnsupported", !testConfig.Scenarios.ContainerFileCreate && !testConfig.Scenarios.ContainerFileList && !testConfig.Scenarios.ContainerFileRetrieve && !testConfig.Scenarios.ContainerFileContent && !testConfig.Scenarios.ContainerFileDelete},
|
|
{"PassThroughExtraParams", testConfig.Scenarios.PassThroughExtraParams},
|
|
{"StreamErrorStatusCode", testConfig.Scenarios.CompletionStream},
|
|
{"PassthroughAPI", testConfig.Scenarios.PassthroughAPI},
|
|
{"WebSocketResponses", testConfig.Scenarios.WebSocketResponses && testConfig.ChatModel != ""},
|
|
{"Realtime", testConfig.Scenarios.Realtime && testConfig.RealtimeModel != ""},
|
|
{"Compaction", testConfig.Scenarios.Compaction},
|
|
{"InterleavedThinking", testConfig.Scenarios.InterleavedThinking},
|
|
{"FastMode", testConfig.Scenarios.FastMode},
|
|
{"EagerInputStreaming", testConfig.Scenarios.EagerInputStreaming},
|
|
{"ServerToolsViaOpenAIEndpoint", testConfig.Scenarios.ServerToolsViaOpenAIEndpoint},
|
|
}
|
|
|
|
supported := 0
|
|
unsupported := 0
|
|
|
|
t.Logf("\n%s", strings.Repeat("=", 80))
|
|
t.Logf("COMPREHENSIVE TEST SUMMARY FOR PROVIDER: %s", strings.ToUpper(string(testConfig.Provider)))
|
|
t.Logf("%s", strings.Repeat("=", 80))
|
|
|
|
for _, scenario := range testScenarios {
|
|
if scenario.supported {
|
|
supported++
|
|
t.Logf("[ENABLED] SUPPORTED: %-25s [ENABLED] Configured to run", scenario.name)
|
|
} else {
|
|
unsupported++
|
|
t.Logf("[SKIPPED] UNSUPPORTED: %-25s [SKIPPED] Not supported by provider", scenario.name)
|
|
}
|
|
}
|
|
|
|
t.Logf("%s", strings.Repeat("-", 80))
|
|
t.Logf("CONFIGURATION SUMMARY:")
|
|
t.Logf(" [ENABLED] Supported Tests: %d", supported)
|
|
t.Logf(" [SKIPPED] Unsupported Tests: %d", unsupported)
|
|
t.Logf(" [TOTAL] Total Test Types: %d", len(testScenarios))
|
|
t.Logf("")
|
|
t.Logf("%s\n", strings.Repeat("=", 80))
|
|
}
|