first commit
This commit is contained in:
Beyhan Oğur
272
core/internal/llmtests/tests.go
Normal file
272
core/internal/llmtests/tests.go
Normal file
@@ -0,0 +1,272 @@
|
||||
package llmtests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
bifrost "github.com/maximhq/bifrost/core"
|
||||
"github.com/maximhq/bifrost/core/schemas"
|
||||
)
|
||||
|
||||
// TestScenarioFunc defines the function signature for test scenario functions
|
||||
type TestScenarioFunc func(*testing.T, *bifrost.Bifrost, context.Context, ComprehensiveTestConfig)
|
||||
|
||||
// RunAllComprehensiveTests executes all comprehensive test scenarios for a given configuration
|
||||
func RunAllComprehensiveTests(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if testConfig.SkipReason != "" {
|
||||
t.Skipf("Skipping %s: %s", testConfig.Provider, testConfig.SkipReason)
|
||||
return
|
||||
}
|
||||
|
||||
t.Logf("🚀 Running comprehensive tests for provider: %s", testConfig.Provider)
|
||||
|
||||
// Define all test scenario functions in a slice
|
||||
testScenarios := []TestScenarioFunc{
|
||||
RunTextCompletionTest,
|
||||
RunTextCompletionStreamTest,
|
||||
RunSimpleChatTest,
|
||||
RunChatCompletionStreamTest,
|
||||
RunResponsesStreamTest,
|
||||
RunMultiTurnConversationTest,
|
||||
RunToolCallsTest,
|
||||
RunToolCallsWithEmptyPropertiesTest,
|
||||
RunToolCallsWithNilPropertiesTest,
|
||||
RunToolCallsStreamingTest,
|
||||
RunMultipleToolCallsTest,
|
||||
RunEnd2EndToolCallingTest,
|
||||
RunAutomaticFunctionCallingTest,
|
||||
RunWebSearchToolTest,
|
||||
RunWebSearchToolStreamTest,
|
||||
RunWebSearchToolWithDomainsTest,
|
||||
RunWebSearchToolContextSizesTest,
|
||||
RunWebSearchToolMultiTurnTest,
|
||||
RunWebSearchToolMaxUsesTest,
|
||||
RunImageURLTest,
|
||||
RunImageBase64Test,
|
||||
RunMultipleImagesTest,
|
||||
RunFileBase64Test,
|
||||
RunFileURLTest,
|
||||
RunCompleteEnd2EndTest,
|
||||
RunSpeechSynthesisTest,
|
||||
RunSpeechSynthesisAdvancedTest,
|
||||
RunSpeechSynthesisStreamTest,
|
||||
RunSpeechSynthesisStreamAdvancedTest,
|
||||
RunTranscriptionTest,
|
||||
RunTranscriptionAdvancedTest,
|
||||
RunTranscriptionStreamTest,
|
||||
RunTranscriptionStreamAdvancedTest,
|
||||
RunEmbeddingTest,
|
||||
RunRerankTest,
|
||||
RunChatCompletionReasoningTest,
|
||||
RunMultiTurnReasoningTest,
|
||||
RunResponsesReasoningTest,
|
||||
RunListModelsTest,
|
||||
RunListModelsResponseMarshalTest,
|
||||
RunListModelsErrorMarshalTest,
|
||||
RunListModelsPaginationTest,
|
||||
RunPromptCachingTest,
|
||||
RunPromptCachingToolBlocksTest,
|
||||
RunPromptCachingMultipleToolCallsTest,
|
||||
RunPromptCachingMultiTurnTest,
|
||||
RunImageGenerationTest,
|
||||
RunImageGenerationStreamTest,
|
||||
RunImageEditTest,
|
||||
RunImageEditStreamTest,
|
||||
RunImageVariationTest,
|
||||
RunImageVariationStreamTest,
|
||||
RunVideoGenerationTest,
|
||||
RunVideoRetrieveTest,
|
||||
RunVideoRemixTest,
|
||||
RunVideoDownloadTest,
|
||||
RunVideoListTest,
|
||||
RunVideoDeleteTest,
|
||||
RunBatchCreateTest,
|
||||
RunBatchListTest,
|
||||
RunBatchRetrieveTest,
|
||||
RunBatchCancelTest,
|
||||
RunBatchResultsTest,
|
||||
RunBatchUnsupportedTest,
|
||||
RunFileUploadTest,
|
||||
RunFileListTest,
|
||||
RunFileRetrieveTest,
|
||||
RunFileDeleteTest,
|
||||
RunFileContentTest,
|
||||
RunFileUnsupportedTest,
|
||||
RunFileAndBatchIntegrationTest,
|
||||
RunCountTokenTest,
|
||||
RunChatAudioTest,
|
||||
RunChatAudioStreamTest,
|
||||
RunStructuredOutputChatTest,
|
||||
RunStructuredOutputChatStreamTest,
|
||||
RunStructuredOutputResponsesTest,
|
||||
RunStructuredOutputResponsesStreamTest,
|
||||
RunContainerCreateTest,
|
||||
RunContainerListTest,
|
||||
RunContainerRetrieveTest,
|
||||
RunContainerDeleteTest,
|
||||
RunContainerUnsupportedTest,
|
||||
RunContainerFileCreateTest,
|
||||
RunContainerFileListTest,
|
||||
RunContainerFileRetrieveTest,
|
||||
RunContainerFileContentTest,
|
||||
RunContainerFileDeleteTest,
|
||||
RunContainerFileUnsupportedTest,
|
||||
RunPassthroughExtraParamsTest,
|
||||
RunStreamErrorStatusCodeTest,
|
||||
RunPassthroughAPITest,
|
||||
RunWebSocketResponsesTest,
|
||||
RunRealtimeTest,
|
||||
RunCompactionTest,
|
||||
RunInterleavedThinkingTest,
|
||||
RunFastModeTest,
|
||||
RunEagerInputStreamingTest,
|
||||
RunServerToolsViaOpenAIEndpointTest,
|
||||
}
|
||||
|
||||
// Execute all test scenarios without raw request/response (default behavior)
|
||||
for _, scenarioFunc := range testScenarios {
|
||||
scenarioFunc(t, client, ctx, testConfig)
|
||||
}
|
||||
|
||||
// Execute all test scenarios WITH raw request/response enabled
|
||||
t.Run("WithRawRequestResponse", func(t *testing.T) {
|
||||
rawCtx := context.WithValue(ctx, schemas.BifrostContextKeySendBackRawRequest, true)
|
||||
rawCtx = context.WithValue(rawCtx, schemas.BifrostContextKeySendBackRawResponse, true)
|
||||
rawConfig := testConfig
|
||||
rawConfig.ExpectRawRequestResponse = true
|
||||
for _, scenarioFunc := range testScenarios {
|
||||
scenarioFunc(t, client, rawCtx, rawConfig)
|
||||
}
|
||||
})
|
||||
|
||||
// Print comprehensive summary based on configuration
|
||||
printTestSummary(t, testConfig)
|
||||
}
|
||||
|
||||
// printTestSummary prints a detailed summary of all test scenarios
|
||||
func printTestSummary(t *testing.T, testConfig ComprehensiveTestConfig) {
|
||||
testScenarios := []struct {
|
||||
name string
|
||||
supported bool
|
||||
}{
|
||||
{"TextCompletion", testConfig.Scenarios.TextCompletion && testConfig.TextModel != ""},
|
||||
{"SimpleChat", testConfig.Scenarios.SimpleChat},
|
||||
{"CompletionStream", testConfig.Scenarios.CompletionStream},
|
||||
{"MultiTurnConversation", testConfig.Scenarios.MultiTurnConversation},
|
||||
{"ToolCalls", testConfig.Scenarios.ToolCalls},
|
||||
{"ToolCallsWithEmptyProperties", testConfig.Scenarios.ToolCalls},
|
||||
{"ToolCallsWithNilProperties", testConfig.Scenarios.ToolCalls},
|
||||
{"ToolCallsStreaming", testConfig.Scenarios.ToolCallsStreaming},
|
||||
{"MultipleToolCalls", testConfig.Scenarios.MultipleToolCalls},
|
||||
{"End2EndToolCalling", testConfig.Scenarios.End2EndToolCalling},
|
||||
{"AutomaticFunctionCall", testConfig.Scenarios.AutomaticFunctionCall},
|
||||
{"ImageURL", testConfig.Scenarios.ImageURL},
|
||||
{"ImageBase64", testConfig.Scenarios.ImageBase64},
|
||||
{"MultipleImages", testConfig.Scenarios.MultipleImages},
|
||||
{"FileBase64", testConfig.Scenarios.FileBase64},
|
||||
{"FileURL", testConfig.Scenarios.FileURL},
|
||||
{"CompleteEnd2End", testConfig.Scenarios.CompleteEnd2End},
|
||||
{"WebSearchTool", testConfig.Scenarios.WebSearchTool},
|
||||
{"SpeechSynthesis", testConfig.Scenarios.SpeechSynthesis},
|
||||
{"SpeechSynthesisStream", testConfig.Scenarios.SpeechSynthesisStream},
|
||||
{"Transcription", testConfig.Scenarios.Transcription},
|
||||
{"TranscriptionStream", testConfig.Scenarios.TranscriptionStream},
|
||||
{"Embedding", testConfig.Scenarios.Embedding && testConfig.EmbeddingModel != ""},
|
||||
{"Rerank", testConfig.Scenarios.Rerank && testConfig.RerankModel != ""},
|
||||
{"ChatCompletionReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""},
|
||||
{"MultiTurnReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""},
|
||||
{"ResponsesReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""},
|
||||
{"ListModels", testConfig.Scenarios.ListModels},
|
||||
{"ListModelsResponseMarshal", testConfig.Scenarios.ListModels},
|
||||
{"ListModelsErrorMarshal", testConfig.Scenarios.ListModels},
|
||||
{"PromptCaching", testConfig.Scenarios.SimpleChat && testConfig.PromptCachingModel != ""},
|
||||
{"PromptCachingToolBlocks", testConfig.Scenarios.PromptCaching && testConfig.PromptCachingModel != ""},
|
||||
{"PromptCachingMultipleToolCalls", testConfig.Scenarios.PromptCaching && testConfig.PromptCachingModel != ""},
|
||||
{"PromptCachingMultiTurn", testConfig.Scenarios.PromptCaching && testConfig.PromptCachingModel != ""},
|
||||
{"ImageGeneration", testConfig.Scenarios.ImageGeneration && testConfig.ImageGenerationModel != ""},
|
||||
{"ImageGenerationStream", testConfig.Scenarios.ImageGenerationStream && testConfig.ImageGenerationModel != ""},
|
||||
{"ImageEdit", testConfig.Scenarios.ImageEdit && testConfig.ImageEditModel != ""},
|
||||
{"ImageEditStream", testConfig.Scenarios.ImageEditStream && testConfig.ImageEditModel != ""},
|
||||
{"ImageVariation", testConfig.Scenarios.ImageVariation && testConfig.ImageVariationModel != ""},
|
||||
{"ImageVariationStream", testConfig.Scenarios.ImageVariationStream && testConfig.ImageVariationModel != ""},
|
||||
{"VideoGeneration", testConfig.Scenarios.VideoGeneration && testConfig.VideoGenerationModel != ""},
|
||||
{"VideoRetrieve", testConfig.Scenarios.VideoRetrieve && testConfig.VideoGenerationModel != ""},
|
||||
{"VideoRemix", testConfig.Scenarios.VideoRemix && testConfig.VideoGenerationModel != ""},
|
||||
{"VideoDownload", testConfig.Scenarios.VideoDownload && testConfig.VideoGenerationModel != ""},
|
||||
{"VideoList", testConfig.Scenarios.VideoList},
|
||||
{"VideoDelete", testConfig.Scenarios.VideoDelete},
|
||||
{"VideoUnsupported", !testConfig.Scenarios.VideoGeneration &&
|
||||
!testConfig.Scenarios.VideoRetrieve &&
|
||||
!testConfig.Scenarios.VideoRemix &&
|
||||
!testConfig.Scenarios.VideoDownload &&
|
||||
!testConfig.Scenarios.VideoList &&
|
||||
!testConfig.Scenarios.VideoDelete},
|
||||
{"BatchCreate", testConfig.Scenarios.BatchCreate},
|
||||
{"BatchList", testConfig.Scenarios.BatchList},
|
||||
{"BatchRetrieve", testConfig.Scenarios.BatchRetrieve},
|
||||
{"BatchCancel", testConfig.Scenarios.BatchCancel},
|
||||
{"BatchResults", testConfig.Scenarios.BatchResults},
|
||||
{"BatchUnsupported", !testConfig.Scenarios.BatchCreate && !testConfig.Scenarios.BatchList && !testConfig.Scenarios.BatchRetrieve && !testConfig.Scenarios.BatchCancel && !testConfig.Scenarios.BatchResults},
|
||||
{"FileUpload", testConfig.Scenarios.FileUpload},
|
||||
{"FileList", testConfig.Scenarios.FileList},
|
||||
{"FileRetrieve", testConfig.Scenarios.FileRetrieve},
|
||||
{"FileDelete", testConfig.Scenarios.FileDelete},
|
||||
{"FileContent", testConfig.Scenarios.FileContent},
|
||||
{"FileUnsupported", !testConfig.Scenarios.FileUpload && !testConfig.Scenarios.FileList && !testConfig.Scenarios.FileRetrieve && !testConfig.Scenarios.FileDelete && !testConfig.Scenarios.FileContent},
|
||||
{"FileAndBatchIntegration", testConfig.Scenarios.FileBatchInput},
|
||||
{"CountTokens", testConfig.Scenarios.CountTokens},
|
||||
{"ChatAudio", testConfig.Scenarios.ChatAudio && testConfig.ChatAudioModel != ""},
|
||||
{"ChatAudioStream", testConfig.Scenarios.ChatAudio && testConfig.ChatAudioModel != ""},
|
||||
{"StructuredOutputChat", testConfig.Scenarios.StructuredOutputs},
|
||||
{"StructuredOutputChatStream", testConfig.Scenarios.StructuredOutputs && testConfig.Scenarios.CompletionStream},
|
||||
{"StructuredOutputResponses", testConfig.Scenarios.StructuredOutputs},
|
||||
{"StructuredOutputResponsesStream", testConfig.Scenarios.StructuredOutputs && testConfig.Scenarios.CompletionStream},
|
||||
{"ContainerCreate", testConfig.Scenarios.ContainerCreate},
|
||||
{"ContainerList", testConfig.Scenarios.ContainerList},
|
||||
{"ContainerRetrieve", testConfig.Scenarios.ContainerRetrieve},
|
||||
{"ContainerDelete", testConfig.Scenarios.ContainerDelete},
|
||||
{"ContainerUnsupported", !testConfig.Scenarios.ContainerCreate && !testConfig.Scenarios.ContainerList && !testConfig.Scenarios.ContainerRetrieve && !testConfig.Scenarios.ContainerDelete},
|
||||
{"ContainerFileCreate", testConfig.Scenarios.ContainerFileCreate},
|
||||
{"ContainerFileList", testConfig.Scenarios.ContainerFileList},
|
||||
{"ContainerFileRetrieve", testConfig.Scenarios.ContainerFileRetrieve},
|
||||
{"ContainerFileContent", testConfig.Scenarios.ContainerFileContent},
|
||||
{"ContainerFileDelete", testConfig.Scenarios.ContainerFileDelete},
|
||||
{"ContainerFileUnsupported", !testConfig.Scenarios.ContainerFileCreate && !testConfig.Scenarios.ContainerFileList && !testConfig.Scenarios.ContainerFileRetrieve && !testConfig.Scenarios.ContainerFileContent && !testConfig.Scenarios.ContainerFileDelete},
|
||||
{"PassThroughExtraParams", testConfig.Scenarios.PassThroughExtraParams},
|
||||
{"StreamErrorStatusCode", testConfig.Scenarios.CompletionStream},
|
||||
{"PassthroughAPI", testConfig.Scenarios.PassthroughAPI},
|
||||
{"WebSocketResponses", testConfig.Scenarios.WebSocketResponses && testConfig.ChatModel != ""},
|
||||
{"Realtime", testConfig.Scenarios.Realtime && testConfig.RealtimeModel != ""},
|
||||
{"Compaction", testConfig.Scenarios.Compaction},
|
||||
{"InterleavedThinking", testConfig.Scenarios.InterleavedThinking},
|
||||
{"FastMode", testConfig.Scenarios.FastMode},
|
||||
{"EagerInputStreaming", testConfig.Scenarios.EagerInputStreaming},
|
||||
{"ServerToolsViaOpenAIEndpoint", testConfig.Scenarios.ServerToolsViaOpenAIEndpoint},
|
||||
}
|
||||
|
||||
supported := 0
|
||||
unsupported := 0
|
||||
|
||||
t.Logf("\n%s", strings.Repeat("=", 80))
|
||||
t.Logf("COMPREHENSIVE TEST SUMMARY FOR PROVIDER: %s", strings.ToUpper(string(testConfig.Provider)))
|
||||
t.Logf("%s", strings.Repeat("=", 80))
|
||||
|
||||
for _, scenario := range testScenarios {
|
||||
if scenario.supported {
|
||||
supported++
|
||||
t.Logf("[ENABLED] SUPPORTED: %-25s [ENABLED] Configured to run", scenario.name)
|
||||
} else {
|
||||
unsupported++
|
||||
t.Logf("[SKIPPED] UNSUPPORTED: %-25s [SKIPPED] Not supported by provider", scenario.name)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("%s", strings.Repeat("-", 80))
|
||||
t.Logf("CONFIGURATION SUMMARY:")
|
||||
t.Logf(" [ENABLED] Supported Tests: %d", supported)
|
||||
t.Logf(" [SKIPPED] Unsupported Tests: %d", unsupported)
|
||||
t.Logf(" [TOTAL] Total Test Types: %d", len(testScenarios))
|
||||
t.Logf("")
|
||||
t.Logf("%s\n", strings.Repeat("=", 80))
|
||||
}
|
||||
Reference in New Issue
Block a user