271 lines
11 KiB
Go
271 lines
11 KiB
Go
package llmtests
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
bifrost "github.com/maximhq/bifrost/core"
|
|
"github.com/maximhq/bifrost/core/schemas"
|
|
)
|
|
|
|
// HelloWorldPDFBase64 is a base64 encoded PDF file containing "Hello World!" text.
|
|
// This is a minimal valid PDF for testing document input functionality.
|
|
const HelloWorldPDFBase64 = "data:application/pdf;base64,JVBERi0xLjcKCjEgMCBvYmogICUgZW50cnkgcG9pbnQKPDwKICAvVHlwZSAvQ2F0YWxvZwogIC" +
|
|
"9QYWdlcyAyIDAgUgo+PgplbmRvYmoKCjIgMCBvYmoKPDwKICAvVHlwZSAvUGFnZXwKICAvTWV" +
|
|
"kaWFCb3ggWyAwIDAgMjAwIDIwMCBdCiAgL0NvdW50IDEKICAvS2lkcyBbIDMgMCBSIF0KPj4K" +
|
|
"ZW5kb2JqCgozIDAgb2JqCjw8CiAgL1R5cGUgL1BhZ2UKICAvUGFyZW50IDIgMCBSCiAgL1Jlc" +
|
|
"291cmNlcyA8PAogICAgL0ZvbnQgPDwKICAgICAgL0YxIDQgMCBSCj4+CiAgPj4KICAvQ29udG" +
|
|
"VudHMgNSAwIFIKPj4KZW5kb2JqCgo0IDAgb2JqCjw8CiAgL1R5cGUgL0ZvbnQKICAvU3VidHl" +
|
|
"wZSAvVHlwZTEKICAvQmFzZUZvbnQgL1RpbWVzLVJvbWFuCj4+CmVuZG9iagoKNSAwIG9iago8" +
|
|
"PAogIC9MZW5ndGggNDQKPj4Kc3RyZWFtCkJUCjcwIDUwIFRECi9GMSAxMiBUZgooSGVsbG8gV" +
|
|
"29ybGQhKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCgp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NT" +
|
|
"M1IGYgCjAwMDAwMDAwMTAgMDAwMDAgbiAKMDAwMDAwMDA2MCAwMDAwMCBuIAowMDAwMDAwMTU" +
|
|
"3IDAwMDAwIG4gCjAwMDAwMDAyNTUgMDAwMDAgbiAKMDAwMDAwMDM1MyAwMDAwMCBuIAp0cmFp" +
|
|
"bGVyCjw8CiAgL1NpemUgNgogIC9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0NDkKJSVFT0YK"
|
|
|
|
// CreateDocumentChatMessage creates a ChatMessage with a PDF document in base64 format
|
|
func CreateDocumentChatMessage(text, documentBase64 string) schemas.ChatMessage {
|
|
return schemas.ChatMessage{
|
|
Role: schemas.ChatMessageRoleUser,
|
|
Content: &schemas.ChatMessageContent{
|
|
ContentBlocks: []schemas.ChatContentBlock{
|
|
{Type: schemas.ChatContentBlockTypeText, Text: bifrost.Ptr(text)},
|
|
{
|
|
Type: schemas.ChatContentBlockTypeFile,
|
|
File: &schemas.ChatInputFile{
|
|
FileData: bifrost.Ptr(documentBase64),
|
|
Filename: bifrost.Ptr("test_document.pdf"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// CreateDocumentResponsesMessage creates a ResponsesMessage with a PDF document in base64 format
|
|
func CreateDocumentResponsesMessage(text, documentBase64 string) schemas.ResponsesMessage {
|
|
return schemas.ResponsesMessage{
|
|
Type: bifrost.Ptr(schemas.ResponsesMessageTypeMessage),
|
|
Role: bifrost.Ptr(schemas.ResponsesInputMessageRoleUser),
|
|
Content: &schemas.ResponsesMessageContent{
|
|
ContentBlocks: []schemas.ResponsesMessageContentBlock{
|
|
{Type: schemas.ResponsesInputMessageContentBlockTypeText, Text: bifrost.Ptr(text)},
|
|
{
|
|
Type: schemas.ResponsesInputMessageContentBlockTypeFile,
|
|
ResponsesInputMessageContentBlockFile: &schemas.ResponsesInputMessageContentBlockFile{
|
|
FileData: bifrost.Ptr(documentBase64),
|
|
Filename: bifrost.Ptr("test_document.pdf"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// RunFileBase64Test executes the PDF file input test scenario with separate subtests for each API
|
|
func RunFileBase64Test(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
|
if !testConfig.Scenarios.FileBase64 {
|
|
t.Logf("File base64 not supported for provider %s", testConfig.Provider)
|
|
return
|
|
}
|
|
|
|
// Run Chat Completions subtest
|
|
RunFileBase64ChatCompletionsTest(t, client, ctx, testConfig)
|
|
|
|
// Run Responses API subtest
|
|
RunFileBase64ResponsesTest(t, client, ctx, testConfig)
|
|
}
|
|
|
|
// RunFileBase64ChatCompletionsTest executes the file base64 test using Chat Completions API
|
|
func RunFileBase64ChatCompletionsTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
|
if !testConfig.Scenarios.FileBase64 {
|
|
t.Logf("File base64 not supported for provider %s", testConfig.Provider)
|
|
return
|
|
}
|
|
|
|
t.Run("FileBase64-ChatCompletions", func(t *testing.T) {
|
|
if os.Getenv("SKIP_PARALLEL_TESTS") != "true" {
|
|
t.Parallel()
|
|
}
|
|
|
|
// Create messages for Chat Completions API with base64 PDF document
|
|
chatMessages := []schemas.ChatMessage{
|
|
CreateDocumentChatMessage("What is the main content of this PDF document? Summarize it.", HelloWorldPDFBase64),
|
|
}
|
|
|
|
// Use retry framework for document input requests
|
|
retryConfig := GetTestRetryConfigForScenario("FileInput", testConfig)
|
|
retryContext := TestRetryContext{
|
|
ScenarioName: "FileBase64-ChatCompletions",
|
|
ExpectedBehavior: map[string]interface{}{
|
|
"should_process_pdf": true,
|
|
"should_read_document": true,
|
|
"should_extract_content": true,
|
|
"document_understanding": true,
|
|
},
|
|
TestMetadata: map[string]interface{}{
|
|
"provider": testConfig.Provider,
|
|
"model": testConfig.ChatModel,
|
|
"file_type": "pdf",
|
|
"encoding": "base64",
|
|
"test_content": "Hello World!",
|
|
"expected_keywords": []string{"hello", "world", "pdf", "document"},
|
|
},
|
|
}
|
|
|
|
// Enhanced validation for PDF document processing
|
|
expectations := GetExpectationsForScenario("FileInput", testConfig, map[string]interface{}{})
|
|
expectations = ModifyExpectationsForProvider(expectations, testConfig.Provider)
|
|
expectations.ShouldContainAnyOf = append(expectations.ShouldContainAnyOf, "hello", "world", "pdf", "document")
|
|
expectations.ShouldNotContainWords = append(expectations.ShouldNotContainWords, []string{
|
|
"cannot process", "invalid format", "decode error",
|
|
"unable to read", "no file", "corrupted", "unsupported",
|
|
}...) // PDF processing failure indicators
|
|
|
|
chatRetryConfig := ChatRetryConfig{
|
|
MaxAttempts: retryConfig.MaxAttempts,
|
|
BaseDelay: retryConfig.BaseDelay,
|
|
MaxDelay: retryConfig.MaxDelay,
|
|
Conditions: []ChatRetryCondition{},
|
|
OnRetry: retryConfig.OnRetry,
|
|
OnFinalFail: retryConfig.OnFinalFail,
|
|
}
|
|
|
|
response, chatError := WithChatTestRetry(t, chatRetryConfig, retryContext, expectations, "FileBase64", func() (*schemas.BifrostChatResponse, *schemas.BifrostError) {
|
|
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
|
chatReq := &schemas.BifrostChatRequest{
|
|
Provider: testConfig.Provider,
|
|
Model: testConfig.ChatModel,
|
|
Input: chatMessages,
|
|
Params: &schemas.ChatParameters{
|
|
MaxCompletionTokens: bifrost.Ptr(500),
|
|
},
|
|
Fallbacks: testConfig.Fallbacks,
|
|
}
|
|
return client.ChatCompletionRequest(bfCtx, chatReq)
|
|
})
|
|
|
|
if chatError != nil {
|
|
t.Fatalf("❌ FileBase64 Chat Completions test failed: %v", GetErrorMessage(chatError))
|
|
}
|
|
|
|
// Additional validation for PDF document processing
|
|
content := GetChatContent(response)
|
|
validateDocumentContent(t, content, "Chat Completions")
|
|
|
|
t.Logf("🎉 Chat Completions API passed FileBase64 test!")
|
|
})
|
|
}
|
|
|
|
// RunFileBase64ResponsesTest executes the file base64 test using Responses API
|
|
func RunFileBase64ResponsesTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
|
if !testConfig.Scenarios.FileBase64 {
|
|
t.Logf("File base64 not supported for provider %s", testConfig.Provider)
|
|
return
|
|
}
|
|
|
|
t.Run("FileBase64-Responses", func(t *testing.T) {
|
|
if os.Getenv("SKIP_PARALLEL_TESTS") != "true" {
|
|
t.Parallel()
|
|
}
|
|
|
|
// Create messages for Responses API with base64 PDF document
|
|
responsesMessages := []schemas.ResponsesMessage{
|
|
CreateDocumentResponsesMessage("What is the main content of this PDF document? Summarize it.", HelloWorldPDFBase64),
|
|
}
|
|
|
|
// Set up retry context for document input requests
|
|
retryContext := TestRetryContext{
|
|
ScenarioName: "FileBase64-Responses",
|
|
ExpectedBehavior: map[string]interface{}{
|
|
"should_process_pdf": true,
|
|
"should_read_document": true,
|
|
"should_extract_content": true,
|
|
"document_understanding": true,
|
|
},
|
|
TestMetadata: map[string]interface{}{
|
|
"provider": testConfig.Provider,
|
|
"model": testConfig.ChatModel,
|
|
"file_type": "pdf",
|
|
"encoding": "base64",
|
|
"test_content": "Hello World!",
|
|
"expected_keywords": []string{"hello", "world", "pdf", "document"},
|
|
},
|
|
}
|
|
|
|
// Enhanced validation for PDF document processing
|
|
expectations := GetExpectationsForScenario("FileInput", testConfig, map[string]interface{}{})
|
|
expectations = ModifyExpectationsForProvider(expectations, testConfig.Provider)
|
|
expectations.ShouldContainAnyOf = append(expectations.ShouldContainAnyOf, "hello", "world", "pdf", "document")
|
|
expectations.ShouldNotContainWords = append(expectations.ShouldNotContainWords, []string{
|
|
"cannot process", "invalid format", "decode error",
|
|
"unable to read", "no file", "corrupted", "unsupported",
|
|
}...) // PDF processing failure indicators
|
|
|
|
retryConfig := GetTestRetryConfigForScenario("FileInput", testConfig)
|
|
responsesRetryConfig := ResponsesRetryConfig{
|
|
MaxAttempts: retryConfig.MaxAttempts,
|
|
BaseDelay: retryConfig.BaseDelay,
|
|
MaxDelay: retryConfig.MaxDelay,
|
|
Conditions: []ResponsesRetryCondition{},
|
|
OnRetry: retryConfig.OnRetry,
|
|
OnFinalFail: retryConfig.OnFinalFail,
|
|
}
|
|
|
|
response, responsesError := WithResponsesTestRetry(t, responsesRetryConfig, retryContext, expectations, "FileBase64", func() (*schemas.BifrostResponsesResponse, *schemas.BifrostError) {
|
|
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
|
responsesReq := &schemas.BifrostResponsesRequest{
|
|
Provider: testConfig.Provider,
|
|
Model: testConfig.ChatModel,
|
|
Input: responsesMessages,
|
|
Params: &schemas.ResponsesParameters{
|
|
MaxOutputTokens: bifrost.Ptr(500),
|
|
},
|
|
Fallbacks: testConfig.Fallbacks,
|
|
}
|
|
return client.ResponsesRequest(bfCtx, responsesReq)
|
|
})
|
|
|
|
if responsesError != nil {
|
|
t.Fatalf("❌ FileBase64 Responses test failed: %v", GetErrorMessage(responsesError))
|
|
}
|
|
|
|
// Additional validation for PDF document processing
|
|
content := GetResponsesContent(response)
|
|
validateDocumentContent(t, content, "Responses")
|
|
|
|
t.Logf("🎉 Responses API passed FileBase64 test!")
|
|
})
|
|
}
|
|
|
|
func validateDocumentContent(t *testing.T, content string, apiName string) {
|
|
t.Helper()
|
|
lowerContent := strings.ToLower(content)
|
|
foundHelloWorld := strings.Contains(lowerContent, "hello") && strings.Contains(lowerContent, "world")
|
|
foundDocument := strings.Contains(lowerContent, "document") || strings.Contains(lowerContent, "pdf") ||
|
|
strings.Contains(lowerContent, "file") || strings.Contains(lowerContent, "text")
|
|
|
|
if len(content) < 10 {
|
|
t.Errorf("❌ %s response is too short for document description (got %d chars): %s", apiName, len(content), content)
|
|
return
|
|
}
|
|
|
|
if !foundHelloWorld && !foundDocument {
|
|
t.Errorf("❌ %s model failed to process PDF document - response doesn't reference expected content or document-related terms. Response: %s", apiName, content)
|
|
return
|
|
}
|
|
|
|
if foundHelloWorld {
|
|
t.Logf("✅ %s model successfully extracted 'Hello World' content from PDF document", apiName)
|
|
} else if foundDocument {
|
|
t.Logf("✅ %s model processed PDF document but may not have clearly identified the exact text", apiName)
|
|
} else {
|
|
t.Errorf("❌ %s response doesn't reference document content or expected keywords: %s", apiName, content)
|
|
return
|
|
}
|
|
|
|
t.Logf("✅ %s PDF document processing completed: %s", apiName, content)
|
|
}
|