first commit
This commit is contained in:
Beyhan Oğur
270
core/internal/llmtests/file_base64.go
Normal file
270
core/internal/llmtests/file_base64.go
Normal file
@@ -0,0 +1,270 @@
|
||||
package llmtests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
bifrost "github.com/maximhq/bifrost/core"
|
||||
"github.com/maximhq/bifrost/core/schemas"
|
||||
)
|
||||
|
||||
// HelloWorldPDFBase64 is a base64 encoded PDF file containing "Hello World!" text.
|
||||
// This is a minimal valid PDF for testing document input functionality.
|
||||
const HelloWorldPDFBase64 = "data:application/pdf;base64,JVBERi0xLjcKCjEgMCBvYmogICUgZW50cnkgcG9pbnQKPDwKICAvVHlwZSAvQ2F0YWxvZwogIC" +
|
||||
"9QYWdlcyAyIDAgUgo+PgplbmRvYmoKCjIgMCBvYmoKPDwKICAvVHlwZSAvUGFnZXwKICAvTWV" +
|
||||
"kaWFCb3ggWyAwIDAgMjAwIDIwMCBdCiAgL0NvdW50IDEKICAvS2lkcyBbIDMgMCBSIF0KPj4K" +
|
||||
"ZW5kb2JqCgozIDAgb2JqCjw8CiAgL1R5cGUgL1BhZ2UKICAvUGFyZW50IDIgMCBSCiAgL1Jlc" +
|
||||
"291cmNlcyA8PAogICAgL0ZvbnQgPDwKICAgICAgL0YxIDQgMCBSCj4+CiAgPj4KICAvQ29udG" +
|
||||
"VudHMgNSAwIFIKPj4KZW5kb2JqCgo0IDAgb2JqCjw8CiAgL1R5cGUgL0ZvbnQKICAvU3VidHl" +
|
||||
"wZSAvVHlwZTEKICAvQmFzZUZvbnQgL1RpbWVzLVJvbWFuCj4+CmVuZG9iagoKNSAwIG9iago8" +
|
||||
"PAogIC9MZW5ndGggNDQKPj4Kc3RyZWFtCkJUCjcwIDUwIFRECi9GMSAxMiBUZgooSGVsbG8gV" +
|
||||
"29ybGQhKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCgp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NT" +
|
||||
"M1IGYgCjAwMDAwMDAwMTAgMDAwMDAgbiAKMDAwMDAwMDA2MCAwMDAwMCBuIAowMDAwMDAwMTU" +
|
||||
"3IDAwMDAwIG4gCjAwMDAwMDAyNTUgMDAwMDAgbiAKMDAwMDAwMDM1MyAwMDAwMCBuIAp0cmFp" +
|
||||
"bGVyCjw8CiAgL1NpemUgNgogIC9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0NDkKJSVFT0YK"
|
||||
|
||||
// CreateDocumentChatMessage creates a ChatMessage with a PDF document in base64 format
|
||||
func CreateDocumentChatMessage(text, documentBase64 string) schemas.ChatMessage {
|
||||
return schemas.ChatMessage{
|
||||
Role: schemas.ChatMessageRoleUser,
|
||||
Content: &schemas.ChatMessageContent{
|
||||
ContentBlocks: []schemas.ChatContentBlock{
|
||||
{Type: schemas.ChatContentBlockTypeText, Text: bifrost.Ptr(text)},
|
||||
{
|
||||
Type: schemas.ChatContentBlockTypeFile,
|
||||
File: &schemas.ChatInputFile{
|
||||
FileData: bifrost.Ptr(documentBase64),
|
||||
Filename: bifrost.Ptr("test_document.pdf"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// CreateDocumentResponsesMessage creates a ResponsesMessage with a PDF document in base64 format
|
||||
func CreateDocumentResponsesMessage(text, documentBase64 string) schemas.ResponsesMessage {
|
||||
return schemas.ResponsesMessage{
|
||||
Type: bifrost.Ptr(schemas.ResponsesMessageTypeMessage),
|
||||
Role: bifrost.Ptr(schemas.ResponsesInputMessageRoleUser),
|
||||
Content: &schemas.ResponsesMessageContent{
|
||||
ContentBlocks: []schemas.ResponsesMessageContentBlock{
|
||||
{Type: schemas.ResponsesInputMessageContentBlockTypeText, Text: bifrost.Ptr(text)},
|
||||
{
|
||||
Type: schemas.ResponsesInputMessageContentBlockTypeFile,
|
||||
ResponsesInputMessageContentBlockFile: &schemas.ResponsesInputMessageContentBlockFile{
|
||||
FileData: bifrost.Ptr(documentBase64),
|
||||
Filename: bifrost.Ptr("test_document.pdf"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// RunFileBase64Test executes the PDF file input test scenario with separate subtests for each API
|
||||
func RunFileBase64Test(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.FileBase64 {
|
||||
t.Logf("File base64 not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
// Run Chat Completions subtest
|
||||
RunFileBase64ChatCompletionsTest(t, client, ctx, testConfig)
|
||||
|
||||
// Run Responses API subtest
|
||||
RunFileBase64ResponsesTest(t, client, ctx, testConfig)
|
||||
}
|
||||
|
||||
// RunFileBase64ChatCompletionsTest executes the file base64 test using Chat Completions API
|
||||
func RunFileBase64ChatCompletionsTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.FileBase64 {
|
||||
t.Logf("File base64 not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("FileBase64-ChatCompletions", func(t *testing.T) {
|
||||
if os.Getenv("SKIP_PARALLEL_TESTS") != "true" {
|
||||
t.Parallel()
|
||||
}
|
||||
|
||||
// Create messages for Chat Completions API with base64 PDF document
|
||||
chatMessages := []schemas.ChatMessage{
|
||||
CreateDocumentChatMessage("What is the main content of this PDF document? Summarize it.", HelloWorldPDFBase64),
|
||||
}
|
||||
|
||||
// Use retry framework for document input requests
|
||||
retryConfig := GetTestRetryConfigForScenario("FileInput", testConfig)
|
||||
retryContext := TestRetryContext{
|
||||
ScenarioName: "FileBase64-ChatCompletions",
|
||||
ExpectedBehavior: map[string]interface{}{
|
||||
"should_process_pdf": true,
|
||||
"should_read_document": true,
|
||||
"should_extract_content": true,
|
||||
"document_understanding": true,
|
||||
},
|
||||
TestMetadata: map[string]interface{}{
|
||||
"provider": testConfig.Provider,
|
||||
"model": testConfig.ChatModel,
|
||||
"file_type": "pdf",
|
||||
"encoding": "base64",
|
||||
"test_content": "Hello World!",
|
||||
"expected_keywords": []string{"hello", "world", "pdf", "document"},
|
||||
},
|
||||
}
|
||||
|
||||
// Enhanced validation for PDF document processing
|
||||
expectations := GetExpectationsForScenario("FileInput", testConfig, map[string]interface{}{})
|
||||
expectations = ModifyExpectationsForProvider(expectations, testConfig.Provider)
|
||||
expectations.ShouldContainAnyOf = append(expectations.ShouldContainAnyOf, "hello", "world", "pdf", "document")
|
||||
expectations.ShouldNotContainWords = append(expectations.ShouldNotContainWords, []string{
|
||||
"cannot process", "invalid format", "decode error",
|
||||
"unable to read", "no file", "corrupted", "unsupported",
|
||||
}...) // PDF processing failure indicators
|
||||
|
||||
chatRetryConfig := ChatRetryConfig{
|
||||
MaxAttempts: retryConfig.MaxAttempts,
|
||||
BaseDelay: retryConfig.BaseDelay,
|
||||
MaxDelay: retryConfig.MaxDelay,
|
||||
Conditions: []ChatRetryCondition{},
|
||||
OnRetry: retryConfig.OnRetry,
|
||||
OnFinalFail: retryConfig.OnFinalFail,
|
||||
}
|
||||
|
||||
response, chatError := WithChatTestRetry(t, chatRetryConfig, retryContext, expectations, "FileBase64", func() (*schemas.BifrostChatResponse, *schemas.BifrostError) {
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
chatReq := &schemas.BifrostChatRequest{
|
||||
Provider: testConfig.Provider,
|
||||
Model: testConfig.ChatModel,
|
||||
Input: chatMessages,
|
||||
Params: &schemas.ChatParameters{
|
||||
MaxCompletionTokens: bifrost.Ptr(500),
|
||||
},
|
||||
Fallbacks: testConfig.Fallbacks,
|
||||
}
|
||||
return client.ChatCompletionRequest(bfCtx, chatReq)
|
||||
})
|
||||
|
||||
if chatError != nil {
|
||||
t.Fatalf("❌ FileBase64 Chat Completions test failed: %v", GetErrorMessage(chatError))
|
||||
}
|
||||
|
||||
// Additional validation for PDF document processing
|
||||
content := GetChatContent(response)
|
||||
validateDocumentContent(t, content, "Chat Completions")
|
||||
|
||||
t.Logf("🎉 Chat Completions API passed FileBase64 test!")
|
||||
})
|
||||
}
|
||||
|
||||
// RunFileBase64ResponsesTest executes the file base64 test using Responses API
|
||||
func RunFileBase64ResponsesTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.FileBase64 {
|
||||
t.Logf("File base64 not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("FileBase64-Responses", func(t *testing.T) {
|
||||
if os.Getenv("SKIP_PARALLEL_TESTS") != "true" {
|
||||
t.Parallel()
|
||||
}
|
||||
|
||||
// Create messages for Responses API with base64 PDF document
|
||||
responsesMessages := []schemas.ResponsesMessage{
|
||||
CreateDocumentResponsesMessage("What is the main content of this PDF document? Summarize it.", HelloWorldPDFBase64),
|
||||
}
|
||||
|
||||
// Set up retry context for document input requests
|
||||
retryContext := TestRetryContext{
|
||||
ScenarioName: "FileBase64-Responses",
|
||||
ExpectedBehavior: map[string]interface{}{
|
||||
"should_process_pdf": true,
|
||||
"should_read_document": true,
|
||||
"should_extract_content": true,
|
||||
"document_understanding": true,
|
||||
},
|
||||
TestMetadata: map[string]interface{}{
|
||||
"provider": testConfig.Provider,
|
||||
"model": testConfig.ChatModel,
|
||||
"file_type": "pdf",
|
||||
"encoding": "base64",
|
||||
"test_content": "Hello World!",
|
||||
"expected_keywords": []string{"hello", "world", "pdf", "document"},
|
||||
},
|
||||
}
|
||||
|
||||
// Enhanced validation for PDF document processing
|
||||
expectations := GetExpectationsForScenario("FileInput", testConfig, map[string]interface{}{})
|
||||
expectations = ModifyExpectationsForProvider(expectations, testConfig.Provider)
|
||||
expectations.ShouldContainAnyOf = append(expectations.ShouldContainAnyOf, "hello", "world", "pdf", "document")
|
||||
expectations.ShouldNotContainWords = append(expectations.ShouldNotContainWords, []string{
|
||||
"cannot process", "invalid format", "decode error",
|
||||
"unable to read", "no file", "corrupted", "unsupported",
|
||||
}...) // PDF processing failure indicators
|
||||
|
||||
retryConfig := GetTestRetryConfigForScenario("FileInput", testConfig)
|
||||
responsesRetryConfig := ResponsesRetryConfig{
|
||||
MaxAttempts: retryConfig.MaxAttempts,
|
||||
BaseDelay: retryConfig.BaseDelay,
|
||||
MaxDelay: retryConfig.MaxDelay,
|
||||
Conditions: []ResponsesRetryCondition{},
|
||||
OnRetry: retryConfig.OnRetry,
|
||||
OnFinalFail: retryConfig.OnFinalFail,
|
||||
}
|
||||
|
||||
response, responsesError := WithResponsesTestRetry(t, responsesRetryConfig, retryContext, expectations, "FileBase64", func() (*schemas.BifrostResponsesResponse, *schemas.BifrostError) {
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
responsesReq := &schemas.BifrostResponsesRequest{
|
||||
Provider: testConfig.Provider,
|
||||
Model: testConfig.ChatModel,
|
||||
Input: responsesMessages,
|
||||
Params: &schemas.ResponsesParameters{
|
||||
MaxOutputTokens: bifrost.Ptr(500),
|
||||
},
|
||||
Fallbacks: testConfig.Fallbacks,
|
||||
}
|
||||
return client.ResponsesRequest(bfCtx, responsesReq)
|
||||
})
|
||||
|
||||
if responsesError != nil {
|
||||
t.Fatalf("❌ FileBase64 Responses test failed: %v", GetErrorMessage(responsesError))
|
||||
}
|
||||
|
||||
// Additional validation for PDF document processing
|
||||
content := GetResponsesContent(response)
|
||||
validateDocumentContent(t, content, "Responses")
|
||||
|
||||
t.Logf("🎉 Responses API passed FileBase64 test!")
|
||||
})
|
||||
}
|
||||
|
||||
func validateDocumentContent(t *testing.T, content string, apiName string) {
|
||||
t.Helper()
|
||||
lowerContent := strings.ToLower(content)
|
||||
foundHelloWorld := strings.Contains(lowerContent, "hello") && strings.Contains(lowerContent, "world")
|
||||
foundDocument := strings.Contains(lowerContent, "document") || strings.Contains(lowerContent, "pdf") ||
|
||||
strings.Contains(lowerContent, "file") || strings.Contains(lowerContent, "text")
|
||||
|
||||
if len(content) < 10 {
|
||||
t.Errorf("❌ %s response is too short for document description (got %d chars): %s", apiName, len(content), content)
|
||||
return
|
||||
}
|
||||
|
||||
if !foundHelloWorld && !foundDocument {
|
||||
t.Errorf("❌ %s model failed to process PDF document - response doesn't reference expected content or document-related terms. Response: %s", apiName, content)
|
||||
return
|
||||
}
|
||||
|
||||
if foundHelloWorld {
|
||||
t.Logf("✅ %s model successfully extracted 'Hello World' content from PDF document", apiName)
|
||||
} else if foundDocument {
|
||||
t.Logf("✅ %s model processed PDF document but may not have clearly identified the exact text", apiName)
|
||||
} else {
|
||||
t.Errorf("❌ %s response doesn't reference document content or expected keywords: %s", apiName, content)
|
||||
return
|
||||
}
|
||||
|
||||
t.Logf("✅ %s PDF document processing completed: %s", apiName, content)
|
||||
}
|
||||
Reference in New Issue
Block a user