first commit
This commit is contained in:
Beyhan Oğur
458
core/internal/llmtests/video.go
Normal file
458
core/internal/llmtests/video.go
Normal file
@@ -0,0 +1,458 @@
|
||||
package llmtests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
bifrost "github.com/maximhq/bifrost/core"
|
||||
"github.com/maximhq/bifrost/core/schemas"
|
||||
)
|
||||
|
||||
const (
|
||||
videoTestPrompt = "A cinematic aerial shot of mountains at sunrise with soft clouds"
|
||||
videoRemixPrompt = "Add dramatic evening lighting with golden hour colors"
|
||||
videoRetrievePollDelay = 5 * time.Second
|
||||
videoCompletionTimeout = 6 * time.Minute
|
||||
videoRetrieveMaxRetries = 6
|
||||
)
|
||||
|
||||
func RunVideoGenerationTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.VideoGeneration {
|
||||
t.Logf("Video generation not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
if testConfig.VideoGenerationModel == "" {
|
||||
t.Logf("Video generation model not configured for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("VideoGeneration", func(t *testing.T) {
|
||||
ShouldRunParallel(t, testConfig, "VideoGeneration")
|
||||
|
||||
resp, err := createVideoJob(client, ctx, testConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("❌ Video generation failed: %s", GetErrorMessage(err))
|
||||
}
|
||||
if resp == nil {
|
||||
t.Fatal("❌ Video generation response is nil")
|
||||
}
|
||||
if resp.ID == "" {
|
||||
t.Fatal("❌ Video generation returned empty ID")
|
||||
}
|
||||
if !isValidVideoStatus(resp.Status) {
|
||||
t.Fatalf("❌ Video generation returned invalid status: %s", resp.Status)
|
||||
}
|
||||
|
||||
if resp.ExtraFields.Provider == "" {
|
||||
t.Fatal("❌ Video generation extra_fields.provider is empty")
|
||||
}
|
||||
if resp.ExtraFields.OriginalModelRequested == "" {
|
||||
t.Fatal("❌ Video generation extra_fields.original_model_requested is empty")
|
||||
}
|
||||
|
||||
t.Logf("✅ Video generation created job: id=%s status=%s", resp.ID, resp.Status)
|
||||
})
|
||||
}
|
||||
|
||||
func RunVideoRetrieveTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.VideoRetrieve {
|
||||
t.Logf("Video retrieve not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
if testConfig.VideoGenerationModel == "" {
|
||||
t.Logf("Video retrieve skipped: video model not configured for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("VideoRetrieve", func(t *testing.T) {
|
||||
ShouldRunParallel(t, testConfig, "VideoRetrieve")
|
||||
|
||||
created, err := createVideoJob(client, ctx, testConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("❌ Video generation (for retrieve test) failed: %s", GetErrorMessage(err))
|
||||
}
|
||||
if created == nil || created.ID == "" {
|
||||
t.Fatal("❌ Video generation (for retrieve test) returned invalid response")
|
||||
}
|
||||
|
||||
retrieved, err := retrieveVideoWithRetries(client, ctx, testConfig, created.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("❌ Video retrieve failed: %s", GetErrorMessage(err))
|
||||
}
|
||||
if retrieved == nil {
|
||||
t.Fatal("❌ Video retrieve returned nil response")
|
||||
}
|
||||
if retrieved.ID == "" {
|
||||
t.Fatal("❌ Video retrieve returned empty ID")
|
||||
}
|
||||
if !isValidVideoStatus(retrieved.Status) {
|
||||
t.Fatalf("❌ Video retrieve returned invalid status: %s", retrieved.Status)
|
||||
}
|
||||
|
||||
t.Logf("✅ Video retrieve successful: id=%s status=%s", retrieved.ID, retrieved.Status)
|
||||
})
|
||||
}
|
||||
|
||||
func RunVideoRemixTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.VideoRemix {
|
||||
t.Logf("Video remix not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
if testConfig.VideoGenerationModel == "" {
|
||||
t.Logf("Video remix skipped: video model not configured for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("VideoRemix", func(t *testing.T) {
|
||||
ShouldRunParallel(t, testConfig, "VideoRemix")
|
||||
|
||||
created, err := createVideoJob(client, ctx, testConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("❌ Video generation (for remix test) failed: %s", GetErrorMessage(err))
|
||||
}
|
||||
if created == nil || created.ID == "" {
|
||||
t.Fatal("❌ Video generation (for remix test) returned invalid response")
|
||||
}
|
||||
|
||||
completed, pollErr := waitForVideoCompletion(client, ctx, testConfig, created.ID, false)
|
||||
if pollErr != nil {
|
||||
t.Fatalf("❌ Video completion polling (for remix test) failed: %s", GetErrorMessage(pollErr))
|
||||
}
|
||||
if completed == nil {
|
||||
t.Fatal("❌ Video completion polling (for remix test) returned nil response")
|
||||
}
|
||||
if completed.Status != schemas.VideoStatusCompleted {
|
||||
t.Fatalf("❌ Video did not complete before remix: status=%s, error=%s", completed.Status, completed.Error.Message)
|
||||
}
|
||||
|
||||
remixReq := &schemas.BifrostVideoRemixRequest{
|
||||
Provider: testConfig.Provider,
|
||||
ID: created.ID,
|
||||
Input: &schemas.VideoGenerationInput{
|
||||
Prompt: videoRemixPrompt,
|
||||
},
|
||||
}
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
remixResp, remixErr := client.VideoRemixRequest(bfCtx, remixReq)
|
||||
if remixErr != nil {
|
||||
t.Fatalf("❌ Video remix failed: %s", GetErrorMessage(remixErr))
|
||||
}
|
||||
if remixResp == nil {
|
||||
t.Fatal("❌ Video remix returned nil response")
|
||||
}
|
||||
if remixResp.ID == "" {
|
||||
t.Fatal("❌ Video remix returned empty ID")
|
||||
}
|
||||
if !isValidVideoStatus(remixResp.Status) {
|
||||
t.Fatalf("❌ Video remix returned invalid status: %s", remixResp.Status)
|
||||
}
|
||||
if remixResp.RemixedFromVideoID == nil || *remixResp.RemixedFromVideoID == "" {
|
||||
t.Fatal("❌ Video remix returned empty remixed_from_video_id")
|
||||
}
|
||||
if remixResp.ExtraFields.Provider == "" {
|
||||
t.Fatal("❌ Video remix extra_fields.provider is empty")
|
||||
}
|
||||
if remixResp.ExtraFields.RequestType != schemas.VideoRemixRequest {
|
||||
t.Fatalf("❌ Video remix extra_fields.request_type is %s, expected video_remix", remixResp.ExtraFields.RequestType)
|
||||
}
|
||||
|
||||
t.Logf("✅ Video remix successful: id=%s status=%s remixed_from=%s", remixResp.ID, remixResp.Status, *remixResp.RemixedFromVideoID)
|
||||
})
|
||||
}
|
||||
|
||||
func RunVideoDownloadTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.VideoDownload {
|
||||
t.Logf("Video download not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
if testConfig.VideoGenerationModel == "" {
|
||||
t.Logf("Video download skipped: video model not configured for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("VideoDownload", func(t *testing.T) {
|
||||
ShouldRunParallel(t, testConfig, "VideoDownload")
|
||||
|
||||
created, err := createVideoJob(client, ctx, testConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("❌ Video generation (for download test) failed: %s", GetErrorMessage(err))
|
||||
}
|
||||
if created == nil || created.ID == "" {
|
||||
t.Fatal("❌ Video generation (for download test) returned invalid response")
|
||||
}
|
||||
|
||||
requireURL := testConfig.Provider == schemas.Runway
|
||||
completed, pollErr := waitForVideoCompletion(client, ctx, testConfig, created.ID, requireURL)
|
||||
if pollErr != nil {
|
||||
t.Fatalf("❌ Video completion polling failed: %s", GetErrorMessage(pollErr))
|
||||
}
|
||||
if completed == nil {
|
||||
t.Fatal("❌ Video completion polling returned nil response")
|
||||
}
|
||||
if completed.Status != schemas.VideoStatusCompleted {
|
||||
t.Fatalf("❌ Video did not complete successfully: status=%s, error=%s", completed.Status, completed.Error.Message)
|
||||
}
|
||||
|
||||
downloadReq := &schemas.BifrostVideoDownloadRequest{
|
||||
Provider: testConfig.Provider,
|
||||
ID: created.ID,
|
||||
}
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
downloadResp, downloadErr := client.VideoDownloadRequest(bfCtx, downloadReq)
|
||||
if downloadErr != nil {
|
||||
t.Fatalf("❌ Video download failed: %s", GetErrorMessage(downloadErr))
|
||||
}
|
||||
if downloadResp == nil {
|
||||
t.Fatal("❌ Video download returned nil response")
|
||||
}
|
||||
if len(downloadResp.Content) == 0 {
|
||||
t.Fatal("❌ Video download returned empty content")
|
||||
}
|
||||
if downloadResp.ContentType == "" {
|
||||
t.Fatal("❌ Video download returned empty content type")
|
||||
}
|
||||
|
||||
t.Logf("✅ Video download successful: id=%s bytes=%d content_type=%s", created.ID, len(downloadResp.Content), downloadResp.ContentType)
|
||||
})
|
||||
}
|
||||
|
||||
func RunVideoListTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.VideoList {
|
||||
t.Logf("Video list not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("VideoList", func(t *testing.T) {
|
||||
ShouldRunParallel(t, testConfig, "VideoList")
|
||||
|
||||
order := "desc"
|
||||
limit := 5
|
||||
req := &schemas.BifrostVideoListRequest{
|
||||
Provider: testConfig.Provider,
|
||||
Order: bifrost.Ptr(order),
|
||||
Limit: bifrost.Ptr(limit),
|
||||
}
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
resp, err := client.VideoListRequest(bfCtx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("❌ Video list failed: %s", GetErrorMessage(err))
|
||||
}
|
||||
if resp == nil {
|
||||
t.Fatal("❌ Video list returned nil response")
|
||||
}
|
||||
if resp.Object == "" {
|
||||
t.Fatal("❌ Video list returned empty object")
|
||||
}
|
||||
|
||||
t.Logf("✅ Video list successful: object=%s items=%d", resp.Object, len(resp.Data))
|
||||
})
|
||||
}
|
||||
|
||||
func RunVideoDeleteTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if !testConfig.Scenarios.VideoDelete {
|
||||
t.Logf("Video delete not supported for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
if testConfig.VideoGenerationModel == "" {
|
||||
t.Logf("Video delete skipped: video model not configured for provider %s", testConfig.Provider)
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("VideoDelete", func(t *testing.T) {
|
||||
ShouldRunParallel(t, testConfig, "VideoDelete")
|
||||
|
||||
created, err := createVideoJob(client, ctx, testConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("❌ Video generation (for delete test) failed: %s", GetErrorMessage(err))
|
||||
}
|
||||
if created == nil || created.ID == "" {
|
||||
t.Fatal("❌ Video generation (for delete test) returned invalid response")
|
||||
}
|
||||
|
||||
// OpenAI video jobs cannot be deleted while still processing.
|
||||
// Wait until the job reaches a terminal state before delete.
|
||||
terminalResp, terminalErr := waitForVideoCompletion(client, ctx, testConfig, created.ID, false)
|
||||
if terminalErr != nil {
|
||||
t.Fatalf("❌ Video terminal-state polling failed before delete: %s", GetErrorMessage(terminalErr))
|
||||
}
|
||||
if terminalResp == nil {
|
||||
t.Fatal("❌ Video terminal-state polling returned nil response")
|
||||
}
|
||||
if terminalResp.Status == schemas.VideoStatusQueued || terminalResp.Status == schemas.VideoStatusInProgress {
|
||||
t.Fatalf("❌ Video is not in terminal state before delete: status=%s", terminalResp.Status)
|
||||
}
|
||||
|
||||
deleteReq := &schemas.BifrostVideoDeleteRequest{
|
||||
Provider: testConfig.Provider,
|
||||
ID: created.ID,
|
||||
}
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
deleteResp, deleteErr := client.VideoDeleteRequest(bfCtx, deleteReq)
|
||||
if deleteErr != nil {
|
||||
t.Fatalf("❌ Video delete failed: %s", GetErrorMessage(deleteErr))
|
||||
}
|
||||
if deleteResp == nil {
|
||||
t.Fatal("❌ Video delete returned nil response")
|
||||
}
|
||||
if !deleteResp.Deleted {
|
||||
t.Fatal("❌ Video delete returned deleted=false")
|
||||
}
|
||||
if deleteResp.ID == "" {
|
||||
t.Fatal("❌ Video delete returned empty ID")
|
||||
}
|
||||
|
||||
t.Logf("✅ Video delete successful: id=%s", deleteResp.ID)
|
||||
})
|
||||
}
|
||||
|
||||
func RunVideoUnsupportedTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
|
||||
if testConfig.Scenarios.VideoList || testConfig.Scenarios.VideoDelete || testConfig.Scenarios.VideoRemix {
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("VideoUnsupported", func(t *testing.T) {
|
||||
ShouldRunParallel(t, testConfig, "VideoUnsupported")
|
||||
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
|
||||
_, listErr := client.VideoListRequest(bfCtx, &schemas.BifrostVideoListRequest{
|
||||
Provider: testConfig.Provider,
|
||||
})
|
||||
if !isUnsupportedOperationError(listErr) {
|
||||
t.Fatalf("❌ Expected unsupported_operation for VideoList, got: %s", GetErrorMessage(listErr))
|
||||
}
|
||||
|
||||
_, deleteErr := client.VideoDeleteRequest(bfCtx, &schemas.BifrostVideoDeleteRequest{
|
||||
Provider: testConfig.Provider,
|
||||
ID: "video_test_id",
|
||||
})
|
||||
if !isUnsupportedOperationError(deleteErr) {
|
||||
t.Fatalf("❌ Expected unsupported_operation for VideoDelete, got: %s", GetErrorMessage(deleteErr))
|
||||
}
|
||||
|
||||
_, remixErr := client.VideoRemixRequest(bfCtx, &schemas.BifrostVideoRemixRequest{
|
||||
Provider: testConfig.Provider,
|
||||
ID: "video_test_id",
|
||||
Input: &schemas.VideoGenerationInput{Prompt: "test remix prompt"},
|
||||
})
|
||||
if !isUnsupportedOperationError(remixErr) {
|
||||
t.Fatalf("❌ Expected unsupported_operation for VideoRemix, got: %s", GetErrorMessage(remixErr))
|
||||
}
|
||||
|
||||
t.Logf("✅ Video unsupported behavior verified for provider %s", testConfig.Provider)
|
||||
})
|
||||
}
|
||||
|
||||
func createVideoJob(client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) {
|
||||
req := &schemas.BifrostVideoGenerationRequest{
|
||||
Provider: testConfig.Provider,
|
||||
Model: testConfig.VideoGenerationModel,
|
||||
Input: &schemas.VideoGenerationInput{
|
||||
Prompt: videoTestPrompt,
|
||||
},
|
||||
Params: &schemas.VideoGenerationParameters{
|
||||
Seconds: bifrost.Ptr("4"),
|
||||
},
|
||||
Fallbacks: testConfig.Fallbacks,
|
||||
}
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
return client.VideoGenerationRequest(bfCtx, req)
|
||||
}
|
||||
|
||||
func retrieveVideoWithRetries(client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig, videoID string) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) {
|
||||
var lastErr *schemas.BifrostError
|
||||
for attempt := 0; attempt < videoRetrieveMaxRetries; attempt++ {
|
||||
req := &schemas.BifrostVideoRetrieveRequest{
|
||||
Provider: testConfig.Provider,
|
||||
ID: videoID,
|
||||
}
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
resp, err := client.VideoRetrieveRequest(bfCtx, req)
|
||||
if err == nil && resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
lastErr = err
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
}
|
||||
return nil, &schemas.BifrostError{
|
||||
IsBifrostError: true,
|
||||
Error: &schemas.ErrorField{
|
||||
Message: "video retrieve failed after retries",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func waitForVideoCompletion(
|
||||
client *bifrost.Bifrost,
|
||||
ctx context.Context,
|
||||
testConfig ComprehensiveTestConfig,
|
||||
videoID string,
|
||||
requireURL bool,
|
||||
) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) {
|
||||
deadline := time.Now().Add(videoCompletionTimeout)
|
||||
var lastResp *schemas.BifrostVideoGenerationResponse
|
||||
var lastErr *schemas.BifrostError
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
req := &schemas.BifrostVideoRetrieveRequest{
|
||||
Provider: testConfig.Provider,
|
||||
ID: videoID,
|
||||
}
|
||||
bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
|
||||
resp, err := client.VideoRetrieveRequest(bfCtx, req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
time.Sleep(videoRetrievePollDelay)
|
||||
continue
|
||||
}
|
||||
if resp == nil {
|
||||
time.Sleep(videoRetrievePollDelay)
|
||||
continue
|
||||
}
|
||||
|
||||
lastResp = resp
|
||||
if resp.Status == schemas.VideoStatusFailed {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
if resp.Status == schemas.VideoStatusCompleted {
|
||||
if !requireURL || (len(resp.Videos) > 0) {
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(videoRetrievePollDelay)
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
}
|
||||
if lastResp != nil {
|
||||
return lastResp, nil
|
||||
}
|
||||
|
||||
return nil, &schemas.BifrostError{
|
||||
IsBifrostError: true,
|
||||
Error: &schemas.ErrorField{
|
||||
Message: fmt.Sprintf("timed out waiting for video completion for id %s", videoID),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func isValidVideoStatus(status schemas.VideoStatus) bool {
|
||||
switch status {
|
||||
case schemas.VideoStatusQueued, schemas.VideoStatusInProgress, schemas.VideoStatusCompleted, schemas.VideoStatusFailed:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isUnsupportedOperationError(err *schemas.BifrostError) bool {
|
||||
return err != nil && err.Error != nil && err.Error.Code != nil && *err.Error.Code == "unsupported_operation"
|
||||
}
|
||||
Reference in New Issue
Block a user