package llmtests import ( "context" "fmt" "testing" "time" bifrost "github.com/maximhq/bifrost/core" "github.com/maximhq/bifrost/core/schemas" ) const ( videoTestPrompt = "A cinematic aerial shot of mountains at sunrise with soft clouds" videoRemixPrompt = "Add dramatic evening lighting with golden hour colors" videoRetrievePollDelay = 5 * time.Second videoCompletionTimeout = 6 * time.Minute videoRetrieveMaxRetries = 6 ) func RunVideoGenerationTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { if !testConfig.Scenarios.VideoGeneration { t.Logf("Video generation not supported for provider %s", testConfig.Provider) return } if testConfig.VideoGenerationModel == "" { t.Logf("Video generation model not configured for provider %s", testConfig.Provider) return } t.Run("VideoGeneration", func(t *testing.T) { ShouldRunParallel(t, testConfig, "VideoGeneration") resp, err := createVideoJob(client, ctx, testConfig) if err != nil { t.Fatalf("❌ Video generation failed: %s", GetErrorMessage(err)) } if resp == nil { t.Fatal("❌ Video generation response is nil") } if resp.ID == "" { t.Fatal("❌ Video generation returned empty ID") } if !isValidVideoStatus(resp.Status) { t.Fatalf("❌ Video generation returned invalid status: %s", resp.Status) } if resp.ExtraFields.Provider == "" { t.Fatal("❌ Video generation extra_fields.provider is empty") } if resp.ExtraFields.OriginalModelRequested == "" { t.Fatal("❌ Video generation extra_fields.original_model_requested is empty") } t.Logf("✅ Video generation created job: id=%s status=%s", resp.ID, resp.Status) }) } func RunVideoRetrieveTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { if !testConfig.Scenarios.VideoRetrieve { t.Logf("Video retrieve not supported for provider %s", testConfig.Provider) return } if testConfig.VideoGenerationModel == "" { t.Logf("Video retrieve skipped: video model not configured for provider %s", testConfig.Provider) return } t.Run("VideoRetrieve", func(t *testing.T) { ShouldRunParallel(t, testConfig, "VideoRetrieve") created, err := createVideoJob(client, ctx, testConfig) if err != nil { t.Fatalf("❌ Video generation (for retrieve test) failed: %s", GetErrorMessage(err)) } if created == nil || created.ID == "" { t.Fatal("❌ Video generation (for retrieve test) returned invalid response") } retrieved, err := retrieveVideoWithRetries(client, ctx, testConfig, created.ID) if err != nil { t.Fatalf("❌ Video retrieve failed: %s", GetErrorMessage(err)) } if retrieved == nil { t.Fatal("❌ Video retrieve returned nil response") } if retrieved.ID == "" { t.Fatal("❌ Video retrieve returned empty ID") } if !isValidVideoStatus(retrieved.Status) { t.Fatalf("❌ Video retrieve returned invalid status: %s", retrieved.Status) } t.Logf("✅ Video retrieve successful: id=%s status=%s", retrieved.ID, retrieved.Status) }) } func RunVideoRemixTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { if !testConfig.Scenarios.VideoRemix { t.Logf("Video remix not supported for provider %s", testConfig.Provider) return } if testConfig.VideoGenerationModel == "" { t.Logf("Video remix skipped: video model not configured for provider %s", testConfig.Provider) return } t.Run("VideoRemix", func(t *testing.T) { ShouldRunParallel(t, testConfig, "VideoRemix") created, err := createVideoJob(client, ctx, testConfig) if err != nil { t.Fatalf("❌ Video generation (for remix test) failed: %s", GetErrorMessage(err)) } if created == nil || created.ID == "" { t.Fatal("❌ Video generation (for remix test) returned invalid response") } completed, pollErr := waitForVideoCompletion(client, ctx, testConfig, created.ID, false) if pollErr != nil { t.Fatalf("❌ Video completion polling (for remix test) failed: %s", GetErrorMessage(pollErr)) } if completed == nil { t.Fatal("❌ Video completion polling (for remix test) returned nil response") } if completed.Status != schemas.VideoStatusCompleted { t.Fatalf("❌ Video did not complete before remix: status=%s, error=%s", completed.Status, completed.Error.Message) } remixReq := &schemas.BifrostVideoRemixRequest{ Provider: testConfig.Provider, ID: created.ID, Input: &schemas.VideoGenerationInput{ Prompt: videoRemixPrompt, }, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) remixResp, remixErr := client.VideoRemixRequest(bfCtx, remixReq) if remixErr != nil { t.Fatalf("❌ Video remix failed: %s", GetErrorMessage(remixErr)) } if remixResp == nil { t.Fatal("❌ Video remix returned nil response") } if remixResp.ID == "" { t.Fatal("❌ Video remix returned empty ID") } if !isValidVideoStatus(remixResp.Status) { t.Fatalf("❌ Video remix returned invalid status: %s", remixResp.Status) } if remixResp.RemixedFromVideoID == nil || *remixResp.RemixedFromVideoID == "" { t.Fatal("❌ Video remix returned empty remixed_from_video_id") } if remixResp.ExtraFields.Provider == "" { t.Fatal("❌ Video remix extra_fields.provider is empty") } if remixResp.ExtraFields.RequestType != schemas.VideoRemixRequest { t.Fatalf("❌ Video remix extra_fields.request_type is %s, expected video_remix", remixResp.ExtraFields.RequestType) } t.Logf("✅ Video remix successful: id=%s status=%s remixed_from=%s", remixResp.ID, remixResp.Status, *remixResp.RemixedFromVideoID) }) } func RunVideoDownloadTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { if !testConfig.Scenarios.VideoDownload { t.Logf("Video download not supported for provider %s", testConfig.Provider) return } if testConfig.VideoGenerationModel == "" { t.Logf("Video download skipped: video model not configured for provider %s", testConfig.Provider) return } t.Run("VideoDownload", func(t *testing.T) { ShouldRunParallel(t, testConfig, "VideoDownload") created, err := createVideoJob(client, ctx, testConfig) if err != nil { t.Fatalf("❌ Video generation (for download test) failed: %s", GetErrorMessage(err)) } if created == nil || created.ID == "" { t.Fatal("❌ Video generation (for download test) returned invalid response") } requireURL := testConfig.Provider == schemas.Runway completed, pollErr := waitForVideoCompletion(client, ctx, testConfig, created.ID, requireURL) if pollErr != nil { t.Fatalf("❌ Video completion polling failed: %s", GetErrorMessage(pollErr)) } if completed == nil { t.Fatal("❌ Video completion polling returned nil response") } if completed.Status != schemas.VideoStatusCompleted { t.Fatalf("❌ Video did not complete successfully: status=%s, error=%s", completed.Status, completed.Error.Message) } downloadReq := &schemas.BifrostVideoDownloadRequest{ Provider: testConfig.Provider, ID: created.ID, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) downloadResp, downloadErr := client.VideoDownloadRequest(bfCtx, downloadReq) if downloadErr != nil { t.Fatalf("❌ Video download failed: %s", GetErrorMessage(downloadErr)) } if downloadResp == nil { t.Fatal("❌ Video download returned nil response") } if len(downloadResp.Content) == 0 { t.Fatal("❌ Video download returned empty content") } if downloadResp.ContentType == "" { t.Fatal("❌ Video download returned empty content type") } t.Logf("✅ Video download successful: id=%s bytes=%d content_type=%s", created.ID, len(downloadResp.Content), downloadResp.ContentType) }) } func RunVideoListTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { if !testConfig.Scenarios.VideoList { t.Logf("Video list not supported for provider %s", testConfig.Provider) return } t.Run("VideoList", func(t *testing.T) { ShouldRunParallel(t, testConfig, "VideoList") order := "desc" limit := 5 req := &schemas.BifrostVideoListRequest{ Provider: testConfig.Provider, Order: bifrost.Ptr(order), Limit: bifrost.Ptr(limit), } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) resp, err := client.VideoListRequest(bfCtx, req) if err != nil { t.Fatalf("❌ Video list failed: %s", GetErrorMessage(err)) } if resp == nil { t.Fatal("❌ Video list returned nil response") } if resp.Object == "" { t.Fatal("❌ Video list returned empty object") } t.Logf("✅ Video list successful: object=%s items=%d", resp.Object, len(resp.Data)) }) } func RunVideoDeleteTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { if !testConfig.Scenarios.VideoDelete { t.Logf("Video delete not supported for provider %s", testConfig.Provider) return } if testConfig.VideoGenerationModel == "" { t.Logf("Video delete skipped: video model not configured for provider %s", testConfig.Provider) return } t.Run("VideoDelete", func(t *testing.T) { ShouldRunParallel(t, testConfig, "VideoDelete") created, err := createVideoJob(client, ctx, testConfig) if err != nil { t.Fatalf("❌ Video generation (for delete test) failed: %s", GetErrorMessage(err)) } if created == nil || created.ID == "" { t.Fatal("❌ Video generation (for delete test) returned invalid response") } // OpenAI video jobs cannot be deleted while still processing. // Wait until the job reaches a terminal state before delete. terminalResp, terminalErr := waitForVideoCompletion(client, ctx, testConfig, created.ID, false) if terminalErr != nil { t.Fatalf("❌ Video terminal-state polling failed before delete: %s", GetErrorMessage(terminalErr)) } if terminalResp == nil { t.Fatal("❌ Video terminal-state polling returned nil response") } if terminalResp.Status == schemas.VideoStatusQueued || terminalResp.Status == schemas.VideoStatusInProgress { t.Fatalf("❌ Video is not in terminal state before delete: status=%s", terminalResp.Status) } deleteReq := &schemas.BifrostVideoDeleteRequest{ Provider: testConfig.Provider, ID: created.ID, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) deleteResp, deleteErr := client.VideoDeleteRequest(bfCtx, deleteReq) if deleteErr != nil { t.Fatalf("❌ Video delete failed: %s", GetErrorMessage(deleteErr)) } if deleteResp == nil { t.Fatal("❌ Video delete returned nil response") } if !deleteResp.Deleted { t.Fatal("❌ Video delete returned deleted=false") } if deleteResp.ID == "" { t.Fatal("❌ Video delete returned empty ID") } t.Logf("✅ Video delete successful: id=%s", deleteResp.ID) }) } func RunVideoUnsupportedTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { if testConfig.Scenarios.VideoList || testConfig.Scenarios.VideoDelete || testConfig.Scenarios.VideoRemix { return } t.Run("VideoUnsupported", func(t *testing.T) { ShouldRunParallel(t, testConfig, "VideoUnsupported") bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) _, listErr := client.VideoListRequest(bfCtx, &schemas.BifrostVideoListRequest{ Provider: testConfig.Provider, }) if !isUnsupportedOperationError(listErr) { t.Fatalf("❌ Expected unsupported_operation for VideoList, got: %s", GetErrorMessage(listErr)) } _, deleteErr := client.VideoDeleteRequest(bfCtx, &schemas.BifrostVideoDeleteRequest{ Provider: testConfig.Provider, ID: "video_test_id", }) if !isUnsupportedOperationError(deleteErr) { t.Fatalf("❌ Expected unsupported_operation for VideoDelete, got: %s", GetErrorMessage(deleteErr)) } _, remixErr := client.VideoRemixRequest(bfCtx, &schemas.BifrostVideoRemixRequest{ Provider: testConfig.Provider, ID: "video_test_id", Input: &schemas.VideoGenerationInput{Prompt: "test remix prompt"}, }) if !isUnsupportedOperationError(remixErr) { t.Fatalf("❌ Expected unsupported_operation for VideoRemix, got: %s", GetErrorMessage(remixErr)) } t.Logf("✅ Video unsupported behavior verified for provider %s", testConfig.Provider) }) } func createVideoJob(client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { req := &schemas.BifrostVideoGenerationRequest{ Provider: testConfig.Provider, Model: testConfig.VideoGenerationModel, Input: &schemas.VideoGenerationInput{ Prompt: videoTestPrompt, }, Params: &schemas.VideoGenerationParameters{ Seconds: bifrost.Ptr("4"), }, Fallbacks: testConfig.Fallbacks, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.VideoGenerationRequest(bfCtx, req) } func retrieveVideoWithRetries(client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig, videoID string) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { var lastErr *schemas.BifrostError for attempt := 0; attempt < videoRetrieveMaxRetries; attempt++ { req := &schemas.BifrostVideoRetrieveRequest{ Provider: testConfig.Provider, ID: videoID, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) resp, err := client.VideoRetrieveRequest(bfCtx, req) if err == nil && resp != nil { return resp, nil } lastErr = err time.Sleep(2 * time.Second) } if lastErr != nil { return nil, lastErr } return nil, &schemas.BifrostError{ IsBifrostError: true, Error: &schemas.ErrorField{ Message: "video retrieve failed after retries", }, } } func waitForVideoCompletion( client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig, videoID string, requireURL bool, ) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { deadline := time.Now().Add(videoCompletionTimeout) var lastResp *schemas.BifrostVideoGenerationResponse var lastErr *schemas.BifrostError for time.Now().Before(deadline) { req := &schemas.BifrostVideoRetrieveRequest{ Provider: testConfig.Provider, ID: videoID, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) resp, err := client.VideoRetrieveRequest(bfCtx, req) if err != nil { lastErr = err time.Sleep(videoRetrievePollDelay) continue } if resp == nil { time.Sleep(videoRetrievePollDelay) continue } lastResp = resp if resp.Status == schemas.VideoStatusFailed { return resp, nil } if resp.Status == schemas.VideoStatusCompleted { if !requireURL || (len(resp.Videos) > 0) { return resp, nil } } time.Sleep(videoRetrievePollDelay) } if lastErr != nil { return nil, lastErr } if lastResp != nil { return lastResp, nil } return nil, &schemas.BifrostError{ IsBifrostError: true, Error: &schemas.ErrorField{ Message: fmt.Sprintf("timed out waiting for video completion for id %s", videoID), }, } } func isValidVideoStatus(status schemas.VideoStatus) bool { switch status { case schemas.VideoStatusQueued, schemas.VideoStatusInProgress, schemas.VideoStatusCompleted, schemas.VideoStatusFailed: return true default: return false } } func isUnsupportedOperationError(err *schemas.BifrostError) bool { return err != nil && err.Error != nil && err.Error.Code != nil && *err.Error.Code == "unsupported_operation" }