first commit

core/internal/llmtests/simple_chat.go

@@ -0,0 +1,152 @@
+package llmtests
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	bifrost "github.com/maximhq/bifrost/core"
+	"github.com/maximhq/bifrost/core/schemas"
+)
+
+// RunSimpleChatTest executes the simple chat test scenario using dual API testing framework
+func RunSimpleChatTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) {
+	if !testConfig.Scenarios.SimpleChat {
+		t.Logf("Simple chat not supported for provider %s", testConfig.Provider)
+		return
+	}
+
+	t.Run("SimpleChat", func(t *testing.T) {
+		if os.Getenv("SKIP_PARALLEL_TESTS") != "true" {
+			t.Parallel()
+		}
+
+		chatMessages := []schemas.ChatMessage{
+			CreateBasicChatMessage("Hello! What's the capital of France?"),
+		}
+		responsesMessages := []schemas.ResponsesMessage{
+			CreateBasicResponsesMessage("Hello! What's the capital of France?"),
+		}
+
+		// Use retry framework with enhanced validation
+		retryConfig := GetTestRetryConfigForScenario("SimpleChat", testConfig)
+		retryContext := TestRetryContext{
+			ScenarioName: "SimpleChat",
+			ExpectedBehavior: map[string]interface{}{
+				"should_mention_paris": true,
+				"should_be_factual":    true,
+			},
+			TestMetadata: map[string]interface{}{
+				"provider": testConfig.Provider,
+				"model":    testConfig.ChatModel,
+			},
+		}
+
+		// Enhanced validation expectations (same for both APIs)
+		expectations := GetExpectationsForScenario("SimpleChat", testConfig, map[string]interface{}{})
+		expectations = ModifyExpectationsForProvider(expectations, testConfig.Provider)
+		expectations.ShouldContainKeywords = append(expectations.ShouldContainKeywords, "paris")                                   // Should mention Paris as the capital
+		expectations.ShouldNotContainWords = append(expectations.ShouldNotContainWords, []string{"berlin", "london", "madrid"}...) // Common wrong answers
+
+		// Create Chat Completions API retry config
+		chatRetryConfig := ChatRetryConfig{
+			MaxAttempts: retryConfig.MaxAttempts,
+			BaseDelay:   retryConfig.BaseDelay,
+			MaxDelay:    retryConfig.MaxDelay,
+			Conditions:  []ChatRetryCondition{}, // Add specific chat retry conditions as needed
+			OnRetry:     retryConfig.OnRetry,
+			OnFinalFail: retryConfig.OnFinalFail,
+		}
+
+		// Create Responses API retry config
+		responsesRetryConfig := ResponsesRetryConfig{
+			MaxAttempts: retryConfig.MaxAttempts,
+			BaseDelay:   retryConfig.BaseDelay,
+			MaxDelay:    retryConfig.MaxDelay,
+			Conditions:  []ResponsesRetryCondition{}, // Add specific responses retry conditions as needed
+			OnRetry:     retryConfig.OnRetry,
+			OnFinalFail: retryConfig.OnFinalFail,
+		}
+
+		// Test Chat Completions API
+		chatOperation := func() (*schemas.BifrostChatResponse, *schemas.BifrostError) {
+			bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
+			chatReq := &schemas.BifrostChatRequest{
+				Provider: testConfig.Provider,
+				Model:    testConfig.ChatModel,
+				Input:    chatMessages,
+				Params: &schemas.ChatParameters{
+					MaxCompletionTokens: bifrost.Ptr(150),
+				},
+				Fallbacks: testConfig.Fallbacks,
+			}
+			response, err := client.ChatCompletionRequest(bfCtx, chatReq)
+			if err != nil {
+				return nil, err
+			}
+			if response != nil {
+				return response, nil
+			}
+			return nil, &schemas.BifrostError{
+				IsBifrostError: true,
+				Error: &schemas.ErrorField{
+					Message: "No chat response returned",
+				},
+			}
+		}
+
+		chatResponse, chatError := WithChatTestRetry(t, chatRetryConfig, retryContext, expectations, "SimpleChat_Chat", chatOperation)
+
+		// Test Responses API
+		responsesOperation := func() (*schemas.BifrostResponsesResponse, *schemas.BifrostError) {
+			bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
+			responsesReq := &schemas.BifrostResponsesRequest{
+				Provider:  testConfig.Provider,
+				Model:     testConfig.ChatModel,
+				Input:     responsesMessages,
+				Fallbacks: testConfig.Fallbacks,
+			}
+			response, err := client.ResponsesRequest(bfCtx, responsesReq)
+			if err != nil {
+				return nil, err
+			}
+			if response != nil {
+				return response, nil
+			}
+			return nil, &schemas.BifrostError{
+				IsBifrostError: true,
+				Error: &schemas.ErrorField{
+					Message: "No responses response returned",
+				},
+			}
+		}
+
+		responsesResponse, responsesError := WithResponsesTestRetry(t, responsesRetryConfig, retryContext, expectations, "SimpleChat_Responses", responsesOperation)
+
+		// Check that both APIs succeeded
+		if chatError != nil {
+			t.Fatalf("❌ Chat Completions API failed: %s", GetErrorMessage(chatError))
+		}
+		if responsesError != nil {
+			t.Fatalf("❌ Responses API failed: %s", GetErrorMessage(responsesError))
+		}
+
+		// Log results from both APIs
+		if chatResponse != nil {
+			chatContent := GetChatContent(chatResponse)
+			t.Logf("✅ Chat Completions API result: %s", chatContent)
+		}
+
+		if responsesResponse != nil {
+			responsesContent := GetResponsesContent(responsesResponse)
+			t.Logf("✅ Responses API result: %s", responsesContent)
+		}
+
+		// Fail test if either API failed
+		if chatError != nil || responsesError != nil {
+			t.Fatalf("❌ SimpleChat test failed - one or both APIs failed")
+		}
+
+		t.Logf("🎉 Both Chat Completions and Responses APIs passed SimpleChat test!")
+	})
+}