first commit
This commit is contained in:
Beyhan Oğur
240
tests/governance/providerbudget_test.go
Normal file
240
tests/governance/providerbudget_test.go
Normal file
@@ -0,0 +1,240 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestProviderBudgetExceeded tests provider-specific budgets within a VK by making requests until budget is consumed
|
||||
func TestProviderBudgetExceeded(t *testing.T) {
|
||||
t.Parallel()
|
||||
testData := NewGlobalTestData()
|
||||
defer testData.Cleanup(t)
|
||||
|
||||
// Create a VK with different budgets for different providers
|
||||
createVKResp := MakeRequest(t, APIRequest{
|
||||
Method: "POST",
|
||||
Path: "/api/governance/virtual-keys",
|
||||
Body: CreateVirtualKeyRequest{
|
||||
Name: "test-vk-provider-budget-" + generateRandomID(),
|
||||
Budget: &BudgetRequest{
|
||||
MaxLimit: 1.0, // High overall budget
|
||||
ResetDuration: "1h",
|
||||
},
|
||||
ProviderConfigs: []ProviderConfigRequest{
|
||||
{
|
||||
Provider: "openai",
|
||||
Weight: float64Ptr(1.0),
|
||||
AllowedModels: []string{"*"},
|
||||
KeyIDs: []string{"*"},
|
||||
Budget: &BudgetRequest{
|
||||
MaxLimit: 0.01, // Specific OpenAI budget
|
||||
ResetDuration: "1h",
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "anthropic",
|
||||
Weight: float64Ptr(1.0),
|
||||
AllowedModels: []string{"*"},
|
||||
KeyIDs: []string{"*"},
|
||||
Budget: &BudgetRequest{
|
||||
MaxLimit: 0.01, // Specific Anthropic budget
|
||||
ResetDuration: "1h",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if createVKResp.StatusCode != 200 {
|
||||
t.Fatalf("Failed to create VK: status %d", createVKResp.StatusCode)
|
||||
}
|
||||
|
||||
vkID := ExtractIDFromResponse(t, createVKResp)
|
||||
testData.AddVirtualKey(vkID)
|
||||
|
||||
vk := createVKResp.Body["virtual_key"].(map[string]interface{})
|
||||
vkValue := vk["value"].(string)
|
||||
|
||||
t.Logf("Created VK with OpenAI budget $0.01 and Anthropic budget $0.01")
|
||||
|
||||
// Test OpenAI provider budget exceeded
|
||||
t.Run("OpenAIProviderBudgetExceeded", func(t *testing.T) {
|
||||
providerBudget := 0.01
|
||||
consumedBudget := 0.0
|
||||
requestNum := 1
|
||||
var lastSuccessfulCost float64
|
||||
var shouldStop = false
|
||||
|
||||
for requestNum <= 50 {
|
||||
longPrompt := "Please provide a comprehensive and detailed response to the following question. " +
|
||||
"I need extensive information covering all aspects of the topic. " +
|
||||
"Provide multiple paragraphs with detailed explanations. " +
|
||||
"Request number " + strconv.Itoa(requestNum) + ". " +
|
||||
"Here is a detailed prompt that will consume significant tokens: " +
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. " +
|
||||
"Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. " +
|
||||
"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. " +
|
||||
"Nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit. " +
|
||||
"In voluptate velit esse cillum dolore eu fugiat nulla pariatur. " +
|
||||
"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt. " +
|
||||
"Mollit anim id est laborum. Lorem ipsum dolor sit amet, consectetur adipiscing elit. " +
|
||||
"Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. " +
|
||||
"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. " +
|
||||
"Nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit. " +
|
||||
"In voluptate velit esse cillum dolore eu fugiat nulla pariatur. " +
|
||||
"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt. " +
|
||||
"Mollit anim id est laborum."
|
||||
|
||||
resp := MakeRequest(t, APIRequest{
|
||||
Method: "POST",
|
||||
Path: "/v1/chat/completions",
|
||||
Body: ChatCompletionRequest{
|
||||
Model: "openai/gpt-4o",
|
||||
Messages: []ChatMessage{
|
||||
{
|
||||
Role: "user",
|
||||
Content: longPrompt,
|
||||
},
|
||||
},
|
||||
},
|
||||
VKHeader: &vkValue,
|
||||
})
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
if CheckErrorMessage(t, resp, "budget") || CheckErrorMessage(t, resp, "provider") {
|
||||
t.Logf("Request %d correctly rejected: OpenAI provider budget exceeded", requestNum)
|
||||
t.Logf("Consumed budget: $%.6f (limit: $%.2f)", consumedBudget, providerBudget)
|
||||
t.Logf("Last successful request cost: $%.6f", lastSuccessfulCost)
|
||||
|
||||
if requestNum == 1 {
|
||||
t.Fatalf("First request should have succeeded but was rejected due to budget")
|
||||
}
|
||||
return // Test passed
|
||||
} else {
|
||||
t.Fatalf("Request %d failed with unexpected error (not budget): %v", requestNum, resp.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// Request succeeded - extract actual token usage from response
|
||||
if usage, ok := resp.Body["usage"].(map[string]interface{}); ok {
|
||||
if prompt, ok := usage["prompt_tokens"].(float64); ok {
|
||||
if completion, ok := usage["completion_tokens"].(float64); ok {
|
||||
actualInputTokens := int(prompt)
|
||||
actualOutputTokens := int(completion)
|
||||
actualCost, _ := CalculateCost("openai/gpt-4o", actualInputTokens, actualOutputTokens)
|
||||
|
||||
consumedBudget += actualCost
|
||||
lastSuccessfulCost = actualCost
|
||||
|
||||
t.Logf("Request %d succeeded: input_tokens=%d, output_tokens=%d, cost=$%.6f, consumed=$%.6f/$%.2f",
|
||||
requestNum, actualInputTokens, actualOutputTokens, actualCost, consumedBudget, providerBudget)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
requestNum++
|
||||
|
||||
if shouldStop {
|
||||
break
|
||||
}
|
||||
|
||||
if consumedBudget >= providerBudget {
|
||||
shouldStop = true
|
||||
}
|
||||
}
|
||||
|
||||
t.Fatalf("Made %d requests but never hit provider budget limit (consumed $%.6f / $%.2f) - budget not being enforced",
|
||||
requestNum-1, consumedBudget, providerBudget)
|
||||
})
|
||||
|
||||
// Test Anthropic provider budget exceeded
|
||||
t.Run("AnthropicProviderBudgetExceeded", func(t *testing.T) {
|
||||
providerBudget := 0.01
|
||||
consumedBudget := 0.0
|
||||
requestNum := 1
|
||||
var lastSuccessfulCost float64
|
||||
var shouldStop = false
|
||||
|
||||
for requestNum <= 50 {
|
||||
longPrompt := "Please provide a comprehensive and detailed response to the following question. " +
|
||||
"I need extensive information covering all aspects of the topic. " +
|
||||
"Provide multiple paragraphs with detailed explanations. " +
|
||||
"Request number " + strconv.Itoa(requestNum) + ". " +
|
||||
"Here is a detailed prompt that will consume significant tokens: " +
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. " +
|
||||
"Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. " +
|
||||
"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. " +
|
||||
"Nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit. " +
|
||||
"In voluptate velit esse cillum dolore eu fugiat nulla pariatur. " +
|
||||
"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt. " +
|
||||
"Mollit anim id est laborum. Lorem ipsum dolor sit amet, consectetur adipiscing elit. " +
|
||||
"Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. " +
|
||||
"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. " +
|
||||
"Nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit. " +
|
||||
"In voluptate velit esse cillum dolore eu fugiat nulla pariatur. " +
|
||||
"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt. " +
|
||||
"Mollit anim id est laborum."
|
||||
|
||||
resp := MakeRequest(t, APIRequest{
|
||||
Method: "POST",
|
||||
Path: "/v1/chat/completions",
|
||||
Body: ChatCompletionRequest{
|
||||
Model: "anthropic/claude-3-7-sonnet-20250219",
|
||||
Messages: []ChatMessage{
|
||||
{
|
||||
Role: "user",
|
||||
Content: longPrompt,
|
||||
},
|
||||
},
|
||||
},
|
||||
VKHeader: &vkValue,
|
||||
})
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
if CheckErrorMessage(t, resp, "budget") || CheckErrorMessage(t, resp, "provider") {
|
||||
t.Logf("Request %d correctly rejected: Anthropic provider budget exceeded", requestNum)
|
||||
t.Logf("Consumed budget: $%.6f (limit: $%.2f)", consumedBudget, providerBudget)
|
||||
t.Logf("Last successful request cost: $%.6f", lastSuccessfulCost)
|
||||
|
||||
if requestNum == 1 {
|
||||
t.Fatalf("First request should have succeeded but was rejected due to budget")
|
||||
}
|
||||
return // Test passed
|
||||
} else {
|
||||
t.Fatalf("Request %d failed with unexpected error (not budget): %v", requestNum, resp.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// Request succeeded - extract actual token usage from response
|
||||
if usage, ok := resp.Body["usage"].(map[string]interface{}); ok {
|
||||
if prompt, ok := usage["prompt_tokens"].(float64); ok {
|
||||
if completion, ok := usage["completion_tokens"].(float64); ok {
|
||||
actualInputTokens := int(prompt)
|
||||
actualOutputTokens := int(completion)
|
||||
actualCost, _ := CalculateCost("anthropic/claude-3-7-sonnet-20250219", actualInputTokens, actualOutputTokens)
|
||||
|
||||
consumedBudget += actualCost
|
||||
lastSuccessfulCost = actualCost
|
||||
|
||||
t.Logf("Request %d succeeded: input_tokens=%d, output_tokens=%d, cost=$%.6f, consumed=$%.6f/$%.2f",
|
||||
requestNum, actualInputTokens, actualOutputTokens, actualCost, consumedBudget, providerBudget)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
requestNum++
|
||||
|
||||
if shouldStop {
|
||||
break
|
||||
}
|
||||
|
||||
if consumedBudget >= providerBudget {
|
||||
shouldStop = true
|
||||
}
|
||||
}
|
||||
|
||||
t.Fatalf("Made %d requests but never hit provider budget limit (consumed $%.6f / $%.2f) - budget not being enforced",
|
||||
requestNum-1, consumedBudget, providerBudget)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user