beyhan/bifrost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Beyhan Oğur
2026-04-26 21:52:23 +03:00
commit 880f412e2c
2662 changed files with 866266 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

507
framework/modelcatalog/pricing_overrides_test.go Normal file
View File

@@ -0,0 +1,507 @@
package modelcatalog
import (
"testing"
bifrost "github.com/maximhq/bifrost/core"
"github.com/maximhq/bifrost/core/schemas"
configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type noOpLogger struct{}
func (noOpLogger) Debug(string, ...any) {}
func (noOpLogger) Info(string, ...any) {}
func (noOpLogger) Warn(string, ...any) {}
func (noOpLogger) Error(string, ...any) {}
func (noOpLogger) Fatal(string, ...any) {}
func (noOpLogger) SetLevel(schemas.LogLevel) {}
func (noOpLogger) SetOutputType(schemas.LoggerOutputType) {}
func (noOpLogger) LogHTTPRequest(schemas.LogLevel, string) schemas.LogEventBuilder {
return schemas.NoopLogEvent
}
func TestGetPricing_OverridePrecedenceExactWildcard(t *testing.T) {
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{
Model: "gpt-4o",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
providerID := "openai"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "openai-override-0",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeWildcard),
Pattern: "gpt-*",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":10}`,
},
{
ID: "openai-override-1",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-4o",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":20}`,
},
}))
pricing := mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
require.NotNil(t, pricing.InputCostPerToken)
assert.Equal(t, 20.0, *pricing.InputCostPerToken)
}
func TestGetPricing_RequestTypeSpecificOverrideBeatsGeneric(t *testing.T) {
t.Skip()
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("gpt-4o", "openai", "responses")] = configstoreTables.TableModelPricing{
Model: "gpt-4o",
Provider: "openai",
Mode: "responses",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
providerID := "openai"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "openai-generic",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-4o",
PricingPatchJSON: `{"input_cost_per_token":9}`,
},
{
ID: "openai-specific",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-4o",
RequestTypes: []schemas.RequestType{schemas.ResponsesRequest},
PricingPatchJSON: `{"input_cost_per_token":15}`,
},
}))
pricing := mc.resolvePricing("openai", "gpt-4o", "", schemas.ResponsesRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
assert.Equal(t, 15.0, pricing.InputCostPerToken)
}
func TestGetPricing_AppliesOverrideAfterFallbackResolution(t *testing.T) {
t.Skip()
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("gpt-4o", "vertex", "chat")] = configstoreTables.TableModelPricing{
Model: "gpt-4o",
Provider: "vertex",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
geminiProviderID := "gemini"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "gemini-override",
ScopeKind: string(ScopeKindProvider),
ProviderID: &geminiProviderID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-4o",
PricingPatchJSON: `{"input_cost_per_token":7}`,
},
}))
pricing := mc.resolvePricing("gemini", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "gemini"})
require.NotNil(t, pricing)
assert.Equal(t, 7.0, pricing.InputCostPerToken)
}
func TestGetPricing_DeploymentLookupUsesResolvedModelForOverrideMatching(t *testing.T) {
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("dep-gpt4o", "openai", "chat")] = configstoreTables.TableModelPricing{
Model: "dep-gpt4o",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
providerID := "openai"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "resolved-model-override",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeExact),
Pattern: "dep-gpt4o",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":7}`,
},
}))
// Override pattern matches the resolved model name ("dep-gpt4o"), not the
// originally requested name ("gpt-4o"), because resolved model has priority.
pricing := mc.resolvePricing("openai", "gpt-4o", "dep-gpt4o", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
require.NotNil(t, pricing.InputCostPerToken)
assert.Equal(t, 7.0, *pricing.InputCostPerToken)
}
func TestGetPricing_FallbackUsesRequestedProviderForScopeMatching(t *testing.T) {
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("gpt-4o", "vertex", "chat")] = configstoreTables.TableModelPricing{
Model: "gpt-4o",
Provider: "vertex",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
geminiProviderID := "gemini"
vertexProviderID := "vertex"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "gemini-provider-override",
ScopeKind: string(ScopeKindProvider),
ProviderID: &geminiProviderID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-4o",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":5}`,
},
{
ID: "vertex-provider-override",
ScopeKind: string(ScopeKindProvider),
ProviderID: &vertexProviderID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-4o",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":9}`,
},
}))
pricing := mc.resolvePricing("gemini", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "gemini"})
require.NotNil(t, pricing)
require.NotNil(t, pricing.InputCostPerToken)
assert.Equal(t, 5.0, *pricing.InputCostPerToken)
}
func TestGetPricing_ExactOverrideDoesNotMatchProviderPrefixedModel(t *testing.T) {
t.Skip()
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("openai/gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{
Model: "openai/gpt-4o",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
providerID := "openai"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "openai-override-0",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-4o",
PricingPatchJSON: `{"input_cost_per_token":19}`,
},
}))
pricing := mc.resolvePricing("openai", "openai/gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
assert.Equal(t, 1.0, pricing.InputCostPerToken)
}
func TestGetPricing_NoMatchingOverrideLeavesPricingUnchanged(t *testing.T) {
t.Skip()
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
baseCacheRead := 0.4
mc.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{
Model: "gpt-4o",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
CacheReadInputTokenCost: &baseCacheRead,
}
providerID := "openai"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "openai-override-0",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeWildcard),
Pattern: "claude-*",
PricingPatchJSON: `{"input_cost_per_token":9}`,
},
}))
pricing := mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
assert.Equal(t, 1.0, pricing.InputCostPerToken)
assert.Equal(t, 2.0, pricing.OutputCostPerToken)
require.NotNil(t, pricing.CacheReadInputTokenCost)
assert.Equal(t, 0.4, *pricing.CacheReadInputTokenCost)
}
func TestDeleteProviderPricingOverrides_StopsApplying(t *testing.T) {
t.Skip()
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{
Model: "gpt-4o",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
providerID := "openai"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "openai-override-0",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-4o",
PricingPatchJSON: `{"input_cost_per_token":11}`,
},
}))
pricing := mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
assert.Equal(t, 11.0, pricing.InputCostPerToken)
require.NoError(t, mc.SetPricingOverrides(nil))
pricing = mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
assert.Equal(t, 1.0, pricing.InputCostPerToken)
}
func TestGetPricing_WildcardSpecificityLongerLiteralWins(t *testing.T) {
t.Skip()
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("gpt-4o-mini", "openai", "chat")] = configstoreTables.TableModelPricing{
Model: "gpt-4o-mini",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
providerID := "openai"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "openai-override-0",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeWildcard),
Pattern: "gpt-*",
PricingPatchJSON: `{"input_cost_per_token":5}`,
},
{
ID: "openai-override-1",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeWildcard),
Pattern: "gpt-4o*",
PricingPatchJSON: `{"input_cost_per_token":6}`,
},
}))
pricing := mc.resolvePricing("openai", "gpt-4o-mini", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
assert.Equal(t, 6.0, pricing.InputCostPerToken)
}
// TestGetPricing_FirstInsertionWinsOnTie verifies that when multiple wildcard overrides
// match the same model and scope, the first one inserted takes precedence.
func TestGetPricing_FirstInsertionWinsOnTie(t *testing.T) {
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
mc.pricingData[makeKey("gpt-4o-mini", "openai", "chat")] = configstoreTables.TableModelPricing{
Model: "gpt-4o-mini",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
providerID := "openai"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "a-override",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeWildcard),
Pattern: "gpt-4o*",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":8}`,
},
{
ID: "b-override",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerID,
MatchType: string(MatchTypeWildcard),
Pattern: "gpt-4o*",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":9}`,
},
}))
pricing := mc.resolvePricing("openai", "gpt-4o-mini", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"})
require.NotNil(t, pricing)
require.NotNil(t, pricing.InputCostPerToken)
assert.Equal(t, 8.0, *pricing.InputCostPerToken)
}
func TestPatchPricing_PartialPatchOnlyChangesSpecifiedFields(t *testing.T) {
t.Skip()
baseCacheRead := 0.4
baseInputImage := 0.7
base := configstoreTables.TableModelPricing{
Model: "gpt-4o",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
CacheReadInputTokenCost: &baseCacheRead,
InputCostPerImage: &baseInputImage,
}
cacheRead := 0.9
patched := patchPricing(base, PricingOptions{
InputCostPerToken: bifrost.Ptr(3.0),
CacheReadInputTokenCost: &cacheRead,
})
assert.Equal(t, 3.0, patched.InputCostPerToken)
require.NotNil(t, patched.CacheReadInputTokenCost)
assert.Equal(t, 0.9, *patched.CacheReadInputTokenCost)
assert.Equal(t, 2.0, patched.OutputCostPerToken)
require.NotNil(t, patched.InputCostPerImage)
assert.Equal(t, 0.7, *patched.InputCostPerImage)
}
func TestApplyScopedPricingOverrides_ScopePrecedence(t *testing.T) {
mc := newTestCatalog(nil, nil)
mc.logger = noOpLogger{}
providerScopeID := "openai"
providerKeyScopeID := "provider-key-1"
virtualKeyScopeID := "virtual-key-1"
require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{
{
ID: "global",
ScopeKind: string(ScopeKindGlobal),
MatchType: string(MatchTypeExact),
Pattern: "gpt-5-nano",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":2}`,
},
{
ID: "provider",
ScopeKind: string(ScopeKindProvider),
ProviderID: &providerScopeID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-5-nano",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":3}`,
},
{
ID: "provider-key",
ScopeKind: string(ScopeKindProviderKey),
ProviderKeyID: &providerKeyScopeID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-5-nano",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":4}`,
},
{
ID: "virtual-key",
ScopeKind: string(ScopeKindVirtualKey),
VirtualKeyID: &virtualKeyScopeID,
MatchType: string(MatchTypeExact),
Pattern: "gpt-5-nano",
RequestTypes: []schemas.RequestType{schemas.ChatCompletionRequest},
PricingPatchJSON: `{"input_cost_per_token":5}`,
},
}))
base := configstoreTables.TableModelPricing{
Model: "gpt-5-nano",
Provider: "openai",
Mode: "chat",
InputCostPerToken: bifrost.Ptr(1.0),
OutputCostPerToken: bifrost.Ptr(2.0),
}
tests := []struct {
name string
scopes PricingLookupScopes
expected float64
}{
{
name: "virtual key wins over provider key, provider and global",
scopes: PricingLookupScopes{
VirtualKeyID: virtualKeyScopeID,
SelectedKeyID: providerKeyScopeID,
Provider: providerScopeID,
},
expected: 5.0,
},
{
name: "provider key wins over provider and global",
scopes: PricingLookupScopes{
SelectedKeyID: providerKeyScopeID,
Provider: providerScopeID,
},
expected: 4.0,
},
{
name: "provider wins over global",
scopes: PricingLookupScopes{
Provider: providerScopeID,
},
expected: 3.0,
},
{
name: "global applies when no narrower scope is provided",
scopes: PricingLookupScopes{},
expected: 2.0,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
patched, applied := mc.applyPricingOverrides("gpt-5-nano", schemas.ChatCompletionRequest, base, tc.scopes)
require.True(t, applied)
require.NotNil(t, patched.InputCostPerToken)
assert.Equal(t, tc.expected, *patched.InputCostPerToken)
})
}
}