2741 lines
108 KiB
Go
2741 lines
108 KiB
Go
// Package anthropic implements the Anthropic provider for the Bifrost API.
|
|
package anthropic
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/bytedance/sonic"
|
|
|
|
providerUtils "github.com/maximhq/bifrost/core/providers/utils"
|
|
schemas "github.com/maximhq/bifrost/core/schemas"
|
|
"github.com/valyala/fasthttp"
|
|
)
|
|
|
|
// AnthropicProvider implements the Provider interface for Anthropic's Claude API.
|
|
type AnthropicProvider struct {
|
|
logger schemas.Logger // Logger for provider operations
|
|
client *fasthttp.Client // HTTP client for unary API requests (ReadTimeout bounds overall response)
|
|
streamingClient *fasthttp.Client // HTTP client for streaming API requests (no ReadTimeout; idle governed by NewIdleTimeoutReader)
|
|
apiVersion string // API version for the provider
|
|
networkConfig schemas.NetworkConfig // Network configuration including extra headers
|
|
sendBackRawRequest bool // Whether to include raw request in BifrostResponse
|
|
sendBackRawResponse bool // Whether to include raw response in BifrostResponse
|
|
customProviderConfig *schemas.CustomProviderConfig // Custom provider config
|
|
}
|
|
|
|
// anthropicMessageResponsePool provides a pool for Anthropic chat response objects.
|
|
var anthropicMessageResponsePool = sync.Pool{
|
|
New: func() interface{} {
|
|
return &AnthropicMessageResponse{}
|
|
},
|
|
}
|
|
|
|
// anthropicTextResponsePool provides a pool for Anthropic text response objects.
|
|
var anthropicTextResponsePool = sync.Pool{
|
|
New: func() interface{} {
|
|
return &AnthropicTextResponse{}
|
|
},
|
|
}
|
|
|
|
// AcquireAnthropicMessageResponse gets an Anthropic chat response from the pool.
|
|
func AcquireAnthropicMessageResponse() *AnthropicMessageResponse {
|
|
resp := anthropicMessageResponsePool.Get().(*AnthropicMessageResponse)
|
|
*resp = AnthropicMessageResponse{} // Reset the struct
|
|
return resp
|
|
}
|
|
|
|
// ReleaseAnthropicMessageResponse returns an Anthropic chat response to the pool.
|
|
func ReleaseAnthropicMessageResponse(resp *AnthropicMessageResponse) {
|
|
if resp != nil {
|
|
anthropicMessageResponsePool.Put(resp)
|
|
}
|
|
}
|
|
|
|
// acquireAnthropicTextResponse gets an Anthropic text response from the pool.
|
|
func acquireAnthropicTextResponse() *AnthropicTextResponse {
|
|
resp := anthropicTextResponsePool.Get().(*AnthropicTextResponse)
|
|
*resp = AnthropicTextResponse{} // Reset the struct
|
|
return resp
|
|
}
|
|
|
|
// releaseAnthropicTextResponse returns an Anthropic text response to the pool.
|
|
func releaseAnthropicTextResponse(resp *AnthropicTextResponse) {
|
|
if resp != nil {
|
|
anthropicTextResponsePool.Put(resp)
|
|
}
|
|
}
|
|
|
|
// NewAnthropicProvider creates a new Anthropic provider instance.
|
|
// It initializes the HTTP client with the provided configuration and sets up response pools.
|
|
// The client is configured with timeouts, concurrency limits, and optional proxy settings.
|
|
func NewAnthropicProvider(config *schemas.ProviderConfig, logger schemas.Logger) *AnthropicProvider {
|
|
config.CheckAndSetDefaults()
|
|
|
|
requestTimeout := time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds)
|
|
client := &fasthttp.Client{
|
|
ReadTimeout: requestTimeout,
|
|
WriteTimeout: requestTimeout,
|
|
MaxConnsPerHost: config.NetworkConfig.MaxConnsPerHost,
|
|
MaxIdleConnDuration: 30 * time.Second,
|
|
MaxConnWaitTimeout: requestTimeout,
|
|
MaxConnDuration: time.Second * time.Duration(schemas.DefaultMaxConnDurationInSeconds),
|
|
ConnPoolStrategy: fasthttp.FIFO,
|
|
}
|
|
|
|
// Pre-warm response pools
|
|
for i := 0; i < config.ConcurrencyAndBufferSize.Concurrency; i++ {
|
|
anthropicTextResponsePool.Put(&AnthropicTextResponse{})
|
|
anthropicMessageResponsePool.Put(&AnthropicMessageResponse{})
|
|
}
|
|
|
|
// Configure proxy and retry policy
|
|
client = providerUtils.ConfigureProxy(client, config.ProxyConfig, logger)
|
|
client = providerUtils.ConfigureDialer(client)
|
|
client = providerUtils.ConfigureTLS(client, config.NetworkConfig, logger)
|
|
streamingClient := providerUtils.BuildStreamingClient(client)
|
|
// Set default BaseURL if not provided
|
|
if config.NetworkConfig.BaseURL == "" {
|
|
config.NetworkConfig.BaseURL = "https://api.anthropic.com"
|
|
}
|
|
config.NetworkConfig.BaseURL = strings.TrimRight(config.NetworkConfig.BaseURL, "/")
|
|
|
|
return &AnthropicProvider{
|
|
logger: logger,
|
|
client: client,
|
|
streamingClient: streamingClient,
|
|
apiVersion: "2023-06-01",
|
|
networkConfig: config.NetworkConfig,
|
|
sendBackRawRequest: config.SendBackRawRequest,
|
|
sendBackRawResponse: config.SendBackRawResponse,
|
|
customProviderConfig: config.CustomProviderConfig,
|
|
}
|
|
}
|
|
|
|
// GetProviderKey returns the provider identifier for Anthropic.
|
|
func (provider *AnthropicProvider) GetProviderKey() schemas.ModelProvider {
|
|
return providerUtils.GetProviderName(schemas.Anthropic, provider.customProviderConfig)
|
|
}
|
|
|
|
// buildRequestURL constructs the full request URL using the provider's configuration.
|
|
func (provider *AnthropicProvider) buildRequestURL(ctx *schemas.BifrostContext, defaultPath string, requestType schemas.RequestType) string {
|
|
path, isCompleteURL := providerUtils.GetRequestPath(ctx, defaultPath, provider.customProviderConfig, requestType)
|
|
if isCompleteURL {
|
|
return path
|
|
}
|
|
return provider.networkConfig.BaseURL + path
|
|
}
|
|
|
|
func setAnthropicRequestBody(ctx *schemas.BifrostContext, req *fasthttp.Request, body []byte) bool {
|
|
// Keep one request-body path for both modes:
|
|
// - normal mode: send converted JSON/multipart bytes
|
|
// - large payload mode: stream original client body reader
|
|
// Example failure prevented: duplicating large uploads in memory after passthrough
|
|
// was already activated at transport layer.
|
|
usedLargePayloadBody := providerUtils.ApplyLargePayloadRequestBodyWithModelNormalization(ctx, req, schemas.Anthropic)
|
|
if !usedLargePayloadBody {
|
|
req.SetBody(body)
|
|
}
|
|
return usedLargePayloadBody
|
|
}
|
|
|
|
func extractAnthropicResponsesUsageFromPrefetch(data []byte) *schemas.ResponsesResponseUsage {
|
|
node, err := sonic.Get(data, "usage")
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
raw, _ := node.Raw()
|
|
if raw == "" {
|
|
return nil
|
|
}
|
|
var usage struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
}
|
|
if err := sonic.UnmarshalString(raw, &usage); err != nil {
|
|
return nil
|
|
}
|
|
return &schemas.ResponsesResponseUsage{
|
|
InputTokens: usage.InputTokens,
|
|
OutputTokens: usage.OutputTokens,
|
|
TotalTokens: usage.InputTokens + usage.OutputTokens,
|
|
}
|
|
}
|
|
|
|
// completeRequest sends a request to Anthropic's API and handles the response.
|
|
// It constructs the API URL, sets up authentication, and processes the response.
|
|
// Returns the response body or an error if the request fails.
|
|
// When large response streaming is activated (BifrostContextKeyLargeResponseMode set in ctx),
|
|
// returns (nil, latency, nil) — callers must check the context flag.
|
|
func (provider *AnthropicProvider) completeRequest(ctx *schemas.BifrostContext, jsonData []byte, url string, key string, requestType schemas.RequestType) ([]byte, time.Duration, map[string]string, *schemas.BifrostError) {
|
|
// Create the request with the JSON body
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
defer fasthttp.ReleaseRequest(req)
|
|
respOwned := true
|
|
defer func() {
|
|
if respOwned {
|
|
fasthttp.ReleaseResponse(resp)
|
|
}
|
|
}()
|
|
|
|
// Set any extra headers from network config
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(url)
|
|
req.Header.SetMethod(http.MethodPost)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
// Can be empty in case of passthrough or keyless custom provider
|
|
// Here we can avoid this - in case of passthrough completely
|
|
if key != "" && !IsClaudeCodeMaxMode(ctx) {
|
|
req.Header.Set("x-api-key", key)
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
if betaHeaders := FilterBetaHeadersForProvider(MergeBetaHeaders(provider.networkConfig.ExtraHeaders, ctx), schemas.Anthropic, provider.networkConfig.BetaHeaderOverrides); len(betaHeaders) > 0 {
|
|
req.Header.Set(AnthropicBetaHeader, strings.Join(betaHeaders, ","))
|
|
} else {
|
|
req.Header.Del(AnthropicBetaHeader)
|
|
}
|
|
|
|
usedLargePayloadBody := setAnthropicRequestBody(ctx, req, jsonData)
|
|
|
|
requestClient := provider.client
|
|
responseThreshold, _ := ctx.Value(schemas.BifrostContextKeyLargeResponseThreshold).(int64)
|
|
isCountTokens := requestType == schemas.CountTokensRequest
|
|
// CountTokens responses are always tiny — skip streaming client so the response
|
|
// is buffered normally (same approach as OpenAI and Gemini count_tokens handlers).
|
|
if responseThreshold > 0 && !isCountTokens {
|
|
resp.StreamBody = true
|
|
requestClient = providerUtils.BuildLargeResponseClient(provider.client, responseThreshold)
|
|
}
|
|
|
|
// Send the request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, requestClient, req, resp)
|
|
defer wait()
|
|
if usedLargePayloadBody {
|
|
providerUtils.DrainLargePayloadRemainder(ctx)
|
|
}
|
|
if bifrostErr != nil {
|
|
return nil, latency, nil, bifrostErr
|
|
}
|
|
|
|
// Extract provider response headers before status check so error responses also forward them
|
|
providerResponseHeaders := providerUtils.ExtractProviderResponseHeaders(resp)
|
|
|
|
// Handle error response — materialize stream body for error parsing
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
providerUtils.MaterializeStreamErrorBody(ctx, resp)
|
|
provider.logger.Debug("error from %s provider: %s", provider.GetProviderKey(), string(resp.Body()))
|
|
return nil, latency, providerResponseHeaders, parseAnthropicError(resp)
|
|
}
|
|
|
|
// CountTokens uses buffered response (streaming skipped above) — decode directly.
|
|
if isCountTokens {
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
return nil, latency, providerResponseHeaders, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err)
|
|
}
|
|
return body, latency, providerResponseHeaders, nil
|
|
}
|
|
|
|
// Delegate large response detection + normal buffered path to shared utility
|
|
body, isLarge, respErr := providerUtils.FinalizeResponseWithLargeDetection(ctx, resp, provider.logger)
|
|
if respErr != nil {
|
|
return nil, latency, providerResponseHeaders, respErr
|
|
}
|
|
if isLarge {
|
|
respOwned = false
|
|
return nil, latency, providerResponseHeaders, nil
|
|
}
|
|
return body, latency, providerResponseHeaders, nil
|
|
}
|
|
|
|
// listModelsByKey performs a list models request for a single key.
|
|
// Returns the response and latency, or an error if the request fails.
|
|
func (provider *AnthropicProvider) listModelsByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostListModelsRequest) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
defer fasthttp.ReleaseRequest(req)
|
|
defer fasthttp.ReleaseResponse(resp)
|
|
|
|
// Set any extra headers from network config
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
|
|
// Build URL using centralized URL construction
|
|
req.SetRequestURI(provider.buildRequestURL(ctx, fmt.Sprintf("/v1/models?limit=%d", schemas.DefaultPageSize), schemas.ListModelsRequest))
|
|
req.Header.SetMethod(http.MethodGet)
|
|
req.Header.SetContentType("application/json")
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
defer wait()
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// Store provider response headers in context before status check so error responses also forward them
|
|
ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerUtils.ExtractProviderResponseHeaders(resp))
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
return nil, parseAnthropicError(resp)
|
|
}
|
|
|
|
// Parse Anthropic's response
|
|
var anthropicResponse AnthropicListModelsResponse
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(resp.Body(), &anthropicResponse, nil, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse))
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// Create final response
|
|
response := anthropicResponse.ToBifrostListModelsResponse(provider.GetProviderKey(), key.Models, key.BlacklistedModels, key.Aliases, request.Unfiltered)
|
|
response.ExtraFields.Latency = latency.Milliseconds()
|
|
|
|
// Set raw request if enabled
|
|
if providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) {
|
|
response.ExtraFields.RawRequest = rawRequest
|
|
}
|
|
|
|
// Set raw response if enabled
|
|
if providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) {
|
|
response.ExtraFields.RawResponse = rawResponse
|
|
}
|
|
|
|
return response, nil
|
|
}
|
|
|
|
// ListModels performs a list models request to Anthropic's API.
|
|
// It fetches models using all provided keys and aggregates the results.
|
|
// Uses a best-effort approach: continues with remaining keys even if some fail.
|
|
// Requests are made concurrently for improved performance.
|
|
func (provider *AnthropicProvider) ListModels(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostListModelsRequest) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.ListModelsRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
if provider.customProviderConfig != nil && provider.customProviderConfig.IsKeyLess {
|
|
return providerUtils.HandleKeylessListModelsRequest(schemas.Anthropic, func() (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
|
|
return provider.listModelsByKey(ctx, schemas.Key{}, request)
|
|
})
|
|
}
|
|
return providerUtils.HandleMultipleListModelsRequests(
|
|
ctx,
|
|
keys,
|
|
request,
|
|
provider.listModelsByKey,
|
|
)
|
|
}
|
|
|
|
// TextCompletion performs a text completion request to Anthropic's API.
|
|
// It formats the request, sends it to Anthropic, and processes the response.
|
|
// Returns a BifrostResponse containing the completion results or an error if the request fails.
|
|
func (provider *AnthropicProvider) TextCompletion(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostTextCompletionRequest) (*schemas.BifrostTextCompletionResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.TextCompletionRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Convert to Anthropic format using the centralized converter
|
|
jsonData, err := providerUtils.CheckContextAndGetRequestBody(
|
|
ctx,
|
|
request,
|
|
func() (providerUtils.RequestBodyWithExtraParams, error) {
|
|
return ToAnthropicTextCompletionRequest(request), nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Use struct directly for JSON marshaling (no beta headers for text completion)
|
|
responseBody, latency, providerResponseHeaders, err := provider.completeRequest(ctx, jsonData, provider.buildRequestURL(ctx, "/v1/complete", schemas.TextCompletionRequest), key.Value.GetValue(), schemas.TextCompletionRequest)
|
|
if providerResponseHeaders != nil {
|
|
ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerResponseHeaders)
|
|
}
|
|
if err != nil {
|
|
return nil, providerUtils.EnrichError(ctx, err, jsonData, nil, provider.sendBackRawRequest, provider.sendBackRawResponse)
|
|
}
|
|
|
|
// Large response mode: return lightweight response with metadata only
|
|
if isLargeResp, _ := ctx.Value(schemas.BifrostContextKeyLargeResponseMode).(bool); isLargeResp {
|
|
return &schemas.BifrostTextCompletionResponse{
|
|
Model: request.Model,
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
ProviderResponseHeaders: providerResponseHeaders,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
// Create response object from pool
|
|
response := acquireAnthropicTextResponse()
|
|
defer releaseAnthropicTextResponse(response)
|
|
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(responseBody, response, jsonData, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse))
|
|
if bifrostErr != nil {
|
|
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, responseBody, provider.sendBackRawRequest, provider.sendBackRawResponse)
|
|
}
|
|
|
|
bifrostResponse := response.ToBifrostTextCompletionResponse()
|
|
|
|
// Set ExtraFields
|
|
bifrostResponse.ExtraFields.Latency = latency.Milliseconds()
|
|
bifrostResponse.ExtraFields.ProviderResponseHeaders = providerResponseHeaders
|
|
|
|
// Set raw request if enabled
|
|
if providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) {
|
|
bifrostResponse.ExtraFields.RawRequest = rawRequest
|
|
}
|
|
|
|
// Set raw response if enabled
|
|
if providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) {
|
|
bifrostResponse.ExtraFields.RawResponse = rawResponse
|
|
}
|
|
|
|
return bifrostResponse, nil
|
|
}
|
|
|
|
// TextCompletionStream performs a streaming text completion request to Anthropic's API.
|
|
// It formats the request, sends it to Anthropic, and processes the response.
|
|
// Returns a channel of BifrostStreamChunk objects or an error if the request fails.
|
|
func (provider *AnthropicProvider) TextCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostTextCompletionRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.TextCompletionStreamRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ChatCompletion performs a chat completion request to Anthropic's API.
|
|
// It formats the request, sends it to Anthropic, and processes the response.
|
|
// Returns a BifrostResponse containing the completion results or an error if the request fails.
|
|
func (provider *AnthropicProvider) ChatCompletion(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostChatRequest) (*schemas.BifrostChatResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.ChatCompletionRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
// Convert to Anthropic format and get required beta headers
|
|
jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody(
|
|
ctx,
|
|
request,
|
|
func() (providerUtils.RequestBodyWithExtraParams, error) {
|
|
anthropicReq, convErr := ToAnthropicChatRequest(ctx, request)
|
|
if convErr != nil {
|
|
return nil, convErr
|
|
}
|
|
AddMissingBetaHeadersToContext(ctx, anthropicReq, schemas.Anthropic)
|
|
return anthropicReq, nil
|
|
})
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// On the raw-body passthrough path, the typed-struct StripUnsupportedAnthropicFields
|
|
// was not invoked. Apply the JSON-level sanitizer for behavioural parity so
|
|
// unsupported request-level and tool-level fields don't leak to providers that
|
|
// would reject them.
|
|
if useRawBody, ok := ctx.Value(schemas.BifrostContextKeyUseRawRequestBody).(bool); ok && useRawBody {
|
|
// Feature gating keyed to schemas.Anthropic (not provider.GetProviderKey())
|
|
// so custom Anthropic aliases get the same feature lookup as the typed
|
|
// path above (line 445), keeping raw and typed behavior in lockstep.
|
|
sanitized, rawErr := stripUnsupportedFieldsFromRawBody(jsonData, schemas.Anthropic, request.Model)
|
|
if rawErr != nil {
|
|
return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderRequestMarshal, rawErr)
|
|
}
|
|
jsonData = sanitized
|
|
// Auto-inject matching anthropic-beta headers for fields the sanitizer
|
|
// preserved. Probe-unmarshal reuses the typed path's header walker so
|
|
// the two paths stay in lockstep.
|
|
var probe AnthropicMessageRequest
|
|
if err := schemas.Unmarshal(jsonData, &probe); err == nil {
|
|
AddMissingBetaHeadersToContext(ctx, &probe, schemas.Anthropic)
|
|
}
|
|
}
|
|
|
|
// Use struct directly for JSON marshaling
|
|
responseBody, latency, providerResponseHeaders, err := provider.completeRequest(ctx, jsonData, provider.buildRequestURL(ctx, "/v1/messages", schemas.ChatCompletionRequest), key.Value.GetValue(), schemas.ChatCompletionRequest)
|
|
if providerResponseHeaders != nil {
|
|
ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerResponseHeaders)
|
|
}
|
|
if err != nil {
|
|
return nil, providerUtils.EnrichError(ctx, err, jsonData, nil, provider.sendBackRawRequest, provider.sendBackRawResponse)
|
|
}
|
|
|
|
// Large response mode: return lightweight response with metadata only
|
|
if isLargeResp, _ := ctx.Value(schemas.BifrostContextKeyLargeResponseMode).(bool); isLargeResp {
|
|
return &schemas.BifrostChatResponse{
|
|
Model: request.Model,
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
ProviderResponseHeaders: providerResponseHeaders,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
// Create response object from pool
|
|
response := AcquireAnthropicMessageResponse()
|
|
defer ReleaseAnthropicMessageResponse(response)
|
|
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(responseBody, response, jsonData, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse))
|
|
if bifrostErr != nil {
|
|
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, responseBody, provider.sendBackRawRequest, provider.sendBackRawResponse)
|
|
}
|
|
// Create final response
|
|
bifrostResponse := response.ToBifrostChatResponse(ctx)
|
|
|
|
// Set ExtraFields
|
|
bifrostResponse.ExtraFields.Latency = latency.Milliseconds()
|
|
bifrostResponse.ExtraFields.ProviderResponseHeaders = providerResponseHeaders
|
|
|
|
// Set raw request if enabled
|
|
if providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) {
|
|
bifrostResponse.ExtraFields.RawRequest = rawRequest
|
|
}
|
|
|
|
// Set raw response if enabled
|
|
if providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) {
|
|
bifrostResponse.ExtraFields.RawResponse = rawResponse
|
|
}
|
|
|
|
return bifrostResponse, nil
|
|
}
|
|
|
|
// ChatCompletionStream performs a streaming chat completion request to the Anthropic API.
|
|
// It supports real-time streaming of responses using Server-Sent Events (SSE).
|
|
// Returns a channel containing BifrostStreamChunk objects representing the stream or an error if the request fails.
|
|
func (provider *AnthropicProvider) ChatCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostChatRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.ChatCompletionStreamRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Convert to Anthropic format and get required beta headers
|
|
jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody(
|
|
ctx,
|
|
request,
|
|
func() (providerUtils.RequestBodyWithExtraParams, error) {
|
|
anthropicReq, convErr := ToAnthropicChatRequest(ctx, request)
|
|
if convErr != nil {
|
|
return nil, convErr
|
|
}
|
|
anthropicReq.Stream = schemas.Ptr(true)
|
|
AddMissingBetaHeadersToContext(ctx, anthropicReq, schemas.Anthropic)
|
|
return anthropicReq, nil
|
|
})
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// On the raw-body passthrough path, the typed-struct StripUnsupportedAnthropicFields
|
|
// was not invoked. Apply the JSON-level sanitizer for behavioural parity.
|
|
if useRawBody, ok := ctx.Value(schemas.BifrostContextKeyUseRawRequestBody).(bool); ok && useRawBody {
|
|
// Feature gating keyed to schemas.Anthropic (not provider.GetProviderKey())
|
|
// to keep raw and typed paths in lockstep on custom aliases — mirrors
|
|
// the typed path's hardcoded schemas.Anthropic at line 548.
|
|
sanitized, rawErr := stripUnsupportedFieldsFromRawBody(jsonData, schemas.Anthropic, request.Model)
|
|
if rawErr != nil {
|
|
return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderRequestMarshal, rawErr)
|
|
}
|
|
jsonData = sanitized
|
|
// Auto-inject matching anthropic-beta headers for fields the sanitizer
|
|
// preserved. Probe-unmarshal reuses the typed path's header walker.
|
|
var probe AnthropicMessageRequest
|
|
if err := schemas.Unmarshal(jsonData, &probe); err == nil {
|
|
AddMissingBetaHeadersToContext(ctx, &probe, schemas.Anthropic)
|
|
}
|
|
}
|
|
|
|
// Prepare Anthropic headers
|
|
headers := map[string]string{
|
|
"Content-Type": "application/json",
|
|
"anthropic-version": provider.apiVersion,
|
|
"Accept": "text/event-stream",
|
|
"Cache-Control": "no-cache",
|
|
}
|
|
|
|
if key.Value.GetValue() != "" && !IsClaudeCodeMaxMode(ctx) {
|
|
headers["x-api-key"] = key.Value.GetValue()
|
|
}
|
|
|
|
providerUtils.SetStreamIdleTimeoutIfEmpty(ctx, provider.networkConfig.StreamIdleTimeoutInSeconds)
|
|
|
|
// Use shared Anthropic streaming logic
|
|
return HandleAnthropicChatCompletionStreaming(
|
|
ctx,
|
|
provider.streamingClient,
|
|
provider.buildRequestURL(ctx, "/v1/messages", schemas.ChatCompletionStreamRequest),
|
|
jsonData,
|
|
headers,
|
|
provider.networkConfig.ExtraHeaders,
|
|
provider.networkConfig.BetaHeaderOverrides,
|
|
providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest),
|
|
providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse),
|
|
provider.GetProviderKey(),
|
|
postHookRunner,
|
|
nil,
|
|
provider.logger,
|
|
postHookSpanFinalizer,
|
|
)
|
|
}
|
|
|
|
// HandleAnthropicChatCompletionStreaming handles streaming for Anthropic-compatible APIs.
|
|
// This shared function reduces code duplication between providers that use the same SSE event format.
|
|
func HandleAnthropicChatCompletionStreaming(
|
|
ctx *schemas.BifrostContext,
|
|
client *fasthttp.Client,
|
|
url string,
|
|
jsonBody []byte,
|
|
headers map[string]string,
|
|
extraHeaders map[string]string,
|
|
betaHeaderOverrides map[string]bool,
|
|
sendBackRawRequest bool,
|
|
sendBackRawResponse bool,
|
|
providerName schemas.ModelProvider,
|
|
postHookRunner schemas.PostHookRunner,
|
|
postResponseConverter func(*schemas.BifrostChatResponse) *schemas.BifrostChatResponse,
|
|
logger schemas.Logger,
|
|
postHookSpanFinalizer func(context.Context),
|
|
) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
resp.StreamBody = true // Initialize for streaming
|
|
defer fasthttp.ReleaseRequest(req)
|
|
|
|
req.Header.SetMethod(http.MethodPost)
|
|
req.SetRequestURI(url)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
providerUtils.SetExtraHeaders(ctx, req, extraHeaders, []string{AnthropicBetaHeader})
|
|
|
|
if betaHeaders := FilterBetaHeadersForProvider(MergeBetaHeaders(extraHeaders, ctx), providerName, betaHeaderOverrides); len(betaHeaders) > 0 {
|
|
req.Header.Set(AnthropicBetaHeader, strings.Join(betaHeaders, ","))
|
|
} else {
|
|
req.Header.Del(AnthropicBetaHeader)
|
|
}
|
|
|
|
for key, value := range headers {
|
|
req.Header.Set(key, value)
|
|
}
|
|
|
|
usedLargePayloadBody := setAnthropicRequestBody(ctx, req, jsonBody)
|
|
|
|
// Use streaming-aware client when large payload optimization is active — ensures
|
|
// MaxResponseBodySize > 0 so ErrBodyTooLarge triggers StreamBody for Content-Length responses.
|
|
activeClient := providerUtils.PrepareResponseStreaming(ctx, client, resp)
|
|
|
|
// Make the request
|
|
err := activeClient.Do(req, resp)
|
|
if usedLargePayloadBody {
|
|
providerUtils.DrainLargePayloadRemainder(ctx)
|
|
}
|
|
if err != nil {
|
|
defer providerUtils.ReleaseStreamingResponse(resp)
|
|
if errors.Is(err, context.Canceled) {
|
|
return nil, providerUtils.EnrichError(ctx, &schemas.BifrostError{
|
|
IsBifrostError: false,
|
|
Error: &schemas.ErrorField{
|
|
Type: schemas.Ptr(schemas.RequestCancelled),
|
|
Message: schemas.ErrRequestCancelled,
|
|
Error: err,
|
|
},
|
|
}, jsonBody, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
if errors.Is(err, fasthttp.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) {
|
|
return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostTimeoutError(schemas.ErrProviderRequestTimedOut, err), jsonBody, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError(schemas.ErrProviderDoRequest, err), jsonBody, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
|
|
// Store provider response headers in context before status check so error responses also forward them
|
|
ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerUtils.ExtractProviderResponseHeaders(resp))
|
|
|
|
// Check for HTTP errors
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
defer providerUtils.ReleaseStreamingResponse(resp)
|
|
return nil, providerUtils.EnrichError(ctx, parseAnthropicError(resp), jsonBody, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
|
|
// Large payload streaming passthrough — pipe raw upstream SSE to client
|
|
if providerUtils.SetupStreamingPassthrough(ctx, resp) {
|
|
responseChan := make(chan *schemas.BifrostStreamChunk)
|
|
close(responseChan)
|
|
return responseChan, nil
|
|
}
|
|
|
|
// Create response channel
|
|
responseChan := make(chan *schemas.BifrostStreamChunk, schemas.DefaultStreamBufferSize)
|
|
|
|
// Start streaming in a goroutine
|
|
go func() {
|
|
defer providerUtils.EnsureStreamFinalizerCalled(ctx, postHookSpanFinalizer)
|
|
defer func() {
|
|
if ctx.Err() == context.Canceled {
|
|
providerUtils.HandleStreamCancellation(ctx, postHookRunner, responseChan, logger, postHookSpanFinalizer)
|
|
} else if ctx.Err() == context.DeadlineExceeded {
|
|
providerUtils.HandleStreamTimeout(ctx, postHookRunner, responseChan, logger, postHookSpanFinalizer)
|
|
}
|
|
close(responseChan)
|
|
}()
|
|
defer providerUtils.ReleaseStreamingResponse(resp)
|
|
|
|
if resp.BodyStream() == nil {
|
|
bifrostErr := providerUtils.NewBifrostOperationError(
|
|
"Provider returned an empty response",
|
|
fmt.Errorf("provider returned an empty response"),
|
|
)
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
providerUtils.ProcessAndSendBifrostError(ctx, postHookRunner, providerUtils.EnrichError(ctx, bifrostErr, jsonBody, nil, sendBackRawRequest, sendBackRawResponse), responseChan, logger, postHookSpanFinalizer)
|
|
return
|
|
}
|
|
|
|
// Decompress gzip-encoded streams transparently (no-op for non-gzip)
|
|
reader, releaseGzip := providerUtils.DecompressStreamBody(resp)
|
|
defer releaseGzip()
|
|
|
|
// Wrap reader with idle timeout to detect stalled streams.
|
|
reader, stopIdleTimeout := providerUtils.NewIdleTimeoutReader(reader, resp.BodyStream(), providerUtils.GetStreamIdleTimeout(ctx))
|
|
defer stopIdleTimeout()
|
|
|
|
// Setup cancellation handler to close the raw network stream on ctx cancellation,
|
|
// which immediately unblocks any in-progress read (including reads blocked inside a gzip decompression layer).
|
|
stopCancellation := providerUtils.SetupStreamCancellation(ctx, resp.BodyStream(), logger)
|
|
defer stopCancellation()
|
|
|
|
sseReader := providerUtils.GetSSEEventReader(ctx, reader)
|
|
|
|
chunkIndex := 0
|
|
|
|
startTime := time.Now()
|
|
lastChunkTime := startTime
|
|
|
|
// Track minimal state needed for response format
|
|
var messageID string
|
|
var modelName string
|
|
var finishReason *string
|
|
|
|
usage := &schemas.BifrostLLMUsage{}
|
|
|
|
// Check for structured output tool name and track state
|
|
var structuredOutputToolName string
|
|
var isAccumulatingStructuredOutput bool
|
|
if toolName, ok := ctx.Value(schemas.BifrostContextKeyStructuredOutputToolName).(string); ok {
|
|
structuredOutputToolName = toolName
|
|
}
|
|
|
|
// Per-response tool-call index state
|
|
streamState := NewAnthropicStreamState()
|
|
|
|
for {
|
|
// If context was cancelled/timed out, let defer handle it
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
eventType, eventDataBytes, readErr := sseReader.ReadEvent()
|
|
if readErr != nil {
|
|
if readErr != io.EOF {
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
logger.Warn("Error reading %s stream: %v", providerName, readErr)
|
|
providerUtils.ProcessAndSendError(ctx, postHookRunner, readErr, responseChan, logger, postHookSpanFinalizer)
|
|
return
|
|
}
|
|
break
|
|
}
|
|
eventData := string(eventDataBytes)
|
|
if eventType == "" || eventData == "" {
|
|
continue
|
|
}
|
|
var event AnthropicStreamEvent
|
|
if err := sonic.Unmarshal([]byte(eventData), &event); err != nil {
|
|
logger.Warn("Failed to parse message_start event: %v", err)
|
|
continue
|
|
}
|
|
if event.Type == AnthropicStreamEventTypeMessageStart && event.Message != nil && event.Message.ID != "" {
|
|
messageID = event.Message.ID
|
|
}
|
|
// Check for usage in both top-level event.Usage and nested event.Message.Usage
|
|
// message_start events have usage nested in message.usage, while message_delta has it at top level
|
|
var usageToProcess *AnthropicUsage
|
|
if event.Usage != nil {
|
|
usageToProcess = event.Usage
|
|
} else if event.Message != nil && event.Message.Usage != nil {
|
|
usageToProcess = event.Message.Usage
|
|
}
|
|
if usageToProcess != nil {
|
|
// Collect usage information and send at the end of the stream
|
|
// Here in some cases usage comes before final message
|
|
// So we need to check if the response.Usage is nil and then if usage != nil
|
|
// then add up all tokens
|
|
if usageToProcess.InputTokens > usage.PromptTokens {
|
|
usage.PromptTokens = usageToProcess.InputTokens
|
|
}
|
|
if usageToProcess.OutputTokens > usage.CompletionTokens {
|
|
usage.CompletionTokens = usageToProcess.OutputTokens
|
|
}
|
|
calculatedTotal := usage.PromptTokens + usage.CompletionTokens
|
|
if calculatedTotal > usage.TotalTokens {
|
|
usage.TotalTokens = calculatedTotal
|
|
}
|
|
// Handle cached tokens if present
|
|
if usageToProcess.CacheReadInputTokens > 0 {
|
|
if usage.PromptTokensDetails == nil {
|
|
usage.PromptTokensDetails = &schemas.ChatPromptTokensDetails{}
|
|
}
|
|
if usageToProcess.CacheReadInputTokens > usage.PromptTokensDetails.CachedReadTokens {
|
|
usage.PromptTokensDetails.CachedReadTokens = usageToProcess.CacheReadInputTokens
|
|
}
|
|
}
|
|
if usageToProcess.CacheCreationInputTokens > 0 {
|
|
if usage.PromptTokensDetails == nil {
|
|
usage.PromptTokensDetails = &schemas.ChatPromptTokensDetails{}
|
|
}
|
|
if usageToProcess.CacheCreationInputTokens > usage.PromptTokensDetails.CachedWriteTokens {
|
|
usage.PromptTokensDetails.CachedWriteTokens = usageToProcess.CacheCreationInputTokens
|
|
}
|
|
}
|
|
}
|
|
if event.Message != nil {
|
|
modelName = event.Message.Model
|
|
}
|
|
|
|
// Extract finish reason from event delta
|
|
if event.Delta != nil && event.Delta.StopReason != nil {
|
|
mappedReason := ConvertAnthropicFinishReasonToBifrost(*event.Delta.StopReason)
|
|
finishReason = &mappedReason
|
|
|
|
// Override finish reason for structured output
|
|
// When structured output is used, tool_use stop reason should appear as "stop" to the client
|
|
if structuredOutputToolName != "" && *finishReason == string(schemas.BifrostFinishReasonToolCalls) {
|
|
stopReason := string(schemas.BifrostFinishReasonStop)
|
|
finishReason = &stopReason
|
|
}
|
|
}
|
|
|
|
// Handle structured output: intercept tool calls for the structured output tool
|
|
// and convert them to content instead of forwarding as tool calls
|
|
if structuredOutputToolName != "" {
|
|
// Check for tool use start event
|
|
if event.Type == AnthropicStreamEventTypeContentBlockStart {
|
|
if event.ContentBlock != nil && event.ContentBlock.Type == AnthropicContentBlockTypeToolUse {
|
|
if event.ContentBlock.Name != nil && *event.ContentBlock.Name == structuredOutputToolName {
|
|
isAccumulatingStructuredOutput = true
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check for tool use delta event
|
|
if event.Type == AnthropicStreamEventTypeContentBlockDelta && isAccumulatingStructuredOutput {
|
|
if event.Delta != nil && event.Delta.Type == AnthropicStreamDeltaTypeInputJSON && event.Delta.PartialJSON != nil {
|
|
// Convert tool use delta to content delta
|
|
content := *event.Delta.PartialJSON
|
|
response := &schemas.BifrostChatResponse{
|
|
ID: messageID,
|
|
Object: "chat.completion.chunk",
|
|
Choices: []schemas.BifrostResponseChoice{
|
|
{
|
|
Index: 0,
|
|
ChatStreamResponseChoice: &schemas.ChatStreamResponseChoice{
|
|
Delta: &schemas.ChatStreamResponseChoiceDelta{
|
|
Content: &content,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
ChunkIndex: chunkIndex,
|
|
Latency: time.Since(lastChunkTime).Milliseconds(),
|
|
},
|
|
}
|
|
lastChunkTime = time.Now()
|
|
chunkIndex++
|
|
|
|
if sendBackRawResponse {
|
|
response.ExtraFields.RawResponse = eventData
|
|
}
|
|
|
|
providerUtils.ProcessAndSendResponse(ctx, postHookRunner, providerUtils.GetBifrostResponseForStreamResponse(nil, response, nil, nil, nil, nil), responseChan, postHookSpanFinalizer)
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Check for content block stop
|
|
if event.Type == AnthropicStreamEventTypeContentBlockStop && isAccumulatingStructuredOutput {
|
|
isAccumulatingStructuredOutput = false
|
|
continue
|
|
}
|
|
}
|
|
|
|
response, bifrostErr, isLastChunk := event.ToBifrostChatCompletionStream(ctx, structuredOutputToolName, streamState)
|
|
if bifrostErr != nil {
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
providerUtils.ProcessAndSendBifrostError(ctx, postHookRunner, bifrostErr, responseChan, logger, postHookSpanFinalizer)
|
|
break
|
|
}
|
|
if response != nil {
|
|
response.ExtraFields = schemas.BifrostResponseExtraFields{
|
|
ChunkIndex: chunkIndex,
|
|
Latency: time.Since(lastChunkTime).Milliseconds(),
|
|
}
|
|
if postResponseConverter != nil {
|
|
response = postResponseConverter(response)
|
|
if response == nil {
|
|
logger.Warn("postResponseConverter returned nil; skipping chunk")
|
|
continue
|
|
}
|
|
}
|
|
response.ID = messageID
|
|
lastChunkTime = time.Now()
|
|
chunkIndex++
|
|
|
|
if sendBackRawResponse {
|
|
response.ExtraFields.RawResponse = eventData
|
|
}
|
|
|
|
providerUtils.ProcessAndSendResponse(ctx, postHookRunner, providerUtils.GetBifrostResponseForStreamResponse(nil, response, nil, nil, nil, nil), responseChan, postHookSpanFinalizer)
|
|
}
|
|
if isLastChunk {
|
|
break
|
|
}
|
|
}
|
|
if usage.PromptTokensDetails != nil {
|
|
usage.PromptTokens = usage.PromptTokens + usage.PromptTokensDetails.CachedReadTokens + usage.PromptTokensDetails.CachedWriteTokens
|
|
usage.TotalTokens = usage.TotalTokens + usage.PromptTokensDetails.CachedReadTokens + usage.PromptTokensDetails.CachedWriteTokens
|
|
}
|
|
response := providerUtils.CreateBifrostChatCompletionChunkResponse(messageID, usage, finishReason, chunkIndex, modelName, 0)
|
|
if postResponseConverter != nil {
|
|
response = postResponseConverter(response)
|
|
if response == nil {
|
|
logger.Warn("postResponseConverter returned nil; skipping chunk")
|
|
// Setting error on the context to signal to the defer that we need to close the stream
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
return
|
|
}
|
|
}
|
|
// Set raw request if enabled
|
|
if sendBackRawRequest {
|
|
providerUtils.ParseAndSetRawRequest(&response.ExtraFields, jsonBody)
|
|
}
|
|
response.ExtraFields.Latency = time.Since(startTime).Milliseconds()
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
providerUtils.ProcessAndSendResponse(ctx, postHookRunner, providerUtils.GetBifrostResponseForStreamResponse(nil, response, nil, nil, nil, nil), responseChan, postHookSpanFinalizer)
|
|
}()
|
|
|
|
return responseChan, nil
|
|
}
|
|
|
|
// Responses performs a chat completion request to Anthropic's API.
|
|
// It formats the request, sends it to Anthropic, and processes the response.
|
|
// Returns a BifrostResponse containing the completion results or an error if the request fails.
|
|
func (provider *AnthropicProvider) Responses(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostResponsesRequest) (*schemas.BifrostResponsesResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.ResponsesRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
jsonBody, err := getRequestBodyForResponses(ctx, request, false, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
responseBody, latency, providerResponseHeaders, err := provider.completeRequest(ctx, jsonBody, provider.buildRequestURL(ctx, "/v1/messages", schemas.ResponsesRequest), key.Value.GetValue(), schemas.ResponsesRequest)
|
|
if providerResponseHeaders != nil {
|
|
ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerResponseHeaders)
|
|
}
|
|
if err != nil {
|
|
return nil, providerUtils.EnrichError(ctx, err, jsonBody, nil, provider.sendBackRawRequest, provider.sendBackRawResponse)
|
|
}
|
|
|
|
// Large response mode: return lightweight response with usage from preview for plugin pipeline.
|
|
if isLargeResp, _ := ctx.Value(schemas.BifrostContextKeyLargeResponseMode).(bool); isLargeResp {
|
|
preview, _ := ctx.Value(schemas.BifrostContextKeyLargePayloadResponsePreview).(string)
|
|
return &schemas.BifrostResponsesResponse{
|
|
ID: schemas.Ptr("resp_" + providerUtils.GetRandomString(50)),
|
|
Object: "response",
|
|
CreatedAt: int(time.Now().Unix()),
|
|
Model: request.Model,
|
|
Usage: extractAnthropicResponsesUsageFromPrefetch([]byte(preview)),
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
ProviderResponseHeaders: providerResponseHeaders,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
// Create response object from pool
|
|
response := AcquireAnthropicMessageResponse()
|
|
defer ReleaseAnthropicMessageResponse(response)
|
|
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(responseBody, response, jsonBody, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse))
|
|
if bifrostErr != nil {
|
|
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonBody, responseBody, provider.sendBackRawRequest, provider.sendBackRawResponse)
|
|
}
|
|
|
|
// Create final response
|
|
bifrostResponse := response.ToBifrostResponsesResponse(ctx)
|
|
|
|
// Set ExtraFields
|
|
bifrostResponse.ExtraFields.Latency = latency.Milliseconds()
|
|
bifrostResponse.ExtraFields.ProviderResponseHeaders = providerResponseHeaders
|
|
|
|
// Set raw request if enabled
|
|
if providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) {
|
|
bifrostResponse.ExtraFields.RawRequest = rawRequest
|
|
}
|
|
|
|
// Set raw response if enabled
|
|
if providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) {
|
|
bifrostResponse.ExtraFields.RawResponse = rawResponse
|
|
}
|
|
|
|
return bifrostResponse, nil
|
|
}
|
|
|
|
// ResponsesStream performs a streaming responses request to the Anthropic API.
|
|
func (provider *AnthropicProvider) ResponsesStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostResponsesRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.ResponsesStreamRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Convert to Anthropic format using the centralized converter
|
|
jsonBody, err := getRequestBodyForResponses(ctx, request, true, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Prepare Anthropic headers
|
|
headers := map[string]string{
|
|
"Content-Type": "application/json",
|
|
"anthropic-version": provider.apiVersion,
|
|
"Accept": "text/event-stream",
|
|
"Cache-Control": "no-cache",
|
|
}
|
|
|
|
if key.Value.GetValue() != "" && !IsClaudeCodeMaxMode(ctx) {
|
|
headers["x-api-key"] = key.Value.GetValue()
|
|
}
|
|
|
|
providerUtils.SetStreamIdleTimeoutIfEmpty(ctx, provider.networkConfig.StreamIdleTimeoutInSeconds)
|
|
|
|
return HandleAnthropicResponsesStream(
|
|
ctx,
|
|
provider.streamingClient,
|
|
provider.buildRequestURL(ctx, "/v1/messages", schemas.ResponsesStreamRequest),
|
|
jsonBody,
|
|
headers,
|
|
provider.networkConfig.ExtraHeaders,
|
|
provider.networkConfig.BetaHeaderOverrides,
|
|
providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest),
|
|
providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse),
|
|
provider.GetProviderKey(),
|
|
postHookRunner,
|
|
nil,
|
|
provider.logger,
|
|
postHookSpanFinalizer,
|
|
)
|
|
}
|
|
|
|
// HandleAnthropicResponsesStream handles streaming for Anthropic-compatible APIs.
|
|
// This shared function reduces code duplication between providers that use the same SSE event format.
|
|
func HandleAnthropicResponsesStream(
|
|
ctx *schemas.BifrostContext,
|
|
client *fasthttp.Client,
|
|
url string,
|
|
jsonBody []byte,
|
|
headers map[string]string,
|
|
extraHeaders map[string]string,
|
|
betaHeaderOverrides map[string]bool,
|
|
sendBackRawRequest bool,
|
|
sendBackRawResponse bool,
|
|
providerName schemas.ModelProvider,
|
|
postHookRunner schemas.PostHookRunner,
|
|
postResponseConverter func(*schemas.BifrostResponsesStreamResponse) *schemas.BifrostResponsesStreamResponse,
|
|
logger schemas.Logger,
|
|
postHookSpanFinalizer func(context.Context),
|
|
) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
resp.StreamBody = true
|
|
defer fasthttp.ReleaseRequest(req)
|
|
|
|
req.Header.SetMethod(http.MethodPost)
|
|
req.SetRequestURI(url)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
providerUtils.SetExtraHeaders(ctx, req, extraHeaders, []string{AnthropicBetaHeader})
|
|
|
|
if betaHeaders := FilterBetaHeadersForProvider(MergeBetaHeaders(extraHeaders, ctx), providerName, betaHeaderOverrides); len(betaHeaders) > 0 {
|
|
req.Header.Set(AnthropicBetaHeader, strings.Join(betaHeaders, ","))
|
|
} else {
|
|
req.Header.Del(AnthropicBetaHeader)
|
|
}
|
|
|
|
// Set auth/static headers, applied after extra headers so they always win
|
|
for key, value := range headers {
|
|
req.Header.Set(key, value)
|
|
}
|
|
|
|
// Set body
|
|
usedLargePayloadBody := setAnthropicRequestBody(ctx, req, jsonBody)
|
|
|
|
// Use streaming-aware client when large payload optimization is active — ensures
|
|
// MaxResponseBodySize > 0 so ErrBodyTooLarge triggers StreamBody for Content-Length responses.
|
|
activeClient := providerUtils.PrepareResponseStreaming(ctx, client, resp)
|
|
|
|
// Make the request
|
|
err := activeClient.Do(req, resp)
|
|
if usedLargePayloadBody {
|
|
providerUtils.DrainLargePayloadRemainder(ctx)
|
|
}
|
|
if err != nil {
|
|
defer providerUtils.ReleaseStreamingResponse(resp)
|
|
if errors.Is(err, context.Canceled) {
|
|
return nil, providerUtils.EnrichError(ctx, &schemas.BifrostError{
|
|
IsBifrostError: false,
|
|
Error: &schemas.ErrorField{
|
|
Type: schemas.Ptr(schemas.RequestCancelled),
|
|
Message: schemas.ErrRequestCancelled,
|
|
Error: err,
|
|
},
|
|
}, jsonBody, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
if errors.Is(err, fasthttp.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) {
|
|
return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostTimeoutError(schemas.ErrProviderRequestTimedOut, err), jsonBody, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError(schemas.ErrProviderDoRequest, err), jsonBody, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
|
|
// Store provider response headers in context before status check so error responses also forward them
|
|
ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerUtils.ExtractProviderResponseHeaders(resp))
|
|
|
|
// Check for HTTP errors
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
defer providerUtils.ReleaseStreamingResponse(resp)
|
|
return nil, providerUtils.EnrichError(ctx, parseAnthropicError(resp), jsonBody, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
|
|
// Large payload streaming passthrough — pipe raw upstream SSE to client
|
|
if providerUtils.SetupStreamingPassthrough(ctx, resp) {
|
|
responseChan := make(chan *schemas.BifrostStreamChunk)
|
|
close(responseChan)
|
|
return responseChan, nil
|
|
}
|
|
|
|
// Create response channel
|
|
responseChan := make(chan *schemas.BifrostStreamChunk, schemas.DefaultStreamBufferSize)
|
|
|
|
// Start streaming in a goroutine
|
|
go func() {
|
|
defer providerUtils.EnsureStreamFinalizerCalled(ctx, postHookSpanFinalizer)
|
|
defer func() {
|
|
if ctx.Err() == context.Canceled {
|
|
providerUtils.HandleStreamCancellation(ctx, postHookRunner, responseChan, logger, postHookSpanFinalizer)
|
|
} else if ctx.Err() == context.DeadlineExceeded {
|
|
providerUtils.HandleStreamTimeout(ctx, postHookRunner, responseChan, logger, postHookSpanFinalizer)
|
|
}
|
|
close(responseChan)
|
|
}()
|
|
defer providerUtils.ReleaseStreamingResponse(resp)
|
|
// If body stream is nil, return an error
|
|
if resp.BodyStream() == nil {
|
|
bifrostErr := providerUtils.NewBifrostOperationError(
|
|
"Provider returned an empty response",
|
|
fmt.Errorf("provider returned an empty response"),
|
|
)
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
providerUtils.ProcessAndSendBifrostError(ctx, postHookRunner, providerUtils.EnrichError(ctx, bifrostErr, jsonBody, nil, sendBackRawRequest, sendBackRawResponse), responseChan, logger, postHookSpanFinalizer)
|
|
return
|
|
}
|
|
|
|
// Decompress gzip-encoded streams on-the-fly. Returns a reader that is either
|
|
// the gzip reader (if gzip-encoded) or the original body stream. Does NOT modify
|
|
// resp, so ReleaseStreamingResponse can properly drain the underlying connection.
|
|
reader, releaseGzip := providerUtils.DecompressStreamBody(resp)
|
|
defer releaseGzip()
|
|
|
|
// Wrap reader with idle timeout to detect stalled streams.
|
|
reader, stopIdleTimeout := providerUtils.NewIdleTimeoutReader(reader, resp.BodyStream(), providerUtils.GetStreamIdleTimeout(ctx))
|
|
defer stopIdleTimeout()
|
|
|
|
// Setup cancellation handler to close the raw network stream on ctx cancellation,
|
|
// which immediately unblocks any in-progress read (including reads blocked inside a gzip decompression layer).
|
|
stopCancellation := providerUtils.SetupStreamCancellation(ctx, resp.BodyStream(), logger)
|
|
defer stopCancellation()
|
|
|
|
sseReader := providerUtils.GetSSEEventReader(ctx, reader)
|
|
chunkIndex := 0
|
|
|
|
startTime := time.Now()
|
|
lastChunkTime := startTime
|
|
|
|
// Track minimal state needed for response format
|
|
usage := &schemas.ResponsesResponseUsage{}
|
|
|
|
// Create stream state for stateful conversions
|
|
streamState := acquireAnthropicResponsesStreamState()
|
|
defer releaseAnthropicResponsesStreamState(streamState)
|
|
|
|
// Set structured output tool name if present
|
|
if toolName, ok := ctx.Value(schemas.BifrostContextKeyStructuredOutputToolName).(string); ok {
|
|
streamState.StructuredOutputToolName = toolName
|
|
}
|
|
|
|
var modelName string
|
|
|
|
for {
|
|
// If context was cancelled/timed out, let defer handle it
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
eventType, eventDataBytes, readErr := sseReader.ReadEvent()
|
|
if readErr != nil {
|
|
if readErr != io.EOF {
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
logger.Warn("Error reading %s stream: %v", providerName, readErr)
|
|
providerUtils.ProcessAndSendError(ctx, postHookRunner, readErr, responseChan, logger, postHookSpanFinalizer)
|
|
}
|
|
break
|
|
}
|
|
eventData := string(eventDataBytes)
|
|
if eventType == "" || eventData == "" {
|
|
continue
|
|
}
|
|
var event AnthropicStreamEvent
|
|
if err := sonic.Unmarshal([]byte(eventData), &event); err != nil {
|
|
logger.Warn("Failed to parse message_start event: %v", err)
|
|
continue
|
|
}
|
|
if event.Message != nil && modelName == "" {
|
|
modelName = event.Message.Model
|
|
}
|
|
// Note: response.created and response.in_progress are now emitted by ToBifrostResponsesStream
|
|
// from the message_start event, so we don't need to call them manually here
|
|
|
|
// Check for usage in both top-level event.Usage and nested event.Message.Usage
|
|
// message_start events have usage nested in message.usage, while message_delta has it at top level
|
|
var usageToProcess *AnthropicUsage
|
|
if event.Usage != nil {
|
|
usageToProcess = event.Usage
|
|
} else if event.Message != nil && event.Message.Usage != nil {
|
|
usageToProcess = event.Message.Usage
|
|
}
|
|
|
|
if usageToProcess != nil {
|
|
// Collect usage information and send at the end of the stream
|
|
// Here in some cases usage comes before final message
|
|
// So we need to check if the response.Usage is nil and then if usage != nil
|
|
// then add up all tokens
|
|
if usageToProcess.InputTokens > usage.InputTokens {
|
|
usage.InputTokens = usageToProcess.InputTokens
|
|
}
|
|
if usageToProcess.OutputTokens > usage.OutputTokens {
|
|
usage.OutputTokens = usageToProcess.OutputTokens
|
|
}
|
|
calculatedTotal := usage.InputTokens + usage.OutputTokens
|
|
if calculatedTotal > usage.TotalTokens {
|
|
usage.TotalTokens = calculatedTotal
|
|
}
|
|
// Handle cached tokens if present
|
|
if usageToProcess.CacheReadInputTokens > 0 {
|
|
if usage.InputTokensDetails == nil {
|
|
usage.InputTokensDetails = &schemas.ResponsesResponseInputTokens{}
|
|
}
|
|
if usageToProcess.CacheReadInputTokens > usage.InputTokensDetails.CachedReadTokens {
|
|
usage.InputTokensDetails.CachedReadTokens = usageToProcess.CacheReadInputTokens
|
|
}
|
|
}
|
|
// Handle cached tokens if present
|
|
if usageToProcess.CacheCreationInputTokens > 0 {
|
|
if usage.InputTokensDetails == nil {
|
|
usage.InputTokensDetails = &schemas.ResponsesResponseInputTokens{}
|
|
}
|
|
if usageToProcess.CacheCreationInputTokens > usage.InputTokensDetails.CachedWriteTokens {
|
|
usage.InputTokensDetails.CachedWriteTokens = usageToProcess.CacheCreationInputTokens
|
|
}
|
|
}
|
|
}
|
|
|
|
responses, bifrostErr, isLastChunk := event.ToBifrostResponsesStream(ctx, chunkIndex, streamState)
|
|
// Propagate message_delta emission flag to context so the output converter
|
|
// (ToAnthropicResponsesStreamResponse) can skip synthesizing a duplicate.
|
|
if streamState.HasEmittedMessageDelta {
|
|
ctx.SetValue(schemas.BifrostContextKeyHasEmittedMessageDelta, true)
|
|
}
|
|
if bifrostErr != nil {
|
|
// If context was cancelled/timed out, let defer handle it
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
providerUtils.ProcessAndSendBifrostError(ctx, postHookRunner, bifrostErr, responseChan, logger, postHookSpanFinalizer)
|
|
break
|
|
}
|
|
// Passthrough: when conversion returns no responses but we need to forward raw events,
|
|
// create a minimal pass-through response to carry the raw event data.
|
|
// This ensures events like compaction content_block_start/stop are not silently dropped.
|
|
if len(responses) == 0 && sendBackRawResponse {
|
|
passthroughResp := &schemas.BifrostResponsesStreamResponse{
|
|
Type: schemas.ResponsesStreamResponseType(eventType),
|
|
SequenceNumber: chunkIndex,
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
ChunkIndex: chunkIndex,
|
|
Latency: time.Since(lastChunkTime).Milliseconds(),
|
|
RawResponse: eventData,
|
|
},
|
|
}
|
|
lastChunkTime = time.Now()
|
|
chunkIndex++
|
|
providerUtils.ProcessAndSendResponse(ctx, postHookRunner,
|
|
providerUtils.GetBifrostResponseForStreamResponse(nil, nil, passthroughResp, nil, nil, nil),
|
|
responseChan, postHookSpanFinalizer)
|
|
continue
|
|
}
|
|
// Handle each response in the slice
|
|
for i, response := range responses {
|
|
if response != nil {
|
|
response.ExtraFields = schemas.BifrostResponseExtraFields{
|
|
ChunkIndex: chunkIndex,
|
|
Latency: time.Since(lastChunkTime).Milliseconds(),
|
|
}
|
|
if postResponseConverter != nil {
|
|
response = postResponseConverter(response)
|
|
if response == nil {
|
|
logger.Warn("postResponseConverter returned nil; skipping chunk")
|
|
continue
|
|
}
|
|
}
|
|
lastChunkTime = time.Now()
|
|
chunkIndex++
|
|
|
|
// Only add raw response to the last chunk of the incoming event
|
|
if providerUtils.ShouldSendBackRawResponse(ctx, sendBackRawResponse) && i == len(responses)-1 {
|
|
response.ExtraFields.RawResponse = eventData
|
|
}
|
|
|
|
if isLastChunk && i == len(responses)-1 {
|
|
if response.Response == nil {
|
|
response.Response = &schemas.BifrostResponsesResponse{}
|
|
}
|
|
if usage.InputTokensDetails != nil {
|
|
usage.InputTokens = usage.InputTokens + usage.InputTokensDetails.CachedReadTokens + usage.InputTokensDetails.CachedWriteTokens
|
|
usage.TotalTokens = usage.TotalTokens + usage.InputTokensDetails.CachedReadTokens + usage.InputTokensDetails.CachedWriteTokens
|
|
}
|
|
response.Response.Usage = usage
|
|
// Set raw request if enabled
|
|
if sendBackRawRequest {
|
|
providerUtils.ParseAndSetRawRequest(&response.ExtraFields, jsonBody)
|
|
}
|
|
response.ExtraFields.Latency = time.Since(startTime).Milliseconds()
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
providerUtils.ProcessAndSendResponse(ctx, postHookRunner, providerUtils.GetBifrostResponseForStreamResponse(nil, nil, response, nil, nil, nil), responseChan, postHookSpanFinalizer)
|
|
return
|
|
}
|
|
providerUtils.ProcessAndSendResponse(ctx, postHookRunner, providerUtils.GetBifrostResponseForStreamResponse(nil, nil, response, nil, nil, nil), responseChan, postHookSpanFinalizer)
|
|
}
|
|
}
|
|
|
|
}
|
|
}()
|
|
|
|
return responseChan, nil
|
|
}
|
|
|
|
// BatchCreate creates a new batch job.
|
|
func (provider *AnthropicProvider) BatchCreate(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostBatchCreateRequest) (*schemas.BifrostBatchCreateResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.BatchCreateRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
|
|
if len(request.Requests) == 0 {
|
|
return nil, providerUtils.NewBifrostOperationError("requests array is required for Anthropic batch API", nil)
|
|
}
|
|
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
defer fasthttp.ReleaseRequest(req)
|
|
defer fasthttp.ReleaseResponse(resp)
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(provider.buildRequestURL(ctx, "/v1/messages/batches", schemas.BatchCreateRequest))
|
|
req.Header.SetMethod(http.MethodPost)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
// Build request body
|
|
anthropicReq := &AnthropicBatchCreateRequest{
|
|
Requests: make([]AnthropicBatchRequestItem, len(request.Requests)),
|
|
}
|
|
|
|
for i, r := range request.Requests {
|
|
anthropicReq.Requests[i] = AnthropicBatchRequestItem{
|
|
CustomID: r.CustomID,
|
|
Params: r.Params,
|
|
}
|
|
// Use Body if Params is empty
|
|
if anthropicReq.Requests[i].Params == nil && r.Body != nil {
|
|
anthropicReq.Requests[i].Params = r.Body
|
|
}
|
|
}
|
|
|
|
jsonData, err := providerUtils.MarshalSorted(anthropicReq)
|
|
if err != nil {
|
|
return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderRequestMarshal, err)
|
|
}
|
|
usedLargePayloadBody := setAnthropicRequestBody(ctx, req, jsonData)
|
|
|
|
sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest)
|
|
sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
defer wait()
|
|
if usedLargePayloadBody {
|
|
providerUtils.DrainLargePayloadRemainder(ctx)
|
|
}
|
|
if bifrostErr != nil {
|
|
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
return nil, parseAnthropicError(resp)
|
|
}
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err), jsonData, nil, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
|
|
var anthropicResp AnthropicBatchResponse
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(body, &anthropicResp, jsonData, sendBackRawRequest, sendBackRawResponse)
|
|
if bifrostErr != nil {
|
|
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, body, sendBackRawRequest, sendBackRawResponse)
|
|
}
|
|
|
|
return anthropicResp.ToBifrostBatchCreateResponse(latency, sendBackRawRequest, sendBackRawResponse, rawRequest, rawResponse), nil
|
|
}
|
|
|
|
// BatchList lists batch jobs using serial pagination across keys.
|
|
// Exhausts all pages from one key before moving to the next.
|
|
func (provider *AnthropicProvider) BatchList(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchListRequest) (*schemas.BifrostBatchListResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.BatchListRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)
|
|
|
|
// Initialize serial pagination helper (Anthropic uses AfterID for pagination)
|
|
helper, err := providerUtils.NewSerialListHelper(keys, request.AfterID, provider.logger)
|
|
if err != nil {
|
|
return nil, providerUtils.NewBifrostOperationError("invalid pagination cursor", err)
|
|
}
|
|
|
|
// Get current key to query
|
|
key, nativeCursor, ok := helper.GetCurrentKey()
|
|
if !ok {
|
|
// All keys exhausted
|
|
return &schemas.BifrostBatchListResponse{
|
|
Object: "list",
|
|
Data: []schemas.BifrostBatchRetrieveResponse{},
|
|
HasMore: false,
|
|
}, nil
|
|
}
|
|
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
defer fasthttp.ReleaseRequest(req)
|
|
defer fasthttp.ReleaseResponse(resp)
|
|
|
|
// Build URL with query params
|
|
baseURL := provider.buildRequestURL(ctx, "/v1/messages/batches", schemas.BatchListRequest)
|
|
values := url.Values{}
|
|
if request.Limit > 0 {
|
|
values.Set("limit", fmt.Sprintf("%d", request.Limit))
|
|
}
|
|
if request.BeforeID != nil && *request.BeforeID != "" {
|
|
values.Set("before_id", *request.BeforeID)
|
|
}
|
|
// Use native cursor from serial helper instead of request.AfterID
|
|
if nativeCursor != "" {
|
|
values.Set("after_id", nativeCursor)
|
|
}
|
|
requestURL := baseURL
|
|
if encodedValues := values.Encode(); encodedValues != "" {
|
|
requestURL += "?" + encodedValues
|
|
}
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(requestURL)
|
|
req.Header.SetMethod(http.MethodGet)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
defer wait()
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
return nil, parseAnthropicError(resp)
|
|
}
|
|
|
|
body, decodeErr := providerUtils.CheckAndDecodeBody(resp)
|
|
if decodeErr != nil {
|
|
return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, decodeErr)
|
|
}
|
|
|
|
var anthropicResp AnthropicBatchListResponse
|
|
_, _, bifrostErr = providerUtils.HandleProviderResponse(body, &anthropicResp, nil, false, sendBackRawResponse)
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// Convert batches to Bifrost format
|
|
batches := make([]schemas.BifrostBatchRetrieveResponse, 0, len(anthropicResp.Data))
|
|
var lastBatchID string
|
|
for _, batch := range anthropicResp.Data {
|
|
batches = append(batches, *batch.ToBifrostBatchRetrieveResponse(latency, false, false, nil, nil))
|
|
lastBatchID = batch.ID
|
|
}
|
|
|
|
// Build cursor for next request
|
|
// Anthropic uses LastID as the cursor for pagination
|
|
nextCursor, hasMore := helper.BuildNextCursor(anthropicResp.HasMore, lastBatchID)
|
|
|
|
// Convert to Bifrost response
|
|
bifrostResp := &schemas.BifrostBatchListResponse{
|
|
Object: "list",
|
|
Data: batches,
|
|
HasMore: hasMore,
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
},
|
|
}
|
|
if nextCursor != "" {
|
|
bifrostResp.NextCursor = &nextCursor
|
|
}
|
|
|
|
return bifrostResp, nil
|
|
}
|
|
|
|
// BatchRetrieve retrieves a specific batch job by trying each key until found.
|
|
func (provider *AnthropicProvider) BatchRetrieve(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchRetrieveRequest) (*schemas.BifrostBatchRetrieveResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.BatchRetrieveRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// batch id is required
|
|
if request.BatchID == "" {
|
|
return nil, providerUtils.NewBifrostOperationError("batch_id is required", nil)
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest)
|
|
sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)
|
|
|
|
var lastErr *schemas.BifrostError
|
|
for _, key := range keys {
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(provider.buildRequestURL(
|
|
ctx,
|
|
"/v1/messages/batches/"+url.PathEscape(request.BatchID),
|
|
schemas.BatchRetrieveRequest,
|
|
))
|
|
req.Header.SetMethod(http.MethodGet)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
lastErr = parseAnthropicError(resp)
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
continue
|
|
}
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err)
|
|
continue
|
|
}
|
|
|
|
var anthropicResp AnthropicBatchResponse
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(body, &anthropicResp, nil, sendBackRawRequest, sendBackRawResponse)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
|
|
result := anthropicResp.ToBifrostBatchRetrieveResponse(latency, sendBackRawRequest, sendBackRawResponse, rawRequest, rawResponse)
|
|
return result, nil
|
|
}
|
|
|
|
return nil, lastErr
|
|
}
|
|
|
|
// BatchCancel cancels a batch job by trying each key until successful.
|
|
func (provider *AnthropicProvider) BatchCancel(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchCancelRequest) (*schemas.BifrostBatchCancelResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.BatchCancelRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// batch id is required
|
|
if request.BatchID == "" {
|
|
return nil, providerUtils.NewBifrostOperationError("batch_id is required", nil)
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest)
|
|
sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)
|
|
|
|
var lastErr *schemas.BifrostError
|
|
for _, key := range keys {
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(provider.networkConfig.BaseURL + "/v1/messages/batches/" + request.BatchID + "/cancel")
|
|
req.Header.SetMethod(http.MethodPost)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
lastErr = parseAnthropicError(resp)
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
continue
|
|
}
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err)
|
|
continue
|
|
}
|
|
|
|
var anthropicResp AnthropicBatchResponse
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(body, &anthropicResp, nil, sendBackRawRequest, sendBackRawResponse)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
|
|
result := &schemas.BifrostBatchCancelResponse{
|
|
ID: anthropicResp.ID,
|
|
Object: anthropicResp.Type,
|
|
Status: ToBifrostBatchStatus(anthropicResp.ProcessingStatus),
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
},
|
|
}
|
|
|
|
if sendBackRawRequest {
|
|
result.ExtraFields.RawRequest = rawRequest
|
|
}
|
|
|
|
if anthropicResp.CancelInitiatedAt != nil {
|
|
cancellingAt := parseAnthropicTimestamp(*anthropicResp.CancelInitiatedAt)
|
|
result.CancellingAt = &cancellingAt
|
|
}
|
|
|
|
if anthropicResp.RequestCounts != nil {
|
|
result.RequestCounts = schemas.BatchRequestCounts{
|
|
Total: anthropicResp.RequestCounts.Processing + anthropicResp.RequestCounts.Succeeded + anthropicResp.RequestCounts.Errored + anthropicResp.RequestCounts.Canceled + anthropicResp.RequestCounts.Expired,
|
|
Completed: anthropicResp.RequestCounts.Succeeded,
|
|
Failed: anthropicResp.RequestCounts.Errored,
|
|
}
|
|
}
|
|
|
|
if sendBackRawResponse {
|
|
result.ExtraFields.RawResponse = rawResponse
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
return nil, lastErr
|
|
}
|
|
|
|
// BatchDelete is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) BatchDelete(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchDeleteRequest) (*schemas.BifrostBatchDeleteResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchDeleteRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// BatchResults retrieves batch results by trying each key until found.
|
|
func (provider *AnthropicProvider) BatchResults(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchResultsRequest) (*schemas.BifrostBatchResultsResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.BatchResultsRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if request.BatchID == "" {
|
|
return nil, providerUtils.NewBifrostOperationError("batch_id is required", nil)
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
|
|
var lastErr *schemas.BifrostError
|
|
for _, key := range keys {
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(provider.networkConfig.BaseURL + "/v1/messages/batches/" + request.BatchID + "/results")
|
|
req.Header.SetMethod(http.MethodGet)
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
lastErr = parseAnthropicError(resp)
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
continue
|
|
}
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err)
|
|
continue
|
|
}
|
|
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
|
|
// Parse JSONL content - each line is a separate result
|
|
var results []schemas.BatchResultItem
|
|
|
|
parseResult := providerUtils.ParseJSONL(body, func(line []byte) error {
|
|
var anthropicResult AnthropicBatchResultItem
|
|
if err := sonic.Unmarshal(line, &anthropicResult); err != nil {
|
|
provider.logger.Warn("failed to parse batch result line: %v", err)
|
|
return err
|
|
}
|
|
|
|
// Convert to Bifrost format
|
|
resultItem := schemas.BatchResultItem{
|
|
CustomID: anthropicResult.CustomID,
|
|
Result: &schemas.BatchResultData{
|
|
Type: anthropicResult.Result.Type,
|
|
Message: anthropicResult.Result.Message,
|
|
},
|
|
}
|
|
|
|
if anthropicResult.Result.Error != nil {
|
|
resultItem.Error = &schemas.BatchResultError{
|
|
Code: anthropicResult.Result.Error.Type,
|
|
Message: anthropicResult.Result.Error.Message,
|
|
}
|
|
}
|
|
|
|
results = append(results, resultItem)
|
|
return nil
|
|
})
|
|
|
|
batchResultsResp := &schemas.BifrostBatchResultsResponse{
|
|
BatchID: request.BatchID,
|
|
Results: results,
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
},
|
|
}
|
|
|
|
if len(parseResult.Errors) > 0 {
|
|
batchResultsResp.ExtraFields.ParseErrors = parseResult.Errors
|
|
}
|
|
|
|
return batchResultsResp, nil
|
|
}
|
|
|
|
return nil, lastErr
|
|
}
|
|
|
|
// Embedding is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) Embedding(ctx *schemas.BifrostContext, key schemas.Key, input *schemas.BifrostEmbeddingRequest) (*schemas.BifrostEmbeddingResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.EmbeddingRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// Speech is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) Speech(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostSpeechRequest) (*schemas.BifrostSpeechResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.SpeechRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// SpeechStream is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) SpeechStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostSpeechRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.SpeechStreamRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// Transcription is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) Transcription(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostTranscriptionRequest) (*schemas.BifrostTranscriptionResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.TranscriptionRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// TranscriptionStream is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) TranscriptionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostTranscriptionRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.TranscriptionStreamRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ImageGeneration is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ImageGeneration(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageGenerationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageGenerationRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ImageGenerationStream is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ImageGenerationStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostImageGenerationRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageGenerationStreamRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ImageEdit is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ImageEdit(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageEditRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageEditRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ImageEditStream is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ImageEditStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostImageEditRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageEditStreamRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ImageVariation is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ImageVariation(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageVariationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageVariationRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// Rerank is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostRerankRequest) (*schemas.BifrostRerankResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// OCR is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// FileUpload uploads a file to Anthropic's Files API.
|
|
func (provider *AnthropicProvider) FileUpload(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostFileUploadRequest) (*schemas.BifrostFileUploadResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.FileUploadRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
|
|
if len(request.File) == 0 {
|
|
return nil, providerUtils.NewBifrostOperationError("file content is required", nil)
|
|
}
|
|
|
|
// Create multipart form data
|
|
var buf bytes.Buffer
|
|
writer := multipart.NewWriter(&buf)
|
|
|
|
// Add file field
|
|
filename := request.Filename
|
|
if filename == "" {
|
|
filename = "file"
|
|
}
|
|
part, err := writer.CreateFormFile("file", filename)
|
|
if err != nil {
|
|
return nil, providerUtils.NewBifrostOperationError("failed to create form file", err)
|
|
}
|
|
if _, err := part.Write(request.File); err != nil {
|
|
return nil, providerUtils.NewBifrostOperationError("failed to write file content", err)
|
|
}
|
|
|
|
if err := writer.Close(); err != nil {
|
|
return nil, providerUtils.NewBifrostOperationError("failed to close multipart writer", err)
|
|
}
|
|
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
defer fasthttp.ReleaseRequest(req)
|
|
defer fasthttp.ReleaseResponse(resp)
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(provider.buildRequestURL(ctx, "/v1/files", schemas.FileUploadRequest))
|
|
req.Header.SetMethod(http.MethodPost)
|
|
req.Header.SetContentType(writer.FormDataContentType())
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
appendBetaHeader(req, AnthropicFilesAPIBetaHeader)
|
|
req.SetBody(buf.Bytes())
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
defer wait()
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK && resp.StatusCode() != fasthttp.StatusCreated {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
return nil, parseAnthropicError(resp)
|
|
}
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err)
|
|
}
|
|
|
|
var anthropicResp AnthropicFileResponse
|
|
sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest)
|
|
sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(body, &anthropicResp, nil, sendBackRawRequest, sendBackRawResponse)
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
return anthropicResp.ToBifrostFileUploadResponse(latency, sendBackRawRequest, sendBackRawResponse, rawRequest, rawResponse), nil
|
|
}
|
|
|
|
// FileList lists files from all provided keys and aggregates results.
|
|
// FileList lists files using serial pagination across keys.
|
|
// Exhausts all pages from one key before moving to the next.
|
|
func (provider *AnthropicProvider) FileList(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostFileListRequest) (*schemas.BifrostFileListResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.FileListRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest)
|
|
sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)
|
|
|
|
// Initialize serial pagination helper
|
|
helper, err := providerUtils.NewSerialListHelper(keys, request.After, provider.logger)
|
|
if err != nil {
|
|
return nil, providerUtils.NewBifrostOperationError("invalid pagination cursor", err)
|
|
}
|
|
|
|
// Get current key to query
|
|
key, nativeCursor, ok := helper.GetCurrentKey()
|
|
if !ok {
|
|
// All keys exhausted
|
|
return &schemas.BifrostFileListResponse{
|
|
Object: "list",
|
|
Data: []schemas.FileObject{},
|
|
HasMore: false,
|
|
}, nil
|
|
}
|
|
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
defer fasthttp.ReleaseRequest(req)
|
|
defer fasthttp.ReleaseResponse(resp)
|
|
|
|
// Build URL with query params
|
|
requestURL := provider.buildRequestURL(ctx, "/v1/files", schemas.FileListRequest)
|
|
values := url.Values{}
|
|
if request.Limit > 0 {
|
|
values.Set("limit", fmt.Sprintf("%d", request.Limit))
|
|
}
|
|
// Use native cursor from serial helper instead of request.After
|
|
if nativeCursor != "" {
|
|
values.Set("after_id", nativeCursor)
|
|
}
|
|
if encodedValues := values.Encode(); encodedValues != "" {
|
|
requestURL += "?" + encodedValues
|
|
}
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(requestURL)
|
|
req.Header.SetMethod(http.MethodGet)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
appendBetaHeader(req, AnthropicFilesAPIBetaHeader)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
defer wait()
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
return nil, parseAnthropicError(resp)
|
|
}
|
|
|
|
body, decodeErr := providerUtils.CheckAndDecodeBody(resp)
|
|
if decodeErr != nil {
|
|
return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, decodeErr)
|
|
}
|
|
|
|
var anthropicResp AnthropicFileListResponse
|
|
_, _, bifrostErr = providerUtils.HandleProviderResponse(body, &anthropicResp, nil, sendBackRawRequest, sendBackRawResponse)
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
// Convert files to Bifrost format
|
|
files := make([]schemas.FileObject, 0, len(anthropicResp.Data))
|
|
var lastFileID string
|
|
for _, file := range anthropicResp.Data {
|
|
files = append(files, schemas.FileObject{
|
|
ID: file.ID,
|
|
Object: file.Type,
|
|
Bytes: file.SizeBytes,
|
|
CreatedAt: parseAnthropicFileTimestamp(file.CreatedAt),
|
|
Filename: file.Filename,
|
|
Purpose: schemas.FilePurposeBatch,
|
|
Status: schemas.FileStatusProcessed,
|
|
})
|
|
lastFileID = file.ID
|
|
}
|
|
|
|
// Build cursor for next request
|
|
// Anthropic uses LastID as the cursor for pagination
|
|
nextCursor, hasMore := helper.BuildNextCursor(anthropicResp.HasMore, lastFileID)
|
|
|
|
// Convert to Bifrost response
|
|
bifrostResp := &schemas.BifrostFileListResponse{
|
|
Object: "list",
|
|
Data: files,
|
|
HasMore: hasMore,
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
},
|
|
}
|
|
if nextCursor != "" {
|
|
bifrostResp.After = &nextCursor
|
|
}
|
|
|
|
return bifrostResp, nil
|
|
}
|
|
|
|
// FileRetrieve retrieves file metadata from Anthropic's Files API by trying each key until found.
|
|
func (provider *AnthropicProvider) FileRetrieve(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostFileRetrieveRequest) (*schemas.BifrostFileRetrieveResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.FileRetrieveRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
|
|
if request.FileID == "" {
|
|
return nil, providerUtils.NewBifrostOperationError("file_id is required", nil)
|
|
}
|
|
|
|
sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest)
|
|
sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)
|
|
|
|
var lastErr *schemas.BifrostError
|
|
for _, key := range keys {
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(provider.buildRequestURL(
|
|
ctx,
|
|
"/v1/files/"+url.PathEscape(request.FileID),
|
|
schemas.FileRetrieveRequest,
|
|
))
|
|
req.Header.SetMethod(http.MethodGet)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
appendBetaHeader(req, AnthropicFilesAPIBetaHeader)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
lastErr = parseAnthropicError(resp)
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
continue
|
|
}
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err)
|
|
continue
|
|
}
|
|
|
|
var anthropicResp AnthropicFileResponse
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(body, &anthropicResp, nil, sendBackRawRequest, sendBackRawResponse)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
|
|
return anthropicResp.ToBifrostFileRetrieveResponse(latency, sendBackRawRequest, sendBackRawResponse, rawRequest, rawResponse), nil
|
|
}
|
|
|
|
return nil, lastErr
|
|
}
|
|
|
|
// FileDelete deletes a file from Anthropic's Files API by trying each key until successful.
|
|
func (provider *AnthropicProvider) FileDelete(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostFileDeleteRequest) (*schemas.BifrostFileDeleteResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.FileDeleteRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
|
|
if request.FileID == "" {
|
|
return nil, providerUtils.NewBifrostOperationError("file_id is required", nil)
|
|
}
|
|
|
|
sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest)
|
|
sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)
|
|
|
|
var lastErr *schemas.BifrostError
|
|
for _, key := range keys {
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(provider.networkConfig.BaseURL + "/v1/files/" + request.FileID)
|
|
req.Header.SetMethod(http.MethodDelete)
|
|
req.Header.SetContentType("application/json")
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
appendBetaHeader(req, AnthropicFilesAPIBetaHeader)
|
|
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK && resp.StatusCode() != fasthttp.StatusNoContent {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
lastErr = parseAnthropicError(resp)
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
continue
|
|
}
|
|
|
|
// For 204 No Content, return success without parsing body
|
|
if resp.StatusCode() == fasthttp.StatusNoContent {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
return &schemas.BifrostFileDeleteResponse{
|
|
ID: request.FileID,
|
|
Object: "file",
|
|
Deleted: true,
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err)
|
|
continue
|
|
}
|
|
|
|
var anthropicResp AnthropicFileDeleteResponse
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(body, &anthropicResp, nil, sendBackRawRequest, sendBackRawResponse)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
|
|
result := &schemas.BifrostFileDeleteResponse{
|
|
ID: anthropicResp.ID,
|
|
Object: "file",
|
|
Deleted: anthropicResp.Type == "file_deleted",
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
},
|
|
}
|
|
|
|
if sendBackRawRequest {
|
|
result.ExtraFields.RawRequest = rawRequest
|
|
}
|
|
|
|
if sendBackRawResponse {
|
|
result.ExtraFields.RawResponse = rawResponse
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
return nil, lastErr
|
|
}
|
|
|
|
// FileContent downloads file content from Anthropic's Files API by trying each key until found.
|
|
// Note: Only files created by skills or the code execution tool can be downloaded.
|
|
func (provider *AnthropicProvider) FileContent(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostFileContentRequest) (*schemas.BifrostFileContentResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.FileContentRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
providerName := provider.GetProviderKey()
|
|
|
|
if request.FileID == "" {
|
|
return nil, providerUtils.NewBifrostOperationError("file_id is required", nil)
|
|
}
|
|
|
|
var lastErr *schemas.BifrostError
|
|
for _, key := range keys {
|
|
// Create request
|
|
req := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
|
|
// Set headers
|
|
providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil)
|
|
req.SetRequestURI(provider.networkConfig.BaseURL + "/v1/files/" + request.FileID + "/content")
|
|
req.Header.SetMethod(http.MethodGet)
|
|
|
|
if key.Value.GetValue() != "" {
|
|
req.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
req.Header.Set("anthropic-version", provider.apiVersion)
|
|
appendBetaHeader(req, AnthropicFilesAPIBetaHeader)
|
|
// Make request
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp)
|
|
if bifrostErr != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = bifrostErr
|
|
continue
|
|
}
|
|
|
|
// Handle error response
|
|
if resp.StatusCode() != fasthttp.StatusOK {
|
|
provider.logger.Debug("error from %s provider: %s", providerName, string(resp.Body()))
|
|
lastErr = parseAnthropicError(resp)
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
continue
|
|
}
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
lastErr = providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err)
|
|
continue
|
|
}
|
|
|
|
// Get content type from response
|
|
contentType := string(resp.Header.ContentType())
|
|
if contentType == "" {
|
|
contentType = "application/octet-stream"
|
|
}
|
|
|
|
content := append([]byte(nil), body...)
|
|
wait()
|
|
fasthttp.ReleaseRequest(req)
|
|
fasthttp.ReleaseResponse(resp)
|
|
|
|
return &schemas.BifrostFileContentResponse{
|
|
FileID: request.FileID,
|
|
Content: content,
|
|
ContentType: contentType,
|
|
ExtraFields: schemas.BifrostResponseExtraFields{
|
|
Latency: latency.Milliseconds(),
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
return nil, lastErr
|
|
}
|
|
|
|
// CountTokens counts tokens for a given request using Anthropic's API.
|
|
func (provider *AnthropicProvider) CountTokens(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostResponsesRequest) (*schemas.BifrostCountTokensResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.CountTokensRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
jsonBody, err := getRequestBodyForResponses(ctx, request, false, []string{"max_tokens", "temperature"})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
responseBody, latency, providerResponseHeaders, bifrostErr := provider.completeRequest(ctx, jsonBody, provider.buildRequestURL(ctx, "/v1/messages/count_tokens", schemas.CountTokensRequest), key.Value.GetValue(), schemas.CountTokensRequest)
|
|
if providerResponseHeaders != nil {
|
|
ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerResponseHeaders)
|
|
}
|
|
if bifrostErr != nil {
|
|
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonBody, responseBody, provider.sendBackRawRequest, provider.sendBackRawResponse)
|
|
}
|
|
|
|
anthropicResponse := &AnthropicCountTokensResponse{}
|
|
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(
|
|
responseBody,
|
|
anthropicResponse,
|
|
jsonBody,
|
|
providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest),
|
|
providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse),
|
|
)
|
|
|
|
if bifrostErr != nil {
|
|
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonBody, responseBody, provider.sendBackRawRequest, provider.sendBackRawResponse)
|
|
}
|
|
|
|
response := anthropicResponse.ToBifrostCountTokensResponse(request.Model)
|
|
response.Model = request.Model
|
|
|
|
response.ExtraFields.Latency = latency.Milliseconds()
|
|
response.ExtraFields.ProviderResponseHeaders = providerResponseHeaders
|
|
|
|
if providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) {
|
|
response.ExtraFields.RawRequest = rawRequest
|
|
}
|
|
|
|
if providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) {
|
|
response.ExtraFields.RawResponse = rawResponse
|
|
}
|
|
|
|
return response, nil
|
|
}
|
|
|
|
// VideoGeneration is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) VideoGeneration(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoGenerationRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoGenerationRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// VideoRetrieve is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) VideoRetrieve(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoRetrieveRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoRetrieveRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// VideoDownload is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) VideoDownload(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoDownloadRequest) (*schemas.BifrostVideoDownloadResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoDownloadRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// VideoDelete is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) VideoDelete(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoDeleteRequest) (*schemas.BifrostVideoDeleteResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoDeleteRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// VideoList is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) VideoList(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoListRequest) (*schemas.BifrostVideoListResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoListRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// VideoRemix is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) VideoRemix(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoRemixRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoRemixRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerCreate is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerCreate(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostContainerCreateRequest) (*schemas.BifrostContainerCreateResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerCreateRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerList is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerListRequest) (*schemas.BifrostContainerListResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerListRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerRetrieve is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerRetrieveRequest) (*schemas.BifrostContainerRetrieveResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerRetrieveRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerDelete is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerDeleteRequest) (*schemas.BifrostContainerDeleteResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerDeleteRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerFileCreate is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerFileCreate(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostContainerFileCreateRequest) (*schemas.BifrostContainerFileCreateResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileCreateRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerFileList is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerFileList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileListRequest) (*schemas.BifrostContainerFileListResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileListRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerFileRetrieve is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerFileRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileRetrieveRequest) (*schemas.BifrostContainerFileRetrieveResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileRetrieveRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerFileContent is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerFileContent(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileContentRequest) (*schemas.BifrostContainerFileContentResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileContentRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
// ContainerFileDelete is not supported by the Anthropic provider.
|
|
func (provider *AnthropicProvider) ContainerFileDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileDeleteRequest) (*schemas.BifrostContainerFileDeleteResponse, *schemas.BifrostError) {
|
|
return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileDeleteRequest, provider.GetProviderKey())
|
|
}
|
|
|
|
func (provider *AnthropicProvider) Passthrough(
|
|
ctx *schemas.BifrostContext,
|
|
key schemas.Key,
|
|
req *schemas.BifrostPassthroughRequest,
|
|
) (*schemas.BifrostPassthroughResponse, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.PassthroughRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
url := provider.networkConfig.BaseURL + req.Path
|
|
if req.RawQuery != "" {
|
|
url += "?" + req.RawQuery
|
|
}
|
|
|
|
fasthttpReq := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
defer fasthttp.ReleaseResponse(resp)
|
|
defer fasthttp.ReleaseRequest(fasthttpReq)
|
|
|
|
fasthttpReq.Header.SetMethod(req.Method)
|
|
fasthttpReq.SetRequestURI(url)
|
|
|
|
providerUtils.SetExtraHeaders(ctx, fasthttpReq, provider.networkConfig.ExtraHeaders, nil)
|
|
|
|
for k, v := range req.SafeHeaders {
|
|
fasthttpReq.Header.Set(k, v)
|
|
}
|
|
|
|
if key.Value.GetValue() != "" {
|
|
fasthttpReq.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
fasthttpReq.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
fasthttpReq.SetBody(req.Body)
|
|
|
|
latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, fasthttpReq, resp)
|
|
defer wait()
|
|
if bifrostErr != nil {
|
|
return nil, bifrostErr
|
|
}
|
|
|
|
headers := providerUtils.ExtractProviderResponseHeaders(resp)
|
|
|
|
body, err := providerUtils.CheckAndDecodeBody(resp)
|
|
if err != nil {
|
|
return nil, providerUtils.NewBifrostOperationError("failed to decode response body", err)
|
|
}
|
|
|
|
for k := range headers {
|
|
if strings.EqualFold(k, "Content-Encoding") || strings.EqualFold(k, "Content-Length") {
|
|
delete(headers, k)
|
|
}
|
|
}
|
|
|
|
bifrostResponse := &schemas.BifrostPassthroughResponse{
|
|
StatusCode: resp.StatusCode(),
|
|
Headers: headers,
|
|
Body: body,
|
|
}
|
|
|
|
bifrostResponse.ExtraFields.Latency = latency.Milliseconds()
|
|
|
|
if providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) {
|
|
providerUtils.ParseAndSetRawRequestIfJSON(fasthttpReq, &bifrostResponse.ExtraFields)
|
|
}
|
|
|
|
return bifrostResponse, nil
|
|
}
|
|
|
|
func (provider *AnthropicProvider) PassthroughStream(
|
|
ctx *schemas.BifrostContext,
|
|
postHookRunner schemas.PostHookRunner,
|
|
postHookSpanFinalizer func(context.Context),
|
|
key schemas.Key,
|
|
req *schemas.BifrostPassthroughRequest,
|
|
) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) {
|
|
if err := providerUtils.CheckOperationAllowed(schemas.Anthropic, provider.customProviderConfig, schemas.PassthroughStreamRequest); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
url := provider.networkConfig.BaseURL + req.Path
|
|
if req.RawQuery != "" {
|
|
url += "?" + req.RawQuery
|
|
}
|
|
|
|
startTime := time.Now()
|
|
|
|
fasthttpReq := fasthttp.AcquireRequest()
|
|
resp := fasthttp.AcquireResponse()
|
|
resp.StreamBody = true
|
|
defer fasthttp.ReleaseRequest(fasthttpReq)
|
|
|
|
fasthttpReq.Header.SetMethod(req.Method)
|
|
fasthttpReq.SetRequestURI(url)
|
|
|
|
providerUtils.SetExtraHeaders(ctx, fasthttpReq, provider.networkConfig.ExtraHeaders, nil)
|
|
|
|
for k, v := range req.SafeHeaders {
|
|
fasthttpReq.Header.Set(k, v)
|
|
}
|
|
|
|
fasthttpReq.Header.Set("Connection", "close")
|
|
|
|
if key.Value.GetValue() != "" {
|
|
fasthttpReq.Header.Set("x-api-key", key.Value.GetValue())
|
|
}
|
|
fasthttpReq.Header.Set("anthropic-version", provider.apiVersion)
|
|
|
|
fasthttpReq.SetBody(req.Body)
|
|
|
|
activeClient := providerUtils.PrepareResponseStreaming(ctx, provider.streamingClient, resp)
|
|
if err := activeClient.Do(fasthttpReq, resp); err != nil {
|
|
providerUtils.ReleaseStreamingResponse(resp)
|
|
if errors.Is(err, context.Canceled) {
|
|
return nil, &schemas.BifrostError{
|
|
IsBifrostError: false,
|
|
Error: &schemas.ErrorField{
|
|
Type: schemas.Ptr(schemas.RequestCancelled),
|
|
Message: schemas.ErrRequestCancelled,
|
|
Error: err,
|
|
},
|
|
}
|
|
}
|
|
if errors.Is(err, fasthttp.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) {
|
|
return nil, providerUtils.NewBifrostTimeoutError(schemas.ErrProviderRequestTimedOut, err)
|
|
}
|
|
return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderDoRequest, err)
|
|
}
|
|
|
|
headers := providerUtils.ExtractProviderResponseHeaders(resp)
|
|
|
|
bodyStream := resp.BodyStream()
|
|
if bodyStream == nil {
|
|
providerUtils.ReleaseStreamingResponse(resp)
|
|
return nil, providerUtils.NewBifrostOperationError(
|
|
"provider returned an empty stream body",
|
|
fmt.Errorf("provider returned an empty stream body"),
|
|
)
|
|
}
|
|
|
|
// Wrap reader with idle timeout to detect stalled streams.
|
|
providerUtils.SetStreamIdleTimeoutIfEmpty(ctx, provider.networkConfig.StreamIdleTimeoutInSeconds)
|
|
rawBodyStream := bodyStream
|
|
bodyStream, stopIdleTimeout := providerUtils.NewIdleTimeoutReader(bodyStream, rawBodyStream, providerUtils.GetStreamIdleTimeout(ctx))
|
|
|
|
// Cancellation must close the raw stream to unblock reads.
|
|
stopCancellation := providerUtils.SetupStreamCancellation(ctx, rawBodyStream, provider.logger)
|
|
|
|
extraFields := schemas.BifrostResponseExtraFields{}
|
|
statusCode := resp.StatusCode()
|
|
|
|
if providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) {
|
|
providerUtils.ParseAndSetRawRequestIfJSON(fasthttpReq, &extraFields)
|
|
}
|
|
|
|
ch := make(chan *schemas.BifrostStreamChunk, schemas.DefaultStreamBufferSize)
|
|
go func() {
|
|
defer providerUtils.EnsureStreamFinalizerCalled(ctx, postHookSpanFinalizer)
|
|
defer func() {
|
|
if ctx.Err() == context.Canceled {
|
|
providerUtils.HandleStreamCancellation(ctx, postHookRunner, ch, provider.logger, postHookSpanFinalizer)
|
|
} else if ctx.Err() == context.DeadlineExceeded {
|
|
providerUtils.HandleStreamTimeout(ctx, postHookRunner, ch, provider.logger, postHookSpanFinalizer)
|
|
}
|
|
close(ch)
|
|
}()
|
|
defer providerUtils.ReleaseStreamingResponse(resp)
|
|
defer stopIdleTimeout()
|
|
defer stopCancellation()
|
|
|
|
buf := make([]byte, 4096)
|
|
for {
|
|
n, readErr := bodyStream.Read(buf)
|
|
if n > 0 {
|
|
chunk := make([]byte, n)
|
|
copy(chunk, buf[:n])
|
|
select {
|
|
case ch <- &schemas.BifrostStreamChunk{
|
|
BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{
|
|
StatusCode: statusCode,
|
|
Headers: headers,
|
|
Body: chunk,
|
|
ExtraFields: extraFields,
|
|
},
|
|
}:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
if readErr == io.EOF {
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
extraFields.Latency = time.Since(startTime).Milliseconds()
|
|
finalResp := &schemas.BifrostResponse{
|
|
PassthroughResponse: &schemas.BifrostPassthroughResponse{
|
|
StatusCode: statusCode,
|
|
Headers: headers,
|
|
ExtraFields: extraFields,
|
|
},
|
|
}
|
|
postHookRunner(ctx, finalResp, nil)
|
|
if postHookSpanFinalizer != nil {
|
|
postHookSpanFinalizer(ctx)
|
|
}
|
|
return
|
|
}
|
|
if readErr != nil {
|
|
if ctx.Err() != nil {
|
|
return // let defer handle cancel/timeout
|
|
}
|
|
ctx.SetValue(schemas.BifrostContextKeyStreamEndIndicator, true)
|
|
extraFields.Latency = time.Since(startTime).Milliseconds()
|
|
providerUtils.ProcessAndSendError(ctx, postHookRunner, readErr, ch, provider.logger, postHookSpanFinalizer)
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
return ch, nil
|
|
}
|