// Package schemas defines the core schemas and types used by the Bifrost system. package schemas import ( "encoding/json" "errors" "fmt" "strconv" ) const ( DefaultInitialPoolSize = 5000 ) type KeySelector func(ctx *BifrostContext, keys []Key, providerKey ModelProvider, model string) (Key, error) // BifrostConfig represents the configuration for initializing a Bifrost instance. // It contains the necessary components for setting up the system including account details, // plugins, logging, and initial pool size. type BifrostConfig struct { Account Account LLMPlugins []LLMPlugin MCPPlugins []MCPPlugin OAuth2Provider OAuth2Provider Logger Logger Tracer Tracer // Tracer for distributed tracing (nil = NoOpTracer) InitialPoolSize int // Initial pool size for sync pools in Bifrost. Higher values will reduce memory allocations but will increase memory usage. DropExcessRequests bool // If true, in cases where the queue is full, requests will not wait for the queue to be empty and will be dropped instead. MCPConfig *MCPConfig // MCP (Model Context Protocol) configuration for tool integration KeySelector KeySelector // Custom key selector function KVStore KVStore // shared KV store for clustering/session stickiness; nil = disabled } // ModelProvider represents the different AI model providers supported by Bifrost. type ModelProvider string const ( OpenAI ModelProvider = "openai" Azure ModelProvider = "azure" Anthropic ModelProvider = "anthropic" Bedrock ModelProvider = "bedrock" Cohere ModelProvider = "cohere" Vertex ModelProvider = "vertex" Mistral ModelProvider = "mistral" Ollama ModelProvider = "ollama" Groq ModelProvider = "groq" SGL ModelProvider = "sgl" Parasail ModelProvider = "parasail" Perplexity ModelProvider = "perplexity" Cerebras ModelProvider = "cerebras" Gemini ModelProvider = "gemini" OpenRouter ModelProvider = "openrouter" Elevenlabs ModelProvider = "elevenlabs" HuggingFace ModelProvider = "huggingface" Nebius ModelProvider = "nebius" XAI ModelProvider = "xai" Replicate ModelProvider = "replicate" VLLM ModelProvider = "vllm" Runway ModelProvider = "runway" Fireworks ModelProvider = "fireworks" ) // SupportedBaseProviders is the list of base providers allowed for custom providers. var SupportedBaseProviders = []ModelProvider{ Anthropic, Bedrock, Cohere, Gemini, OpenAI, HuggingFace, Replicate, } // StandardProviders is the list of all built-in (non-custom) providers. var StandardProviders = []ModelProvider{ Anthropic, Azure, Bedrock, Cerebras, Cohere, Gemini, Groq, Mistral, Ollama, OpenAI, Parasail, Perplexity, SGL, Vertex, OpenRouter, Elevenlabs, HuggingFace, Nebius, XAI, Replicate, VLLM, Runway, Fireworks, } // RequestType represents the type of request being made to a provider. type RequestType string const ( ListModelsRequest RequestType = "list_models" TextCompletionRequest RequestType = "text_completion" TextCompletionStreamRequest RequestType = "text_completion_stream" ChatCompletionRequest RequestType = "chat_completion" ChatCompletionStreamRequest RequestType = "chat_completion_stream" ResponsesRequest RequestType = "responses" ResponsesStreamRequest RequestType = "responses_stream" EmbeddingRequest RequestType = "embedding" SpeechRequest RequestType = "speech" SpeechStreamRequest RequestType = "speech_stream" TranscriptionRequest RequestType = "transcription" TranscriptionStreamRequest RequestType = "transcription_stream" ImageGenerationRequest RequestType = "image_generation" ImageGenerationStreamRequest RequestType = "image_generation_stream" ImageEditRequest RequestType = "image_edit" ImageEditStreamRequest RequestType = "image_edit_stream" ImageVariationRequest RequestType = "image_variation" VideoGenerationRequest RequestType = "video_generation" VideoRetrieveRequest RequestType = "video_retrieve" VideoDownloadRequest RequestType = "video_download" VideoDeleteRequest RequestType = "video_delete" VideoListRequest RequestType = "video_list" VideoRemixRequest RequestType = "video_remix" BatchCreateRequest RequestType = "batch_create" BatchListRequest RequestType = "batch_list" BatchRetrieveRequest RequestType = "batch_retrieve" BatchCancelRequest RequestType = "batch_cancel" BatchResultsRequest RequestType = "batch_results" BatchDeleteRequest RequestType = "batch_delete" FileUploadRequest RequestType = "file_upload" FileListRequest RequestType = "file_list" FileRetrieveRequest RequestType = "file_retrieve" FileDeleteRequest RequestType = "file_delete" FileContentRequest RequestType = "file_content" ContainerCreateRequest RequestType = "container_create" ContainerListRequest RequestType = "container_list" ContainerRetrieveRequest RequestType = "container_retrieve" ContainerDeleteRequest RequestType = "container_delete" ContainerFileCreateRequest RequestType = "container_file_create" ContainerFileListRequest RequestType = "container_file_list" ContainerFileRetrieveRequest RequestType = "container_file_retrieve" ContainerFileContentRequest RequestType = "container_file_content" ContainerFileDeleteRequest RequestType = "container_file_delete" RerankRequest RequestType = "rerank" OCRRequest RequestType = "ocr" CountTokensRequest RequestType = "count_tokens" MCPToolExecutionRequest RequestType = "mcp_tool_execution" PassthroughRequest RequestType = "passthrough" PassthroughStreamRequest RequestType = "passthrough_stream" UnknownRequest RequestType = "unknown" WebSocketResponsesRequest RequestType = "websocket_responses" RealtimeRequest RequestType = "realtime" ) // BifrostContextKey is a type for context keys used in Bifrost. type BifrostContextKey string // BifrostContextKeyRequestType is a context key for the request type. const ( BifrostContextKeySessionToken BifrostContextKey = "bifrost-session-token" // string (session token for authentication - set by auth middleware) BifrostContextKeyVirtualKey BifrostContextKey = "x-bf-vk" // string BifrostContextKeyAPIKeyName BifrostContextKey = "x-bf-api-key" // string (explicit key name selection) BifrostContextKeyAPIKeyID BifrostContextKey = "x-bf-api-key-id" // string (explicit key ID selection, takes priority over name) BifrostContextKeyRequestID BifrostContextKey = "request-id" // string BifrostContextKeyFallbackRequestID BifrostContextKey = "fallback-request-id" // string BifrostContextKeyDirectKey BifrostContextKey = "bifrost-direct-key" // Key struct // NOTE: []string is used for both keys, and by default all clients/tools are included (when nil). // If "*" is present, all clients/tools are included, and [] means no clients/tools are included. // Request context filtering takes priority over client config - context can override client exclusions. MCPContextKeyIncludeClients BifrostContextKey = "mcp-include-clients" // Context key for whitelist client filtering MCPContextKeyIncludeTools BifrostContextKey = "mcp-include-tools" // Context key for whitelist tool filtering (Note: toolName should be in "clientName-toolName" format for individual tools, or "clientName-*" for wildcard) BifrostContextKeySelectedKeyID BifrostContextKey = "bifrost-selected-key-id" // string (to store the selected key ID (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeySelectedKeyName BifrostContextKey = "bifrost-selected-key-name" // string (to store the selected key name (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceVirtualKeyID BifrostContextKey = "bifrost-governance-virtual-key-id" // string (to store the virtual key ID (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceVirtualKeyName BifrostContextKey = "bifrost-governance-virtual-key-name" // string (to store the virtual key name (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceTeamID BifrostContextKey = "bifrost-governance-team-id" // string (to store the team ID (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceTeamName BifrostContextKey = "bifrost-governance-team-name" // string (to store the team name (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceCustomerID BifrostContextKey = "bifrost-governance-customer-id" // string (to store the customer ID (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceCustomerName BifrostContextKey = "bifrost-governance-customer-name" // string (to store the customer name (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceBusinessUnitID BifrostContextKey = "bifrost-governance-business-unit-id" // string (to store the business unit ID (set by enterprise governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceBusinessUnitName BifrostContextKey = "bifrost-governance-business-unit-name" // string (to store the business unit name (set by enterprise governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceRoutingRuleID BifrostContextKey = "bifrost-governance-routing-rule-id" // string (to store the routing rule ID (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceRoutingRuleName BifrostContextKey = "bifrost-governance-routing-rule-name" // string (to store the routing rule name (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeySelectedPromptName BifrostContextKey = "bifrost-selected-prompt-name" // string (display name of the selected prompt (set by prompts plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeySelectedPromptVersion BifrostContextKey = "bifrost-selected-prompt-version" // string (numeric version as string, e.g. "3" (set by prompts plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeySelectedPromptID BifrostContextKey = "bifrost-selected-prompt-id" // string (id of the selected prompt (set by prompts plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceIncludeOnlyKeys BifrostContextKey = "bf-governance-include-only-keys" // []string (to store the include-only key IDs for provider config routing (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyNumberOfRetries BifrostContextKey = "bifrost-number-of-retries" // int (to store the number of retries (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyFallbackIndex BifrostContextKey = "bifrost-fallback-index" // int (to store the fallback index (set by bifrost - DO NOT SET THIS MANUALLY)) 0 for primary, 1 for first fallback, etc. BifrostContextKeyStreamEndIndicator BifrostContextKey = "bifrost-stream-end-indicator" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyStreamIdleTimeout BifrostContextKey = "bifrost-stream-idle-timeout" // time.Duration (per-chunk idle timeout for streaming) BifrostContextKeySkipKeySelection BifrostContextKey = "bifrost-skip-key-selection" // bool (will pass an empty key to the provider) BifrostContextKeyExtraHeaders BifrostContextKey = "bifrost-extra-headers" // map[string][]string BifrostContextKeyURLPath BifrostContextKey = "bifrost-extra-url-path" // string BifrostContextKeyUseRawRequestBody BifrostContextKey = "bifrost-use-raw-request-body" BifrostContextKeyChangeRequestType BifrostContextKey = "bifrost-change-request-type" // RequestType (set by plugins to trigger request type conversion in core, e.g. text->chat or chat->responses) BifrostContextKeySendBackRawRequest BifrostContextKey = "bifrost-send-back-raw-request" // bool (per-request override — read by bifrost.go, never overwritten) BifrostContextKeySendBackRawResponse BifrostContextKey = "bifrost-send-back-raw-response" // bool (per-request override — read by bifrost.go, never overwritten) BifrostContextKeyIntegrationType BifrostContextKey = "bifrost-integration-type" // integration used in gateway (e.g. openai, anthropic, bedrock, etc.) BifrostContextKeyIsResponsesToChatCompletionFallback BifrostContextKey = "bifrost-is-responses-to-chat-completion-fallback" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostMCPAgentOriginalRequestID BifrostContextKey = "bifrost-mcp-agent-original-request-id" // string (to store the original request ID for MCP agent mode) BifrostContextKeyParentMCPRequestID BifrostContextKey = "bf-parent-mcp-request-id" // string (parent request ID for nested tool calls from executeCode) BifrostContextKeyStructuredOutputToolName BifrostContextKey = "bifrost-structured-output-tool-name" // string (to store the name of the structured output tool (set by bifrost)) BifrostContextKeyUserAgent BifrostContextKey = "bifrost-user-agent" // string (set by bifrost) BifrostContextKeyTraceID BifrostContextKey = "bifrost-trace-id" // string (trace ID for distributed tracing - set by tracing middleware) BifrostContextKeySpanID BifrostContextKey = "bifrost-span-id" // string (current span ID for child span creation - set by tracer) BifrostContextKeyParentSpanID BifrostContextKey = "bifrost-parent-span-id" // string (parent span ID from W3C traceparent header - set by tracing middleware) BifrostContextKeyStreamStartTime BifrostContextKey = "bifrost-stream-start-time" // time.Time (start time for streaming TTFT calculation - set by bifrost) BifrostContextKeyTracer BifrostContextKey = "bifrost-tracer" // Tracer (tracer instance for completing deferred spans - set by bifrost) BifrostContextKeyDeferTraceCompletion BifrostContextKey = "bifrost-defer-trace-completion" // bool (signals trace completion should be deferred for streaming - set by streaming handlers) BifrostContextKeyTraceCompleter BifrostContextKey = "bifrost-trace-completer" // func([]PluginLogEntry) (callback to complete trace after streaming, receives transport plugin logs - set by tracing middleware) BifrostContextKeyAccumulatorID BifrostContextKey = "bifrost-accumulator-id" // string (ID for streaming accumulator lookup - set by tracer for accumulator operations) BifrostContextKeyMCPUserSession BifrostContextKey = "bifrost-mcp-user-session" // string (per-user OAuth session token, automatically generated by bifrost) BifrostContextKeyMCPUserID BifrostContextKey = "bifrost-mcp-user-id" // string (per-user OAuth user identifier from X-Bf-User-Id header) BifrostContextKeyOAuthRedirectURI BifrostContextKey = "bifrost-oauth-redirect-uri" // string (OAuth callback URL, e.g. https://host/api/oauth/callback - set by HTTP middleware) BifrostContextKeyIsMCPGateway BifrostContextKey = "bifrost-is-mcp-gateway" // bool (true when request is being handled via the MCP gateway path) BifrostContextKeyHasEmittedMessageDelta BifrostContextKey = "bifrost-has-emitted-message-delta" // bool (tracks whether message_delta was already emitted during streaming - avoids duplicates) BifrostContextKeySkipDBUpdate BifrostContextKey = "bifrost-skip-db-update" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernancePluginName BifrostContextKey = "governance-plugin-name" // string (name of the governance plugin that processed the request - set by bifrost) BifrostContextKeyPromptsPluginName BifrostContextKey = "prompts-plugin-name" // string (name of the prompts plugin to use - set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyIsEnterprise BifrostContextKey = "is-enterprise" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyAvailableProviders BifrostContextKey = "available-providers" // []ModelProvider (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyStoreRawRequestResponse BifrostContextKey = "bifrost-store-raw-request-response" // bool (per-request override — read by bifrost.go, never overwritten) BifrostContextKeyCaptureRawRequest BifrostContextKey = "bifrost-capture-raw-request" // bool (set by bifrost - DO NOT SET THIS MANUALLY) — true when providers should capture raw request bytes BifrostContextKeyCaptureRawResponse BifrostContextKey = "bifrost-capture-raw-response" // bool (set by bifrost - DO NOT SET THIS MANUALLY) — true when providers should capture raw response bytes BifrostContextKeyDropRawRequestFromClient BifrostContextKey = "bifrost-drop-raw-request-from-client" // bool (set by bifrost - DO NOT SET THIS MANUALLY) — true when raw request should be stripped from the client-facing response BifrostContextKeyDropRawResponseFromClient BifrostContextKey = "bifrost-drop-raw-response-from-client" // bool (set by bifrost - DO NOT SET THIS MANUALLY) — true when raw response should be stripped from the client-facing response BifrostContextKeyShouldStoreRawInLogs BifrostContextKey = "bifrost-should-store-raw-in-logs" // bool (set by bifrost - DO NOT SET THIS MANUALLY) — true when raw request/response should be persisted in log records BifrostContextKeyRetryDBFetch BifrostContextKey = "bifrost-retry-db-fetch" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyIsCustomProvider BifrostContextKey = "bifrost-is-custom-provider" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyHTTPRequestType BifrostContextKey = "bifrost-http-request-type" // RequestType (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyPassthroughExtraParams BifrostContextKey = "bifrost-passthrough-extra-params" // bool BifrostContextKeyRoutingEnginesUsed BifrostContextKey = "bifrost-routing-engines-used" // []string (set by bifrost - DO NOT SET THIS MANUALLY) - list of routing engines used ("routing-rule", "governance", "loadbalancing", etc.) BifrostContextKeyRoutingEngineLogs BifrostContextKey = "bifrost-routing-engine-logs" // []RoutingEngineLogEntry (set by bifrost - DO NOT SET THIS MANUALLY) - list of routing engine log entries BifrostContextKeyTransportPluginLogs BifrostContextKey = "bifrost-transport-plugin-logs" // []PluginLogEntry (transport-layer plugin logs accumulated during HTTP transport hooks) BifrostContextKeyTransportPostHookCompleter BifrostContextKey = "bifrost-transport-posthook-completer" // func() (callback to run HTTPTransportPostHook after streaming - set by transport interceptor middleware) BifrostContextKeySkipPluginPipeline BifrostContextKey = "bifrost-skip-plugin-pipeline" // bool - skip plugin pipeline for the request BifrostContextKeyParentRequestID BifrostContextKey = "bifrost-parent-request-id" // string (parent linkage for grouped request logs like realtime turns) BifrostContextKeyRealtimeSessionID BifrostContextKey = "bifrost-realtime-session-id" // string BifrostContextKeyRealtimeProviderSessionID BifrostContextKey = "bifrost-realtime-provider-session-id" // string BifrostContextKeyRealtimeSource BifrostContextKey = "bifrost-realtime-source" // string ("ei" or "lm") BifrostContextKeyRealtimeEventType BifrostContextKey = "bifrost-realtime-event-type" // string BifrostIsAsyncRequest BifrostContextKey = "bifrost-is-async-request" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) - whether the request is an async request (only used in gateway) BifrostContextKeyRequestHeaders BifrostContextKey = "bifrost-request-headers" // map[string]string (all request headers with lowercased keys) BifrostContextKeySkipListModelsGovernanceFiltering BifrostContextKey = "bifrost-skip-list-models-governance-filtering" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeySCIMClaims BifrostContextKey = "scim_claims" BifrostContextKeyUserID BifrostContextKey = "bifrost-user-id" // string (to store the user ID (set by enterprise auth middleware - DO NOT SET THIS MANUALLY)) BifrostContextKeyUserName BifrostContextKey = "bifrost-user-name" // string (to store the user name (set by enterprise auth middleware - DO NOT SET THIS MANUALLY)) BifrostContextKeyTargetUserID BifrostContextKey = "target_user_id" BifrostContextKeyIsAzureUserAgent BifrostContextKey = "bifrost-is-azure-user-agent" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) - whether the request is an Azure user agent (only used in gateway) BifrostContextKeyVideoOutputRequested BifrostContextKey = "bifrost-video-output-requested" BifrostContextKeyValidateKeys BifrostContextKey = "bifrost-validate-keys" // bool (triggers additional key validation during provider add/update) BifrostContextKeyProviderResponseHeaders BifrostContextKey = "bifrost-provider-response-headers" // map[string]string (set by provider handlers for response header forwarding) BifrostContextKeyMCPAddedTools BifrostContextKey = "bifrost-mcp-added-tools" // []string (set by bifrost - DO NOT SET THIS MANUALLY)) - list of tools added to the request by MCP, all the tool are in the format "clientName-toolName" BifrostContextKeyLargePayloadMode BifrostContextKey = "bifrost-large-payload-mode" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) indicates large payload streaming mode is active BifrostContextKeyLargePayloadReader BifrostContextKey = "bifrost-large-payload-reader" // io.Reader (set by bifrost - DO NOT SET THIS MANUALLY)) upstream reader for large payloads BifrostContextKeyLargePayloadContentLength BifrostContextKey = "bifrost-large-payload-content-length" // int (set by bifrost - DO NOT SET THIS MANUALLY)) content length for large payloads BifrostContextKeyLargePayloadContentType BifrostContextKey = "bifrost-large-payload-content-type" // string (set by enterprise - DO NOT SET THIS MANUALLY)) original content type for large payload passthrough BifrostContextKeyLargePayloadMetadata BifrostContextKey = "bifrost-large-payload-metadata" // *LargePayloadMetadata (set by bifrost - DO NOT SET THIS MANUALLY)) routing metadata for large payloads BifrostContextKeyLargePayloadRequestThreshold BifrostContextKey = "bifrost-large-payload-request-threshold" // int64 (set by enterprise - DO NOT SET THIS MANUALLY)) request threshold used by transport heuristics BifrostContextKeyLargeResponseMode BifrostContextKey = "bifrost-large-response-mode" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) indicates large response streaming mode is active BifrostContextKeyLargePayloadRequestPreview BifrostContextKey = "bifrost-large-payload-request-preview" // string (set by bifrost - DO NOT SET THIS MANUALLY)) truncated request body preview for logging BifrostContextKeyLargePayloadResponsePreview BifrostContextKey = "bifrost-large-payload-response-preview" // string (set by bifrost - DO NOT SET THIS MANUALLY)) truncated response body preview for logging BifrostContextKeyLargeResponseReader BifrostContextKey = "bifrost-large-response-reader" // io.ReadCloser (set by bifrost - DO NOT SET THIS MANUALLY)) upstream reader for large responses BifrostContextKeyLargeResponseContentLength BifrostContextKey = "bifrost-large-response-content-length" // int (set by bifrost - DO NOT SET THIS MANUALLY)) content length for large responses BifrostContextKeyLargeResponseContentType BifrostContextKey = "bifrost-large-response-content-type" // string (set by bifrost - DO NOT SET THIS MANUALLY)) upstream content type for large responses BifrostContextKeyLargeResponseContentDisposition BifrostContextKey = "bifrost-large-response-content-disposition" // string (set by bifrost - DO NOT SET THIS MANUALLY)) downstream content disposition for large responses BifrostContextKeyLargeResponseThreshold BifrostContextKey = "bifrost-large-response-threshold" // int64 (set by enterprise - DO NOT SET THIS MANUALLY)) threshold for response streaming BifrostContextKeyLargePayloadPrefetchSize BifrostContextKey = "bifrost-large-payload-prefetch-size" // int (set by enterprise - DO NOT SET THIS MANUALLY)) prefetch buffer size for metadata extraction from large responses BifrostContextKeyDeferredUsage BifrostContextKey = "bifrost-deferred-usage" // chan *BifrostLLMUsage (set by provider Phase B — delivers usage after response streaming completes) BifrostContextKeyDeferredLargePayloadMetadata BifrostContextKey = "bifrost-deferred-large-payload-metadata" // <-chan *LargePayloadMetadata (set by enterprise Phase B request — delivers metadata after body streaming) BifrostContextKeySSEReaderFactory BifrostContextKey = "bifrost-sse-reader-factory" // *providerUtils.SSEReaderFactory (set by enterprise — replaces default bufio.Scanner SSE readers with streaming readers) BifrostContextKeySessionID BifrostContextKey = "bifrost-session-id" // string session ID for the request (session stickiness) BifrostContextKeySessionTTL BifrostContextKey = "bifrost-session-ttl" // time.Duration session TTL for the request (session stickiness) BifrostContextKeyMCPExtraHeaders BifrostContextKey = "bifrost-mcp-extra-headers" // map[string][]string (these headers are forwarded only to the MCP while tool execution if they are in the allowlist of the MCP client) BifrostContextKeyMCPLogID BifrostContextKey = "bifrost-mcp-log-id" // string (unique UUID for each MCP tool log entry - set per goroutine by agent executor - DO NOT SET THIS MANUALLY) BifrostContextKeyCompatConvertTextToChat BifrostContextKey = "bifrost-compat-convert-text-to-chat" // bool (per-request override from x-bf-compat header) BifrostContextKeyCompatConvertChatToResponses BifrostContextKey = "bifrost-compat-convert-chat-to-responses" // bool (per-request override from x-bf-compat header) BifrostContextKeyCompatShouldDropParams BifrostContextKey = "bifrost-compat-should-drop-params" // bool (per-request override from x-bf-compat header) BifrostContextKeyCompatShouldConvertParams BifrostContextKey = "bifrost-compat-should-convert-params" // bool (per-request override from x-bf-compat header) BifrostContextKeyAttemptTrail BifrostContextKey = "bifrost-attempt-trail" // []KeyAttemptRecord (set by bifrost - DO NOT SET THIS MANUALLY) - per-attempt key selection history ) const ( // DefaultLargePayloadRequestThresholdBytes is the default request-size heuristic // used by transport guards when no enterprise threshold is present on context. DefaultLargePayloadRequestThresholdBytes = 10 * 1024 * 1024 // 10MB ) // RoutingEngine constants const ( RoutingEngineGovernance = "governance" RoutingEngineRoutingRule = "routing-rule" RoutingEngineLoadbalancing = "loadbalancing" ) // KeyAttemptRecord captures the outcome of a single request attempt within executeRequestWithRetries. // One record is appended per attempt regardless of whether the key changed between attempts. // FailReason is supplementary retry metadata: it is populated only when another retry will be // attempted (i.e. a non-terminal attempt), and is nil on any terminal attempt — including success, // non-retryable failure, or a retryable error when no retries remain. type KeyAttemptRecord struct { Attempt int `json:"attempt"` KeyID string `json:"key_id"` KeyName string `json:"key_name"` FailReason *string `json:"fail_reason,omitempty"` } // RoutingEngineLogEntry represents a log entry from a routing engine // format: [timestamp] [engine] - message type RoutingEngineLogEntry struct { Engine string // e.g., "governance", "routing-rule", "openrouter" Message string // Human-readable decision/action message Timestamp int64 // Unix milliseconds } // PluginLogEntry represents a structured log entry emitted by a plugin via ctx.Log(). type PluginLogEntry struct { PluginName string `json:"plugin_name"` Level LogLevel `json:"level"` Message string `json:"message"` Timestamp int64 `json:"timestamp"` // Unix milliseconds } // GroupPluginLogsByName groups a flat slice of plugin log entries by plugin name. // Returns nil if the input is empty. func GroupPluginLogsByName(logs []PluginLogEntry) map[string][]PluginLogEntry { if len(logs) == 0 { return nil } grouped := make(map[string][]PluginLogEntry, min(len(logs), 4)) for _, entry := range logs { grouped[entry.PluginName] = append(grouped[entry.PluginName], entry) } return grouped } // NOTE: for custom plugin implementation dealing with streaming short circuit, // make sure to mark BifrostContextKeyStreamEndIndicator as true at the end of the stream. // LargePayloadMetadata holds routing-relevant metadata selectively extracted from large payloads. // This is used when the full request body is too large to parse (e.g., 400MB video upload). // Only small routing/observability fields are extracted; the body itself streams through unchanged. type LargePayloadMetadata struct { ResponseModalities []string // e.g., ["AUDIO"] for speech, ["IMAGE"] for image generation SpeechConfig bool // true if generationConfig.speechConfig is present Model string // model extracted without full body parsing (openai/anthropic multipart/json) StreamRequested *bool // stream flag when available in request payload metadata } //* Request Structs // Fallback represents a fallback model to be used if the primary model is not available. type Fallback struct { Provider ModelProvider `json:"provider"` Model string `json:"model"` } // BifrostRequest is the request struct for all bifrost requests. // only ONE of the following fields should be set: // - ListModelsRequest // - TextCompletionRequest // - ChatRequest // - ResponsesRequest // - CountTokensRequest // - EmbeddingRequest // - RerankRequest // - SpeechRequest // - TranscriptionRequest // - ImageGenerationRequest // NOTE: Bifrost Request is submitted back to pool after every use so DO NOT keep references to this struct after use, especially in go routines. type BifrostRequest struct { RequestType RequestType ListModelsRequest *BifrostListModelsRequest TextCompletionRequest *BifrostTextCompletionRequest ChatRequest *BifrostChatRequest ResponsesRequest *BifrostResponsesRequest CountTokensRequest *BifrostResponsesRequest EmbeddingRequest *BifrostEmbeddingRequest RerankRequest *BifrostRerankRequest OCRRequest *BifrostOCRRequest SpeechRequest *BifrostSpeechRequest TranscriptionRequest *BifrostTranscriptionRequest ImageGenerationRequest *BifrostImageGenerationRequest ImageEditRequest *BifrostImageEditRequest ImageVariationRequest *BifrostImageVariationRequest VideoGenerationRequest *BifrostVideoGenerationRequest VideoRetrieveRequest *BifrostVideoRetrieveRequest VideoDownloadRequest *BifrostVideoDownloadRequest VideoListRequest *BifrostVideoListRequest VideoRemixRequest *BifrostVideoRemixRequest VideoDeleteRequest *BifrostVideoDeleteRequest FileUploadRequest *BifrostFileUploadRequest FileListRequest *BifrostFileListRequest FileRetrieveRequest *BifrostFileRetrieveRequest FileDeleteRequest *BifrostFileDeleteRequest FileContentRequest *BifrostFileContentRequest BatchCreateRequest *BifrostBatchCreateRequest BatchListRequest *BifrostBatchListRequest BatchRetrieveRequest *BifrostBatchRetrieveRequest BatchCancelRequest *BifrostBatchCancelRequest BatchResultsRequest *BifrostBatchResultsRequest BatchDeleteRequest *BifrostBatchDeleteRequest ContainerCreateRequest *BifrostContainerCreateRequest ContainerListRequest *BifrostContainerListRequest ContainerRetrieveRequest *BifrostContainerRetrieveRequest ContainerDeleteRequest *BifrostContainerDeleteRequest ContainerFileCreateRequest *BifrostContainerFileCreateRequest ContainerFileListRequest *BifrostContainerFileListRequest ContainerFileRetrieveRequest *BifrostContainerFileRetrieveRequest ContainerFileContentRequest *BifrostContainerFileContentRequest ContainerFileDeleteRequest *BifrostContainerFileDeleteRequest PassthroughRequest *BifrostPassthroughRequest } // GetRequestFields returns the provider, model, and fallbacks from the request. func (br *BifrostRequest) GetRequestFields() (provider ModelProvider, model string, fallbacks []Fallback) { switch { case br.ListModelsRequest != nil: return br.ListModelsRequest.Provider, "", nil case br.TextCompletionRequest != nil: return br.TextCompletionRequest.Provider, br.TextCompletionRequest.Model, br.TextCompletionRequest.Fallbacks case br.ChatRequest != nil: return br.ChatRequest.Provider, br.ChatRequest.Model, br.ChatRequest.Fallbacks case br.ResponsesRequest != nil: return br.ResponsesRequest.Provider, br.ResponsesRequest.Model, br.ResponsesRequest.Fallbacks case br.CountTokensRequest != nil: return br.CountTokensRequest.Provider, br.CountTokensRequest.Model, br.CountTokensRequest.Fallbacks case br.EmbeddingRequest != nil: return br.EmbeddingRequest.Provider, br.EmbeddingRequest.Model, br.EmbeddingRequest.Fallbacks case br.RerankRequest != nil: return br.RerankRequest.Provider, br.RerankRequest.Model, br.RerankRequest.Fallbacks case br.OCRRequest != nil: return br.OCRRequest.Provider, br.OCRRequest.Model, br.OCRRequest.Fallbacks case br.SpeechRequest != nil: return br.SpeechRequest.Provider, br.SpeechRequest.Model, br.SpeechRequest.Fallbacks case br.TranscriptionRequest != nil: return br.TranscriptionRequest.Provider, br.TranscriptionRequest.Model, br.TranscriptionRequest.Fallbacks case br.ImageGenerationRequest != nil: return br.ImageGenerationRequest.Provider, br.ImageGenerationRequest.Model, br.ImageGenerationRequest.Fallbacks case br.ImageEditRequest != nil: return br.ImageEditRequest.Provider, br.ImageEditRequest.Model, br.ImageEditRequest.Fallbacks case br.ImageVariationRequest != nil: return br.ImageVariationRequest.Provider, br.ImageVariationRequest.Model, br.ImageVariationRequest.Fallbacks case br.VideoGenerationRequest != nil: return br.VideoGenerationRequest.Provider, br.VideoGenerationRequest.Model, br.VideoGenerationRequest.Fallbacks case br.VideoRetrieveRequest != nil: return br.VideoRetrieveRequest.Provider, "", nil case br.VideoDownloadRequest != nil: return br.VideoDownloadRequest.Provider, "", nil case br.VideoListRequest != nil: return br.VideoListRequest.Provider, "", nil case br.VideoDeleteRequest != nil: return br.VideoDeleteRequest.Provider, "", nil case br.VideoRemixRequest != nil: return br.VideoRemixRequest.Provider, "", nil case br.FileUploadRequest != nil: if br.FileUploadRequest.Model != nil { return br.FileUploadRequest.Provider, *br.FileUploadRequest.Model, nil } return br.FileUploadRequest.Provider, "", nil case br.FileListRequest != nil: if br.FileListRequest.Model != nil { return br.FileListRequest.Provider, *br.FileListRequest.Model, nil } return br.FileListRequest.Provider, "", nil case br.FileRetrieveRequest != nil: if br.FileRetrieveRequest.Model != nil { return br.FileRetrieveRequest.Provider, *br.FileRetrieveRequest.Model, nil } return br.FileRetrieveRequest.Provider, "", nil case br.FileDeleteRequest != nil: if br.FileDeleteRequest.Model != nil { return br.FileDeleteRequest.Provider, *br.FileDeleteRequest.Model, nil } return br.FileDeleteRequest.Provider, "", nil case br.FileContentRequest != nil: if br.FileContentRequest.Model != nil { return br.FileContentRequest.Provider, *br.FileContentRequest.Model, nil } return br.FileContentRequest.Provider, "", nil case br.BatchCreateRequest != nil: if br.BatchCreateRequest.Model != nil { return br.BatchCreateRequest.Provider, *br.BatchCreateRequest.Model, nil } return br.BatchCreateRequest.Provider, "", nil case br.BatchListRequest != nil: if br.BatchListRequest.Model != nil { return br.BatchListRequest.Provider, *br.BatchListRequest.Model, nil } return br.BatchListRequest.Provider, "", nil case br.BatchRetrieveRequest != nil: if br.BatchRetrieveRequest.Model != nil { return br.BatchRetrieveRequest.Provider, *br.BatchRetrieveRequest.Model, nil } return br.BatchRetrieveRequest.Provider, "", nil case br.BatchCancelRequest != nil: if br.BatchCancelRequest.Model != nil { return br.BatchCancelRequest.Provider, *br.BatchCancelRequest.Model, nil } return br.BatchCancelRequest.Provider, "", nil case br.BatchResultsRequest != nil: if br.BatchResultsRequest.Model != nil { return br.BatchResultsRequest.Provider, *br.BatchResultsRequest.Model, nil } return br.BatchResultsRequest.Provider, "", nil case br.BatchDeleteRequest != nil: if br.BatchDeleteRequest.Model != nil { return br.BatchDeleteRequest.Provider, *br.BatchDeleteRequest.Model, nil } return br.BatchDeleteRequest.Provider, "", nil case br.ContainerCreateRequest != nil: return br.ContainerCreateRequest.Provider, "", nil case br.ContainerListRequest != nil: return br.ContainerListRequest.Provider, "", nil case br.ContainerRetrieveRequest != nil: return br.ContainerRetrieveRequest.Provider, "", nil case br.ContainerDeleteRequest != nil: return br.ContainerDeleteRequest.Provider, "", nil case br.ContainerFileCreateRequest != nil: return br.ContainerFileCreateRequest.Provider, "", nil case br.ContainerFileListRequest != nil: return br.ContainerFileListRequest.Provider, "", nil case br.ContainerFileRetrieveRequest != nil: return br.ContainerFileRetrieveRequest.Provider, "", nil case br.ContainerFileContentRequest != nil: return br.ContainerFileContentRequest.Provider, "", nil case br.ContainerFileDeleteRequest != nil: return br.ContainerFileDeleteRequest.Provider, "", nil case br.PassthroughRequest != nil: return br.PassthroughRequest.Provider, br.PassthroughRequest.Model, nil } return "", "", nil } func (br *BifrostRequest) SetProvider(provider ModelProvider) { switch { case br.ListModelsRequest != nil: br.ListModelsRequest.Provider = provider case br.TextCompletionRequest != nil: br.TextCompletionRequest.Provider = provider case br.ChatRequest != nil: br.ChatRequest.Provider = provider case br.ResponsesRequest != nil: br.ResponsesRequest.Provider = provider case br.CountTokensRequest != nil: br.CountTokensRequest.Provider = provider case br.EmbeddingRequest != nil: br.EmbeddingRequest.Provider = provider case br.RerankRequest != nil: br.RerankRequest.Provider = provider case br.OCRRequest != nil: br.OCRRequest.Provider = provider case br.SpeechRequest != nil: br.SpeechRequest.Provider = provider case br.TranscriptionRequest != nil: br.TranscriptionRequest.Provider = provider case br.ImageGenerationRequest != nil: br.ImageGenerationRequest.Provider = provider case br.ImageEditRequest != nil: br.ImageEditRequest.Provider = provider case br.ImageVariationRequest != nil: br.ImageVariationRequest.Provider = provider case br.VideoGenerationRequest != nil: br.VideoGenerationRequest.Provider = provider case br.VideoRetrieveRequest != nil: br.VideoRetrieveRequest.Provider = provider case br.VideoDownloadRequest != nil: br.VideoDownloadRequest.Provider = provider case br.VideoListRequest != nil: br.VideoListRequest.Provider = provider case br.VideoDeleteRequest != nil: br.VideoDeleteRequest.Provider = provider case br.VideoRemixRequest != nil: br.VideoRemixRequest.Provider = provider } } func (br *BifrostRequest) SetModel(model string) { switch { case br.TextCompletionRequest != nil: br.TextCompletionRequest.Model = model case br.ChatRequest != nil: br.ChatRequest.Model = model case br.ResponsesRequest != nil: br.ResponsesRequest.Model = model case br.CountTokensRequest != nil: br.CountTokensRequest.Model = model case br.EmbeddingRequest != nil: br.EmbeddingRequest.Model = model case br.RerankRequest != nil: br.RerankRequest.Model = model case br.OCRRequest != nil: br.OCRRequest.Model = model case br.SpeechRequest != nil: br.SpeechRequest.Model = model case br.TranscriptionRequest != nil: br.TranscriptionRequest.Model = model case br.ImageGenerationRequest != nil: br.ImageGenerationRequest.Model = model case br.ImageEditRequest != nil: br.ImageEditRequest.Model = model case br.ImageVariationRequest != nil: br.ImageVariationRequest.Model = model case br.VideoGenerationRequest != nil: br.VideoGenerationRequest.Model = model case br.BatchCreateRequest != nil: if br.BatchCreateRequest.Model != nil { br.BatchCreateRequest.Model = new(model) } } } func (br *BifrostRequest) SetFallbacks(fallbacks []Fallback) { switch { case br.TextCompletionRequest != nil: br.TextCompletionRequest.Fallbacks = fallbacks case br.ChatRequest != nil: br.ChatRequest.Fallbacks = fallbacks case br.ResponsesRequest != nil: br.ResponsesRequest.Fallbacks = fallbacks case br.CountTokensRequest != nil: br.CountTokensRequest.Fallbacks = fallbacks case br.EmbeddingRequest != nil: br.EmbeddingRequest.Fallbacks = fallbacks case br.RerankRequest != nil: br.RerankRequest.Fallbacks = fallbacks case br.OCRRequest != nil: br.OCRRequest.Fallbacks = fallbacks case br.SpeechRequest != nil: br.SpeechRequest.Fallbacks = fallbacks case br.TranscriptionRequest != nil: br.TranscriptionRequest.Fallbacks = fallbacks case br.ImageGenerationRequest != nil: br.ImageGenerationRequest.Fallbacks = fallbacks case br.ImageEditRequest != nil: br.ImageEditRequest.Fallbacks = fallbacks case br.ImageVariationRequest != nil: br.ImageVariationRequest.Fallbacks = fallbacks case br.VideoGenerationRequest != nil: br.VideoGenerationRequest.Fallbacks = fallbacks } } func (br *BifrostRequest) SetRawRequestBody(rawRequestBody []byte) { switch { case br.TextCompletionRequest != nil: br.TextCompletionRequest.RawRequestBody = rawRequestBody case br.ChatRequest != nil: br.ChatRequest.RawRequestBody = rawRequestBody case br.ResponsesRequest != nil: br.ResponsesRequest.RawRequestBody = rawRequestBody case br.CountTokensRequest != nil: br.CountTokensRequest.RawRequestBody = rawRequestBody case br.EmbeddingRequest != nil: br.EmbeddingRequest.RawRequestBody = rawRequestBody case br.RerankRequest != nil: br.RerankRequest.RawRequestBody = rawRequestBody case br.OCRRequest != nil: br.OCRRequest.RawRequestBody = rawRequestBody case br.SpeechRequest != nil: br.SpeechRequest.RawRequestBody = rawRequestBody case br.TranscriptionRequest != nil: br.TranscriptionRequest.RawRequestBody = rawRequestBody case br.ImageGenerationRequest != nil: br.ImageGenerationRequest.RawRequestBody = rawRequestBody case br.ImageEditRequest != nil: br.ImageEditRequest.RawRequestBody = rawRequestBody case br.ImageVariationRequest != nil: br.ImageVariationRequest.RawRequestBody = rawRequestBody case br.VideoGenerationRequest != nil: br.VideoGenerationRequest.RawRequestBody = rawRequestBody case br.VideoRemixRequest != nil: br.VideoRemixRequest.RawRequestBody = rawRequestBody } } type MCPRequestType string const ( MCPRequestTypeChatToolCall MCPRequestType = "chat_tool_call" // Chat API format MCPRequestTypeResponsesToolCall MCPRequestType = "responses_tool_call" // Responses API format ) // BifrostMCPRequest is the request struct for all MCP requests. // only ONE of the following fields should be set: // - ChatAssistantMessageToolCall // - ResponsesToolMessage type BifrostMCPRequest struct { RequestType MCPRequestType *ChatAssistantMessageToolCall *ResponsesToolMessage } func (r *BifrostMCPRequest) GetToolName() string { if r.ChatAssistantMessageToolCall != nil { if r.ChatAssistantMessageToolCall.Function.Name != nil { return *r.ChatAssistantMessageToolCall.Function.Name } } if r.ResponsesToolMessage != nil { if r.ResponsesToolMessage.Name != nil { return *r.ResponsesToolMessage.Name } } return "" } func (r *BifrostMCPRequest) GetToolArguments() interface{} { if r.ChatAssistantMessageToolCall != nil { return r.ChatAssistantMessageToolCall.Function.Arguments } if r.ResponsesToolMessage != nil { return r.ResponsesToolMessage.Arguments } return nil } //* Response Structs // BifrostResponse represents the complete result from any bifrost request. type BifrostResponse struct { ListModelsResponse *BifrostListModelsResponse TextCompletionResponse *BifrostTextCompletionResponse ChatResponse *BifrostChatResponse ResponsesResponse *BifrostResponsesResponse ResponsesStreamResponse *BifrostResponsesStreamResponse CountTokensResponse *BifrostCountTokensResponse EmbeddingResponse *BifrostEmbeddingResponse RerankResponse *BifrostRerankResponse OCRResponse *BifrostOCRResponse SpeechResponse *BifrostSpeechResponse SpeechStreamResponse *BifrostSpeechStreamResponse TranscriptionResponse *BifrostTranscriptionResponse TranscriptionStreamResponse *BifrostTranscriptionStreamResponse ImageGenerationResponse *BifrostImageGenerationResponse ImageGenerationStreamResponse *BifrostImageGenerationStreamResponse VideoGenerationResponse *BifrostVideoGenerationResponse VideoDownloadResponse *BifrostVideoDownloadResponse VideoListResponse *BifrostVideoListResponse VideoDeleteResponse *BifrostVideoDeleteResponse FileUploadResponse *BifrostFileUploadResponse FileListResponse *BifrostFileListResponse FileRetrieveResponse *BifrostFileRetrieveResponse FileDeleteResponse *BifrostFileDeleteResponse FileContentResponse *BifrostFileContentResponse BatchCreateResponse *BifrostBatchCreateResponse BatchListResponse *BifrostBatchListResponse BatchRetrieveResponse *BifrostBatchRetrieveResponse BatchCancelResponse *BifrostBatchCancelResponse BatchResultsResponse *BifrostBatchResultsResponse BatchDeleteResponse *BifrostBatchDeleteResponse ContainerCreateResponse *BifrostContainerCreateResponse ContainerListResponse *BifrostContainerListResponse ContainerRetrieveResponse *BifrostContainerRetrieveResponse ContainerDeleteResponse *BifrostContainerDeleteResponse ContainerFileCreateResponse *BifrostContainerFileCreateResponse ContainerFileListResponse *BifrostContainerFileListResponse ContainerFileRetrieveResponse *BifrostContainerFileRetrieveResponse ContainerFileContentResponse *BifrostContainerFileContentResponse ContainerFileDeleteResponse *BifrostContainerFileDeleteResponse PassthroughResponse *BifrostPassthroughResponse } func (r *BifrostResponse) GetExtraFields() *BifrostResponseExtraFields { switch { case r.ListModelsResponse != nil: return &r.ListModelsResponse.ExtraFields case r.TextCompletionResponse != nil: return &r.TextCompletionResponse.ExtraFields case r.ChatResponse != nil: return &r.ChatResponse.ExtraFields case r.ResponsesResponse != nil: return &r.ResponsesResponse.ExtraFields case r.ResponsesStreamResponse != nil: return &r.ResponsesStreamResponse.ExtraFields case r.CountTokensResponse != nil: return &r.CountTokensResponse.ExtraFields case r.EmbeddingResponse != nil: return &r.EmbeddingResponse.ExtraFields case r.RerankResponse != nil: return &r.RerankResponse.ExtraFields case r.OCRResponse != nil: return &r.OCRResponse.ExtraFields case r.SpeechResponse != nil: return &r.SpeechResponse.ExtraFields case r.SpeechStreamResponse != nil: return &r.SpeechStreamResponse.ExtraFields case r.TranscriptionResponse != nil: return &r.TranscriptionResponse.ExtraFields case r.TranscriptionStreamResponse != nil: return &r.TranscriptionStreamResponse.ExtraFields case r.ImageGenerationResponse != nil: return &r.ImageGenerationResponse.ExtraFields case r.ImageGenerationStreamResponse != nil: return &r.ImageGenerationStreamResponse.ExtraFields case r.FileUploadResponse != nil: return &r.FileUploadResponse.ExtraFields case r.FileListResponse != nil: return &r.FileListResponse.ExtraFields case r.FileRetrieveResponse != nil: return &r.FileRetrieveResponse.ExtraFields case r.FileDeleteResponse != nil: return &r.FileDeleteResponse.ExtraFields case r.FileContentResponse != nil: return &r.FileContentResponse.ExtraFields case r.VideoGenerationResponse != nil: return &r.VideoGenerationResponse.ExtraFields case r.VideoDownloadResponse != nil: return &r.VideoDownloadResponse.ExtraFields case r.VideoListResponse != nil: return &r.VideoListResponse.ExtraFields case r.VideoDeleteResponse != nil: return &r.VideoDeleteResponse.ExtraFields case r.BatchCreateResponse != nil: return &r.BatchCreateResponse.ExtraFields case r.BatchListResponse != nil: return &r.BatchListResponse.ExtraFields case r.BatchRetrieveResponse != nil: return &r.BatchRetrieveResponse.ExtraFields case r.BatchCancelResponse != nil: return &r.BatchCancelResponse.ExtraFields case r.BatchDeleteResponse != nil: return &r.BatchDeleteResponse.ExtraFields case r.BatchResultsResponse != nil: return &r.BatchResultsResponse.ExtraFields case r.ContainerCreateResponse != nil: return &r.ContainerCreateResponse.ExtraFields case r.ContainerListResponse != nil: return &r.ContainerListResponse.ExtraFields case r.ContainerRetrieveResponse != nil: return &r.ContainerRetrieveResponse.ExtraFields case r.ContainerDeleteResponse != nil: return &r.ContainerDeleteResponse.ExtraFields case r.ContainerFileCreateResponse != nil: return &r.ContainerFileCreateResponse.ExtraFields case r.ContainerFileListResponse != nil: return &r.ContainerFileListResponse.ExtraFields case r.ContainerFileRetrieveResponse != nil: return &r.ContainerFileRetrieveResponse.ExtraFields case r.ContainerFileContentResponse != nil: return &r.ContainerFileContentResponse.ExtraFields case r.ContainerFileDeleteResponse != nil: return &r.ContainerFileDeleteResponse.ExtraFields case r.PassthroughResponse != nil: return &r.PassthroughResponse.ExtraFields } return &BifrostResponseExtraFields{} } // PopulateExtraFields sets RequestType, Provider, OriginalModelRequested, and ResolvedModelUsed on the // active sub-response. Core always calls this both before and after RunPostLLMHooks, so any plugin // modifications to these 4 fields are no-ops — tampering with them inside plugins is discouraged. func (r *BifrostResponse) PopulateExtraFields(requestType RequestType, provider ModelProvider, originalModelRequested string, resolvedModelUsed string) { if r == nil { return } resolvedModel := resolvedModelUsed if resolvedModel == "" { resolvedModel = originalModelRequested } switch { case r.ListModelsResponse != nil: r.ListModelsResponse.ExtraFields.RequestType = requestType r.ListModelsResponse.ExtraFields.Provider = provider r.ListModelsResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ListModelsResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.TextCompletionResponse != nil: r.TextCompletionResponse.ExtraFields.RequestType = requestType r.TextCompletionResponse.ExtraFields.Provider = provider r.TextCompletionResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.TextCompletionResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ChatResponse != nil: r.ChatResponse.ExtraFields.RequestType = requestType r.ChatResponse.ExtraFields.Provider = provider r.ChatResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ChatResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ResponsesResponse != nil: r.ResponsesResponse.ExtraFields.RequestType = requestType r.ResponsesResponse.ExtraFields.Provider = provider r.ResponsesResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ResponsesResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ResponsesStreamResponse != nil: r.ResponsesStreamResponse.ExtraFields.RequestType = requestType r.ResponsesStreamResponse.ExtraFields.Provider = provider r.ResponsesStreamResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ResponsesStreamResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.CountTokensResponse != nil: r.CountTokensResponse.ExtraFields.RequestType = requestType r.CountTokensResponse.ExtraFields.Provider = provider r.CountTokensResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.CountTokensResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.EmbeddingResponse != nil: r.EmbeddingResponse.ExtraFields.RequestType = requestType r.EmbeddingResponse.ExtraFields.Provider = provider r.EmbeddingResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.EmbeddingResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.RerankResponse != nil: r.RerankResponse.ExtraFields.RequestType = requestType r.RerankResponse.ExtraFields.Provider = provider r.RerankResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.RerankResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.SpeechResponse != nil: r.SpeechResponse.ExtraFields.RequestType = requestType r.SpeechResponse.ExtraFields.Provider = provider r.SpeechResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.SpeechResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.SpeechStreamResponse != nil: r.SpeechStreamResponse.ExtraFields.RequestType = requestType r.SpeechStreamResponse.ExtraFields.Provider = provider r.SpeechStreamResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.SpeechStreamResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.TranscriptionResponse != nil: r.TranscriptionResponse.ExtraFields.RequestType = requestType r.TranscriptionResponse.ExtraFields.Provider = provider r.TranscriptionResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.TranscriptionResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.TranscriptionStreamResponse != nil: r.TranscriptionStreamResponse.ExtraFields.RequestType = requestType r.TranscriptionStreamResponse.ExtraFields.Provider = provider r.TranscriptionStreamResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.TranscriptionStreamResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ImageGenerationResponse != nil: r.ImageGenerationResponse.ExtraFields.RequestType = requestType r.ImageGenerationResponse.ExtraFields.Provider = provider r.ImageGenerationResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ImageGenerationResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ImageGenerationStreamResponse != nil: r.ImageGenerationStreamResponse.ExtraFields.RequestType = requestType r.ImageGenerationStreamResponse.ExtraFields.Provider = provider r.ImageGenerationStreamResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ImageGenerationStreamResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.VideoGenerationResponse != nil: r.VideoGenerationResponse.ExtraFields.RequestType = requestType r.VideoGenerationResponse.ExtraFields.Provider = provider r.VideoGenerationResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.VideoGenerationResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.VideoDownloadResponse != nil: r.VideoDownloadResponse.ExtraFields.RequestType = requestType r.VideoDownloadResponse.ExtraFields.Provider = provider r.VideoDownloadResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.VideoDownloadResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.VideoListResponse != nil: r.VideoListResponse.ExtraFields.RequestType = requestType r.VideoListResponse.ExtraFields.Provider = provider r.VideoListResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.VideoListResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.VideoDeleteResponse != nil: r.VideoDeleteResponse.ExtraFields.RequestType = requestType r.VideoDeleteResponse.ExtraFields.Provider = provider r.VideoDeleteResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.VideoDeleteResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.FileUploadResponse != nil: r.FileUploadResponse.ExtraFields.RequestType = requestType r.FileUploadResponse.ExtraFields.Provider = provider r.FileUploadResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.FileUploadResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.FileListResponse != nil: r.FileListResponse.ExtraFields.RequestType = requestType r.FileListResponse.ExtraFields.Provider = provider r.FileListResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.FileListResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.FileRetrieveResponse != nil: r.FileRetrieveResponse.ExtraFields.RequestType = requestType r.FileRetrieveResponse.ExtraFields.Provider = provider r.FileRetrieveResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.FileRetrieveResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.FileDeleteResponse != nil: r.FileDeleteResponse.ExtraFields.RequestType = requestType r.FileDeleteResponse.ExtraFields.Provider = provider r.FileDeleteResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.FileDeleteResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.FileContentResponse != nil: r.FileContentResponse.ExtraFields.RequestType = requestType r.FileContentResponse.ExtraFields.Provider = provider r.FileContentResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.FileContentResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.BatchCreateResponse != nil: r.BatchCreateResponse.ExtraFields.RequestType = requestType r.BatchCreateResponse.ExtraFields.Provider = provider r.BatchCreateResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.BatchCreateResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.BatchListResponse != nil: r.BatchListResponse.ExtraFields.RequestType = requestType r.BatchListResponse.ExtraFields.Provider = provider r.BatchListResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.BatchListResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.BatchRetrieveResponse != nil: r.BatchRetrieveResponse.ExtraFields.RequestType = requestType r.BatchRetrieveResponse.ExtraFields.Provider = provider r.BatchRetrieveResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.BatchRetrieveResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.BatchCancelResponse != nil: r.BatchCancelResponse.ExtraFields.RequestType = requestType r.BatchCancelResponse.ExtraFields.Provider = provider r.BatchCancelResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.BatchCancelResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.BatchDeleteResponse != nil: r.BatchDeleteResponse.ExtraFields.RequestType = requestType r.BatchDeleteResponse.ExtraFields.Provider = provider r.BatchDeleteResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.BatchDeleteResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.BatchResultsResponse != nil: r.BatchResultsResponse.ExtraFields.RequestType = requestType r.BatchResultsResponse.ExtraFields.Provider = provider r.BatchResultsResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.BatchResultsResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerCreateResponse != nil: r.ContainerCreateResponse.ExtraFields.RequestType = requestType r.ContainerCreateResponse.ExtraFields.Provider = provider r.ContainerCreateResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerCreateResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerListResponse != nil: r.ContainerListResponse.ExtraFields.RequestType = requestType r.ContainerListResponse.ExtraFields.Provider = provider r.ContainerListResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerListResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerRetrieveResponse != nil: r.ContainerRetrieveResponse.ExtraFields.RequestType = requestType r.ContainerRetrieveResponse.ExtraFields.Provider = provider r.ContainerRetrieveResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerRetrieveResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerDeleteResponse != nil: r.ContainerDeleteResponse.ExtraFields.RequestType = requestType r.ContainerDeleteResponse.ExtraFields.Provider = provider r.ContainerDeleteResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerDeleteResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerFileCreateResponse != nil: r.ContainerFileCreateResponse.ExtraFields.RequestType = requestType r.ContainerFileCreateResponse.ExtraFields.Provider = provider r.ContainerFileCreateResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerFileCreateResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerFileListResponse != nil: r.ContainerFileListResponse.ExtraFields.RequestType = requestType r.ContainerFileListResponse.ExtraFields.Provider = provider r.ContainerFileListResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerFileListResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerFileRetrieveResponse != nil: r.ContainerFileRetrieveResponse.ExtraFields.RequestType = requestType r.ContainerFileRetrieveResponse.ExtraFields.Provider = provider r.ContainerFileRetrieveResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerFileRetrieveResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerFileContentResponse != nil: r.ContainerFileContentResponse.ExtraFields.RequestType = requestType r.ContainerFileContentResponse.ExtraFields.Provider = provider r.ContainerFileContentResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerFileContentResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.ContainerFileDeleteResponse != nil: r.ContainerFileDeleteResponse.ExtraFields.RequestType = requestType r.ContainerFileDeleteResponse.ExtraFields.Provider = provider r.ContainerFileDeleteResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.ContainerFileDeleteResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.OCRResponse != nil: r.OCRResponse.ExtraFields.RequestType = requestType r.OCRResponse.ExtraFields.Provider = provider r.OCRResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.OCRResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.PassthroughResponse != nil: r.PassthroughResponse.ExtraFields.RequestType = requestType r.PassthroughResponse.ExtraFields.Provider = provider r.PassthroughResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.PassthroughResponse.ExtraFields.ResolvedModelUsed = resolvedModel } } // BifrostMCPResponse is the response struct for all MCP responses. // only ONE of the following fields should be set: // - ChatMessage // - ResponsesMessage type BifrostMCPResponse struct { ChatMessage *ChatMessage ResponsesMessage *ResponsesMessage ExtraFields BifrostMCPResponseExtraFields } // BifrostResponseExtraFields contains additional fields in a response. type BifrostResponseExtraFields struct { RequestType RequestType `json:"request_type"` Provider ModelProvider `json:"provider,omitempty"` OriginalModelRequested string `json:"original_model_requested,omitempty"` // the model alias the caller sent in the request ResolvedModelUsed string `json:"resolved_model_used,omitempty"` // the actual provider API identifier used (equals OriginalModelRequested when no alias mapping exists) Latency int64 `json:"latency"` // in milliseconds (for streaming responses this will be each chunk latency, and the last chunk latency will be the total latency) ChunkIndex int `json:"chunk_index"` // used for streaming responses to identify the chunk index, will be 0 for non-streaming responses RawRequest interface{} `json:"raw_request,omitempty"` RawResponse interface{} `json:"raw_response,omitempty"` CacheDebug *BifrostCacheDebug `json:"cache_debug,omitempty"` ParseErrors []BatchError `json:"parse_errors,omitempty"` // errors encountered while parsing JSONL batch results ConvertedRequestType RequestType `json:"converted_request_type,omitempty"` DroppedCompatPluginParams []string `json:"dropped_compat_plugin_params,omitempty"` // params dropped by the compat plugin based on model catalog ProviderResponseHeaders map[string]string `json:"provider_response_headers,omitempty"` // HTTP response headers from the provider (filtered to exclude transport-level headers) } type BifrostMCPResponseExtraFields struct { ClientName string `json:"client_name"` ToolName string `json:"tool_name"` Latency int64 `json:"latency"` // in milliseconds } // BifrostCacheDebug represents debug information about the cache. type BifrostCacheDebug struct { CacheHit bool `json:"cache_hit"` CacheID *string `json:"cache_id,omitempty"` HitType *string `json:"hit_type,omitempty"` RequestedProvider *string `json:"requested_provider,omitempty"` RequestedModel *string `json:"requested_model,omitempty"` // Semantic cache only (provider, model, and input tokens will be present for semantic cache, even if cache is not hit) ProviderUsed *string `json:"provider_used,omitempty"` ModelUsed *string `json:"model_used,omitempty"` InputTokens *int `json:"input_tokens,omitempty"` // Semantic cache only (only when cache is hit) Threshold *float64 `json:"threshold,omitempty"` Similarity *float64 `json:"similarity,omitempty"` } const ( RequestCancelled = "request_cancelled" RequestTimedOut = "request_timed_out" ) // BifrostStreamChunk represents a stream of responses from the Bifrost system. // Either BifrostResponse or BifrostError will be non-nil. type BifrostStreamChunk struct { *BifrostTextCompletionResponse *BifrostChatResponse *BifrostResponsesStreamResponse *BifrostSpeechStreamResponse *BifrostTranscriptionStreamResponse *BifrostImageGenerationStreamResponse *BifrostPassthroughResponse *BifrostError } // MarshalJSON implements custom JSON marshaling for BifrostStreamChunk. // This ensures that only the non-nil embedded struct is marshaled, func (bs BifrostStreamChunk) MarshalJSON() ([]byte, error) { if bs.BifrostTextCompletionResponse != nil { return MarshalSorted(bs.BifrostTextCompletionResponse) } else if bs.BifrostChatResponse != nil { return MarshalSorted(bs.BifrostChatResponse) } else if bs.BifrostResponsesStreamResponse != nil { return MarshalSorted(bs.BifrostResponsesStreamResponse) } else if bs.BifrostSpeechStreamResponse != nil { return MarshalSorted(bs.BifrostSpeechStreamResponse) } else if bs.BifrostTranscriptionStreamResponse != nil { return MarshalSorted(bs.BifrostTranscriptionStreamResponse) } else if bs.BifrostImageGenerationStreamResponse != nil { return MarshalSorted(bs.BifrostImageGenerationStreamResponse) } else if bs.BifrostPassthroughResponse != nil { return MarshalSorted(bs.BifrostPassthroughResponse) } else if bs.BifrostError != nil { return MarshalSorted(bs.BifrostError) } // Return empty object if both are nil (shouldn't happen in practice) return []byte("{}"), nil } // BifrostError represents an error from the Bifrost system. // // PLUGIN DEVELOPERS: When creating BifrostError in PreLLMHook or PostLLMHook, you can set AllowFallbacks: // - AllowFallbacks = &true: Bifrost will try fallback providers if available // - AllowFallbacks = &false: Bifrost will return this error immediately, no fallbacks // - AllowFallbacks = nil: Treated as true by default (fallbacks allowed for resilience) type BifrostError struct { EventID *string `json:"event_id,omitempty"` Type *string `json:"type,omitempty"` IsBifrostError bool `json:"is_bifrost_error"` StatusCode *int `json:"status_code,omitempty"` Error *ErrorField `json:"error"` AllowFallbacks *bool `json:"-"` // Optional: Controls fallback behavior (nil = true by default) StreamControl *StreamControl `json:"-"` // Optional: Controls stream behavior ExtraFields BifrostErrorExtraFields `json:"extra_fields"` } // PopulateExtraFields sets RequestType, Provider, OriginalModelRequested, and ResolvedModelUsed on the // error's ExtraFields. Core always calls this both before and after RunPostLLMHooks, so any plugin // modifications to these 4 fields are no-ops — tampering with them inside plugins is discouraged. func (e *BifrostError) PopulateExtraFields(requestType RequestType, provider ModelProvider, originalModelRequested string, resolvedModelUsed string) { if e == nil { return } e.ExtraFields.RequestType = requestType e.ExtraFields.Provider = provider e.ExtraFields.OriginalModelRequested = originalModelRequested if resolvedModelUsed != "" { e.ExtraFields.ResolvedModelUsed = resolvedModelUsed } else { e.ExtraFields.ResolvedModelUsed = originalModelRequested } } // StreamControl represents stream control options. type StreamControl struct { LogError *bool `json:"log_error,omitempty"` // Optional: Controls logging of error SkipStream *bool `json:"skip_stream,omitempty"` // Optional: Controls skipping of stream chunk } // ErrorField represents detailed error information. type ErrorField struct { Type *string `json:"type,omitempty"` Code *string `json:"code,omitempty"` Message string `json:"message"` Error error `json:"-"` Param interface{} `json:"param,omitempty"` EventID *string `json:"event_id,omitempty"` } // MarshalJSON implements custom JSON marshaling for ErrorField. // It converts the Error field (error interface) to a string. func (e *ErrorField) MarshalJSON() ([]byte, error) { type Alias ErrorField aux := &struct { Error *string `json:"error,omitempty"` *Alias }{ Alias: (*Alias)(e), } if e.Error != nil { errStr := e.Error.Error() aux.Error = &errStr } return json.Marshal(aux) } func (e *ErrorField) UnmarshalJSON(data []byte) error { aux := &struct { Type *string `json:"type,omitempty"` Code interface{} `json:"code,omitempty"` Message string `json:"message"` Error *string `json:"error,omitempty"` Param interface{} `json:"param,omitempty"` EventID *string `json:"event_id,omitempty"` }{} if err := json.Unmarshal(data, aux); err != nil { return err } e.Type = aux.Type e.Message = aux.Message e.Param = aux.Param e.EventID = aux.EventID if aux.Error != nil { e.Error = errors.New(*aux.Error) } if aux.Code != nil { switch v := aux.Code.(type) { case string: e.Code = &v case float64: s := strconv.FormatInt(int64(v), 10) e.Code = &s default: s := fmt.Sprint(aux.Code) e.Code = &s } } return nil } // BifrostErrorExtraFields contains additional fields in an error response. type BifrostErrorExtraFields struct { Provider ModelProvider `json:"provider,omitempty"` OriginalModelRequested string `json:"original_model_requested,omitempty"` ResolvedModelUsed string `json:"resolved_model_used,omitempty"` RequestType RequestType `json:"request_type,omitempty"` RawRequest interface{} `json:"raw_request,omitempty"` RawResponse interface{} `json:"raw_response,omitempty"` ConvertedRequestType RequestType `json:"converted_request_type,omitempty"` DroppedCompatPluginParams []string `json:"dropped_compat_plugin_params,omitempty"` KeyStatuses []KeyStatus `json:"key_statuses,omitempty"` MCPAuthRequired *MCPUserOAuthRequiredError `json:"mcp_auth_required,omitempty"` // Set when a per-user OAuth MCP tool requires authentication }