first commit

framework/logstore/asyncjob.go

@@ -0,0 +1,318 @@
+package logstore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/bytedance/sonic"
+	"github.com/google/uuid"
+	bifrost "github.com/maximhq/bifrost/core"
+	"github.com/maximhq/bifrost/core/schemas"
+	configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables"
+	"github.com/valyala/fasthttp"
+)
+
+const (
+	// DefaultAsyncJobResultTTL is the default TTL for async job results in seconds (1 hour).
+	DefaultAsyncJobResultTTL = 3600
+)
+
+const (
+	asyncJobCleanupInterval      = 1 * time.Minute
+	asyncJobCleanupTimeout       = 1 * time.Minute
+	asyncJobStaleProcessingHours = 24
+)
+
+// --- AsyncJobExecutor ---
+
+// AsyncOperation represents a function that can be executed asynchronously.
+// It returns the response and an optional BifrostError.
+type AsyncOperation func(ctx *schemas.BifrostContext) (any, *schemas.BifrostError)
+
+// GovernanceStore is an interface that provides access to the governance store.
+type GovernanceStore interface {
+	GetVirtualKey(ctx context.Context, vkValue string) (*configstoreTables.TableVirtualKey, bool)
+}
+
+// AsyncJobExecutor manages async job creation and background execution.
+type AsyncJobExecutor struct {
+	logstore        LogStore
+	governanceStore GovernanceStore
+	logger          schemas.Logger
+}
+
+// NewAsyncJobExecutor creates a new AsyncJobExecutor.
+func NewAsyncJobExecutor(logstore LogStore, governanceStore GovernanceStore, logger schemas.Logger) *AsyncJobExecutor {
+	return &AsyncJobExecutor{
+		logstore:        logstore,
+		governanceStore: governanceStore,
+		logger:          logger,
+	}
+}
+
+// RetrieveJob retrieves a job by its ID.
+func (e *AsyncJobExecutor) RetrieveJob(ctx context.Context, jobID string, vkValue *string, operationType schemas.RequestType) (*AsyncJob, error) {
+	job, err := e.logstore.FindAsyncJobByID(ctx, jobID)
+	if err != nil {
+		if errors.Is(err, ErrNotFound) {
+			return nil, fmt.Errorf("job not found or expired")
+		}
+		return nil, fmt.Errorf("%w: %w", ErrJobInternal, err)
+	}
+	if job.VirtualKeyID != nil {
+		if vkValue == nil {
+			return nil, fmt.Errorf("virtual key is required")
+		}
+		vk, ok := e.governanceStore.GetVirtualKey(ctx, *vkValue)
+		if !ok {
+			return nil, fmt.Errorf("virtual key not found")
+		}
+		if *job.VirtualKeyID != vk.ID {
+			return nil, fmt.Errorf("virtual key mismatch")
+		}
+	}
+	if job.RequestType != operationType {
+		return nil, fmt.Errorf("operation type mismatch")
+	}
+	return job, nil
+}
+
+// SubmitJob creates a pending job, starts background execution, and returns the job record.
+func (e *AsyncJobExecutor) SubmitJob(bifrostCtx *schemas.BifrostContext, resultTTL int, operation AsyncOperation, operationType schemas.RequestType) (*AsyncJob, error) {
+	if resultTTL <= 0 {
+		resultTTL = DefaultAsyncJobResultTTL
+	}
+
+	virtualKeyValue := getVirtualKeyFromContext(bifrostCtx)
+
+	var virtualKeyID *string
+	if virtualKeyValue != nil {
+		vk, ok := e.governanceStore.GetVirtualKey(bifrostCtx, *virtualKeyValue)
+		if !ok {
+			return nil, fmt.Errorf("virtual key not found")
+		}
+		virtualKeyID = &vk.ID
+	}
+
+	now := time.Now().UTC()
+	job := &AsyncJob{
+		ID:           uuid.New().String(),
+		Status:       schemas.AsyncJobStatusPending,
+		RequestType:  operationType,
+		VirtualKeyID: virtualKeyID,
+		ResultTTL:    resultTTL,
+		CreatedAt:    now,
+	}
+
+	ctx := context.Background()
+	if err := e.logstore.CreateAsyncJob(ctx, job); err != nil {
+		return nil, fmt.Errorf("failed to create async job: %w", err)
+	}
+
+	var contextValues map[any]any
+	if bifrostCtx != nil {
+		contextValues = bifrostCtx.GetUserValues()
+	}
+	go e.executeJob(job.ID, job.ResultTTL, operation, contextValues)
+
+	return job, nil
+}
+
+// executeJob runs the operation in the background and updates the job record.
+func (e *AsyncJobExecutor) executeJob(jobID string, resultTTL int, operation AsyncOperation, contextValues map[any]any) {
+	ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline)
+
+	// Restore original request context values (virtual key, tracing headers, etc.)
+	for k, v := range contextValues {
+		ctx.SetValue(k, v)
+	}
+
+	// Clear trace context inherited from the original HTTP request.
+	ctx.ClearValue(schemas.BifrostContextKeyTraceID)
+	ctx.ClearValue(schemas.BifrostContextKeyParentSpanID)
+	ctx.ClearValue(schemas.BifrostContextKeySpanID)
+
+	markFailed := func(msg string) {
+		now := time.Now().UTC()
+		expiresAt := now.Add(time.Duration(resultTTL) * time.Second)
+		errJSON, _ := sonic.Marshal(&schemas.BifrostError{Error: &schemas.ErrorField{Message: msg}})
+		if err := e.logstore.UpdateAsyncJob(ctx, jobID, map[string]any{
+			"status":       schemas.AsyncJobStatusFailed,
+			"status_code":  fasthttp.StatusInternalServerError,
+			"error":        string(errJSON),
+			"completed_at": now,
+			"expires_at":   expiresAt,
+		}); err != nil {
+			e.logger.Warn("failed to update async job to failed: %v", err)
+		}
+	}
+
+	// The bifrost execution flow is very stable and panics are not expected.
+	// This recover is purely defensive to ensure the job always reaches a terminal
+	// state rather than being stuck in "processing" if an unexpected panic occurs.
+	defer func() {
+		if r := recover(); r != nil {
+			e.logger.Warn("async job %s panicked: %v", jobID, r)
+			markFailed(fmt.Sprintf("internal error: %v", r))
+		}
+	}()
+
+	// Mark as processing
+	if err := e.logstore.UpdateAsyncJob(ctx, jobID, map[string]interface{}{
+		"status": schemas.AsyncJobStatusProcessing,
+	}); err != nil {
+		e.logger.Warn("failed to update async job: %v", err)
+	}
+
+	ctx.SetValue(schemas.BifrostIsAsyncRequest, true)
+
+	// Execute the operation
+	resp, bifrostErr := operation(ctx)
+
+	now := time.Now().UTC()
+	expiresAt := now.Add(time.Duration(resultTTL) * time.Second)
+
+	if bifrostErr != nil {
+		errJSON, err := sonic.Marshal(bifrostErr)
+		if err != nil {
+			e.logger.Warn("failed to marshal bifrost error: %v", err)
+			markFailed(fmt.Sprintf("failed to serialize error response: %v", err))
+			return
+		}
+		statusCode := fasthttp.StatusInternalServerError
+		if bifrostErr.StatusCode != nil {
+			statusCode = *bifrostErr.StatusCode
+		}
+		if err := e.logstore.UpdateAsyncJob(ctx, jobID, map[string]interface{}{
+			"status":       schemas.AsyncJobStatusFailed,
+			"status_code":  statusCode,
+			"error":        string(errJSON),
+			"completed_at": now,
+			"expires_at":   expiresAt,
+		}); err != nil {
+			e.logger.Warn("failed to update async job: %v", err)
+		}
+		return
+	}
+
+	respJSON, err := sonic.Marshal(resp)
+	if err != nil {
+		e.logger.Warn("failed to marshal result: %v", err)
+		markFailed(fmt.Sprintf("failed to serialize result: %v", err))
+		return
+	}
+	if err := e.logstore.UpdateAsyncJob(ctx, jobID, map[string]interface{}{
+		"status":       schemas.AsyncJobStatusCompleted,
+		"status_code":  fasthttp.StatusOK,
+		"response":     string(respJSON),
+		"completed_at": now,
+		"expires_at":   expiresAt,
+	}); err != nil {
+		e.logger.Warn("failed to update async job: %v", err)
+	}
+}
+
+// --- Cleaner ---
+
+// AsyncJobCleaner manages the cleanup of expired async jobs.
+type AsyncJobCleaner struct {
+	store       LogStore
+	logger      schemas.Logger
+	stopCleanup chan struct{}
+	mu          sync.Mutex
+}
+
+// NewAsyncJobCleaner creates a new AsyncJobCleaner instance.
+func NewAsyncJobCleaner(store LogStore, logger schemas.Logger) *AsyncJobCleaner {
+	return &AsyncJobCleaner{
+		store:  store,
+		logger: logger,
+	}
+}
+
+// StartCleanupRoutine starts a goroutine that periodically cleans up expired async jobs.
+func (c *AsyncJobCleaner) StartCleanupRoutine() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.stopCleanup != nil {
+		return
+	}
+
+	c.stopCleanup = make(chan struct{})
+	stopCh := c.stopCleanup
+
+	go func() {
+		// Run initial cleanup
+		ctx, cancel := context.WithTimeout(context.Background(), asyncJobCleanupTimeout)
+		c.cleanupExpiredJobs(ctx)
+		cancel()
+
+		ticker := time.NewTicker(asyncJobCleanupInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				ctx, cancel := context.WithTimeout(context.Background(), asyncJobCleanupTimeout)
+				c.cleanupExpiredJobs(ctx)
+				cancel()
+			case <-stopCh:
+				c.logger.Debug("async job cleanup routine stopped")
+				return
+			}
+		}
+	}()
+	c.logger.Debug("async job cleanup routine started (interval: %s)", asyncJobCleanupInterval)
+}
+
+// StopCleanupRoutine gracefully stops the cleanup goroutine.
+func (c *AsyncJobCleaner) StopCleanupRoutine() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.stopCleanup == nil {
+		c.logger.Debug("async job cleanup routine already stopped")
+		return
+	}
+
+	close(c.stopCleanup)
+	c.stopCleanup = nil
+}
+
+// cleanupExpiredJobs deletes expired async jobs and stale processing jobs.
+func (c *AsyncJobCleaner) cleanupExpiredJobs(ctx context.Context) {
+	deleted, err := c.store.DeleteExpiredAsyncJobs(ctx)
+	if err != nil {
+		c.logger.Warn("failed to delete expired async jobs: %v", err)
+	} else if deleted > 0 {
+		c.logger.Debug("async job cleanup completed: deleted %d expired jobs", deleted)
+	}
+
+	// Clean up jobs stuck in "processing" for more than 24 hours
+	// This handles edge cases like marshal failures or server crashes
+	staleSince := time.Now().UTC().Add(-asyncJobStaleProcessingHours * time.Hour)
+	staleDeleted, err := c.store.DeleteStaleAsyncJobs(ctx, staleSince)
+	if err != nil {
+		c.logger.Warn("failed to delete stale processing async jobs: %v", err)
+	} else if staleDeleted > 0 {
+		c.logger.Warn("async job cleanup: deleted %d stale processing jobs (stuck > %dh)", staleDeleted, asyncJobStaleProcessingHours)
+	}
+}
+
+// getVirtualKeyFromContext extracts the virtual key value from context.
+// Returns nil if no VK is present (e.g., direct key mode or no governance),
+// or if the context itself is nil (callers like SubmitJob may be invoked with
+// a nil ctx by background paths that don't carry a VK).
+func getVirtualKeyFromContext(ctx *schemas.BifrostContext) *string {
+	if ctx == nil {
+		return nil
+	}
+	vkValue := bifrost.GetStringFromContext(ctx, schemas.BifrostContextKeyVirtualKey)
+	if vkValue == "" {
+		return nil
+	}
+	return &vkValue
+}

framework/logstore/asyncjob_test.go

@@ -0,0 +1,213 @@
+package logstore
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/maximhq/bifrost/core/schemas"
+	configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/valyala/fasthttp"
+)
+
+type asyncTestLogger struct{}
+
+func (asyncTestLogger) Debug(string, ...any)                   {}
+func (asyncTestLogger) Info(string, ...any)                    {}
+func (asyncTestLogger) Warn(string, ...any)                    {}
+func (asyncTestLogger) Error(string, ...any)                   {}
+func (asyncTestLogger) Fatal(string, ...any)                   {}
+func (asyncTestLogger) SetLevel(schemas.LogLevel)              {}
+func (asyncTestLogger) SetOutputType(schemas.LoggerOutputType) {}
+func (asyncTestLogger) LogHTTPRequest(schemas.LogLevel, string) schemas.LogEventBuilder {
+	return schemas.NoopLogEvent
+}
+
+type testGovernanceStore struct {
+	virtualKeys map[string]*configstoreTables.TableVirtualKey
+}
+
+func (t *testGovernanceStore) GetVirtualKey(_ context.Context, vkValue string) (*configstoreTables.TableVirtualKey, bool) {
+	vk, ok := t.virtualKeys[vkValue]
+	return vk, ok
+}
+
+func newTestAsyncExecutor(t *testing.T) *AsyncJobExecutor {
+	t.Helper()
+	ctx := context.Background()
+
+	store, err := newSqliteLogStore(ctx, &SQLiteConfig{Path: ":memory:"}, asyncTestLogger{})
+	require.NoError(t, err)
+	t.Cleanup(func() { store.Close(ctx) })
+
+	govStore := &testGovernanceStore{
+		virtualKeys: map[string]*configstoreTables.TableVirtualKey{
+			"sk-bf-test": {ID: "vk-123", Value: "sk-bf-test"},
+		},
+	}
+
+	return NewAsyncJobExecutor(store, govStore, asyncTestLogger{})
+}
+
+// waitForJobCompletion polls until the operation callback has been invoked.
+func waitForJobCompletion(t *testing.T, done *atomic.Bool) {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if done.Load() {
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatal("timed out waiting for async job execution")
+}
+
+// waitForJobStatus polls FindAsyncJobByID until the job reaches a terminal
+// status (completed or failed), or times out. This avoids a fragile time.Sleep
+// between the operation callback completing and the DB update finishing.
+// Processing is intermediate and must not be treated as terminal.
+func waitForJobStatus(t *testing.T, store LogStore, jobID string) *AsyncJob {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		job, err := store.FindAsyncJobByID(context.Background(), jobID)
+		if err == nil && (job.Status == schemas.AsyncJobStatusCompleted || job.Status == schemas.AsyncJobStatusFailed) {
+			return job
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatal("timed out waiting for async job to reach terminal status")
+	return nil
+}
+
+func TestSubmitJob_PropagatesContextValues(t *testing.T) {
+	executor := newTestAsyncExecutor(t)
+
+	capturedCtx := schemas.NewBifrostContext(context.Background(), <-time.After(1*time.Minute))
+	capturedCtx.SetValue(schemas.BifrostContextKeyVirtualKey, "sk-bf-test")
+	capturedCtx.SetValue(schemas.BifrostContextKey("x-bf-eh-custom"), "custom-value")
+	capturedCtx.SetValue(schemas.BifrostContextKey("x-bf-prom-env"), "production")
+	var done atomic.Bool
+
+	operation := func(bgCtx *schemas.BifrostContext) (interface{}, *schemas.BifrostError) {
+		capturedCtx = bgCtx
+		done.Store(true)
+		return map[string]string{"status": "ok"}, nil
+	}
+
+	job, err := executor.SubmitJob(capturedCtx, 3600, operation, schemas.ChatCompletionRequest)
+	require.NoError(t, err)
+	require.NotNil(t, job)
+
+	waitForJobCompletion(t, &done)
+
+	assert.Equal(t, "sk-bf-test", capturedCtx.Value(schemas.BifrostContextKeyVirtualKey))
+	assert.Equal(t, "production", capturedCtx.Value(schemas.BifrostContextKey("x-bf-prom-env")))
+	assert.Equal(t, "custom-value", capturedCtx.Value(schemas.BifrostContextKey("x-bf-eh-custom")))
+	assert.Equal(t, true, capturedCtx.Value(schemas.BifrostIsAsyncRequest))
+}
+
+func TestSubmitJob_NilContextValues(t *testing.T) {
+	executor := newTestAsyncExecutor(t)
+
+	var capturedCtx *schemas.BifrostContext
+	var done atomic.Bool
+
+	operation := func(bgCtx *schemas.BifrostContext) (interface{}, *schemas.BifrostError) {
+		capturedCtx = bgCtx
+		done.Store(true)
+		return map[string]string{"status": "ok"}, nil
+	}
+
+	job, err := executor.SubmitJob(capturedCtx, 3600, operation, schemas.ChatCompletionRequest)
+	require.NoError(t, err)
+	require.NotNil(t, job)
+
+	waitForJobCompletion(t, &done)
+
+	assert.NotNil(t, capturedCtx)
+	assert.Equal(t, true, capturedCtx.Value(schemas.BifrostIsAsyncRequest))
+}
+
+func TestSubmitJob_EmptyContextValues(t *testing.T) {
+	executor := newTestAsyncExecutor(t)
+
+	var capturedCtx *schemas.BifrostContext
+	var done atomic.Bool
+
+	operation := func(bgCtx *schemas.BifrostContext) (interface{}, *schemas.BifrostError) {
+		capturedCtx = bgCtx
+		done.Store(true)
+		return map[string]string{"status": "ok"}, nil
+	}
+
+	job, err := executor.SubmitJob(capturedCtx, 3600, operation, schemas.ChatCompletionRequest)
+	require.NoError(t, err)
+	require.NotNil(t, job)
+
+	waitForJobCompletion(t, &done)
+
+	assert.NotNil(t, capturedCtx)
+	assert.Equal(t, true, capturedCtx.Value(schemas.BifrostIsAsyncRequest))
+}
+
+func TestSubmitJob_AsyncFlagOverridesContextValues(t *testing.T) {
+	executor := newTestAsyncExecutor(t)
+
+	inputCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline)
+	inputCtx.SetValue(schemas.BifrostIsAsyncRequest, false)
+
+	var capturedCtx *schemas.BifrostContext
+	var done atomic.Bool
+	operation := func(bgCtx *schemas.BifrostContext) (interface{}, *schemas.BifrostError) {
+		capturedCtx = bgCtx
+		done.Store(true)
+		return map[string]string{"status": "ok"}, nil
+	}
+
+	job, err := executor.SubmitJob(inputCtx, 3600, operation, schemas.ChatCompletionRequest)
+	require.NoError(t, err)
+	require.NotNil(t, job)
+
+	waitForJobCompletion(t, &done)
+
+	// BifrostIsAsyncRequest must be true — set AFTER restoring context values
+	assert.Equal(t, true, capturedCtx.Value(schemas.BifrostIsAsyncRequest))
+}
+
+func TestSubmitJob_OperationFailure_PreservesContext(t *testing.T) {
+	executor := newTestAsyncExecutor(t)
+
+	inputCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline)
+	inputCtx.SetValue(schemas.BifrostContextKeyVirtualKey, "sk-bf-test")
+
+	var capturedCtx *schemas.BifrostContext
+	var done atomic.Bool
+
+	statusCode := fasthttp.StatusBadRequest
+	operation := func(bgCtx *schemas.BifrostContext) (interface{}, *schemas.BifrostError) {
+		capturedCtx = bgCtx
+		done.Store(true)
+		return nil, &schemas.BifrostError{
+			StatusCode: &statusCode,
+			Error:      &schemas.ErrorField{Message: "test error"},
+		}
+	}
+
+	job, err := executor.SubmitJob(inputCtx, 3600, operation, schemas.ChatCompletionRequest)
+	require.NoError(t, err)
+	require.NotNil(t, job)
+
+	waitForJobCompletion(t, &done)
+
+	// Context values should still be available even when operation fails
+	assert.Equal(t, "sk-bf-test", capturedCtx.Value(schemas.BifrostContextKeyVirtualKey))
+	assert.Equal(t, true, capturedCtx.Value(schemas.BifrostIsAsyncRequest))
+
+	// Verify job was marked as failed — poll until DB update completes
+	retrievedJob := waitForJobStatus(t, executor.logstore, job.ID)
+	assert.Equal(t, schemas.AsyncJobStatusFailed, retrievedJob.Status)
+}

framework/logstore/cleaner.go

@@ -0,0 +1,161 @@
+package logstore
+
+import (
+	"context"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/maximhq/bifrost/core/schemas"
+)
+
+const (
+	cleanupInterval      = 24 * time.Hour
+	minJitter            = 15 * time.Minute
+	maxJitter            = 30 * time.Minute
+	batchSize            = 100
+	defaultRetentionDays = 365
+)
+
+// LogRetentionManager defines the interface for managing log retention and deletion
+type LogRetentionManager interface {
+	DeleteLogsBatch(ctx context.Context, cutoff time.Time, batchSize int) (deletedCount int64, err error)
+}
+
+// CleanerConfig holds configuration for the log cleaner
+type CleanerConfig struct {
+	RetentionDays int
+}
+
+// LogsCleaner manages the cleanup of old logs
+type LogsCleaner struct {
+	manager     LogRetentionManager
+	config      CleanerConfig
+	logger      schemas.Logger
+	stopCleanup chan struct{}
+	mu          sync.Mutex
+}
+
+// NewLogsCleaner creates a new LogsCleaner instance
+func NewLogsCleaner(manager LogRetentionManager, config CleanerConfig, logger schemas.Logger) *LogsCleaner {
+	return &LogsCleaner{
+		manager: manager,
+		config:  config,
+		logger:  logger,
+	}
+}
+
+// StartCleanupRoutine starts a goroutine that periodically cleans up old logs
+func (c *LogsCleaner) StartCleanupRoutine() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Return early if already running
+	if c.stopCleanup != nil {
+		c.logger.Debug("log cleanup routine already running")
+		return
+	}
+
+	c.stopCleanup = make(chan struct{})
+	stopCh := c.stopCleanup
+
+	go func() {
+		// At the beginning, we will cleanup the logs
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+		c.cleanupOldLogs(ctx)
+		cancel()
+		// Calculate initial delay with jitter
+		timer := time.NewTimer(calculateNextRunDuration())
+		defer timer.Stop()
+		for {
+			select {
+			case <-timer.C:
+				// Run cleanup
+				ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+				c.cleanupOldLogs(ctx)
+				cancel()
+
+				// Reset timer with new jitter for next run
+				timer.Reset(calculateNextRunDuration())
+
+			case <-stopCh:
+				c.logger.Info("log cleanup routine stopped")
+				return
+			}
+		}
+	}()
+	c.logger.Info("log cleanup routine started")
+}
+
+// StopCleanupRoutine gracefully stops the cleanup goroutine
+func (c *LogsCleaner) StopCleanupRoutine() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Return early if already stopped
+	if c.stopCleanup == nil {
+		c.logger.Debug("log cleanup routine already stopped")
+		return
+	}
+
+	close(c.stopCleanup)
+	c.stopCleanup = nil
+}
+
+// cleanupOldLogs deletes logs older than the retention period in batches
+func (c *LogsCleaner) cleanupOldLogs(ctx context.Context) {
+	retentionDays := c.config.RetentionDays
+	if retentionDays < 1 {
+		retentionDays = defaultRetentionDays
+	}
+
+	// Calculate cutoff time
+	cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays)
+	c.logger.Info("starting log cleanup: deleting logs older than %s (retention: %d days)", cutoff.Format(time.RFC3339), retentionDays)
+
+	totalDeleted := int64(0)
+	batchCount := 0
+
+	for {
+		// Check if context is cancelled
+		select {
+		case <-ctx.Done():
+			c.logger.Warn("log cleanup cancelled: %v", ctx.Err())
+			return
+		default:
+		}
+
+		// Delete logs in batches using the manager
+		deleted, err := c.manager.DeleteLogsBatch(ctx, cutoff, batchSize)
+		if err != nil {
+			c.logger.Error("failed to delete old logs: %v", err)
+			return
+		}
+
+		if deleted == 0 {
+			// No more logs to delete
+			break
+		}
+
+		totalDeleted += deleted
+		batchCount++
+		c.logger.Debug("deleted batch %d: %d logs", batchCount, deleted)
+
+		// If we deleted fewer than the batch size, we're done
+		if deleted < int64(batchSize) {
+			break
+		}
+	}
+
+	if totalDeleted > 0 {
+		c.logger.Info("log cleanup completed: deleted %d logs in %d batches", totalDeleted, batchCount)
+	} else {
+		c.logger.Debug("log cleanup completed: no old logs to delete")
+	}
+}
+
+// calculateNextRunDuration returns 24 hours plus a random jitter between 15-30 minutes
+func calculateNextRunDuration() time.Duration {
+	jitter := minJitter + time.Duration(rand.Int63n(int64(maxJitter-minJitter)))
+	return cleanupInterval + jitter
+}

framework/logstore/config.go

@@ -0,0 +1,68 @@
+// Package logstore provides a logs store for Bifrost.
+package logstore
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/maximhq/bifrost/framework/objectstore"
+)
+
+// Config represents the configuration for the logs store.
+type Config struct {
+	Enabled       bool                `json:"enabled"`
+	Type          LogStoreType        `json:"type"`
+	RetentionDays int                 `json:"retention_days"`
+	Config        any                 `json:"config"`
+	ObjectStorage *objectstore.Config `json:"object_storage,omitempty"`
+}
+
+// UnmarshalJSON is the custom unmarshal logic for Config
+func (c *Config) UnmarshalJSON(data []byte) error {
+	// First, unmarshal into a temporary struct to get the basic fields
+	type TempConfig struct {
+		Enabled       bool                `json:"enabled"`
+		Type          LogStoreType        `json:"type"`
+		Config        json.RawMessage     `json:"config"` // Keep as raw JSON
+		RetentionDays int                 `json:"retention_days"`
+		ObjectStorage *objectstore.Config `json:"object_storage,omitempty"`
+	}
+
+	var temp TempConfig
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return fmt.Errorf("failed to unmarshal logs config: %w", err)
+	}
+
+	// Set basic fields
+	c.Enabled = temp.Enabled
+	c.Type = temp.Type
+	c.RetentionDays = temp.RetentionDays
+	c.ObjectStorage = temp.ObjectStorage
+	if !temp.Enabled {
+		c.Config = nil
+		return nil
+	}
+
+	// Parse the config field based on type
+	switch temp.Type {
+	case LogStoreTypeSQLite:
+		if len(temp.Config) == 0 {
+			return fmt.Errorf("missing sqlite config payload")
+		}
+		var sqliteConfig SQLiteConfig
+		if err := json.Unmarshal(temp.Config, &sqliteConfig); err != nil {
+			return fmt.Errorf("failed to unmarshal sqlite config: %w", err)
+		}
+		c.Config = &sqliteConfig
+	case LogStoreTypePostgres:
+		var postgresConfig PostgresConfig
+		var err error
+		if err = json.Unmarshal(temp.Config, &postgresConfig); err != nil {
+			return fmt.Errorf("failed to unmarshal postgres config: %w", err)
+		}		
+		c.Config = &postgresConfig
+	default:
+		return fmt.Errorf("unknown log store type: %s", temp.Type)
+	}
+	return nil
+}

framework/logstore/errors.go

@@ -0,0 +1,8 @@
+package logstore
+
+import "fmt"
+
+var (
+	ErrNotFound    = fmt.Errorf("log not found")
+	ErrJobInternal = fmt.Errorf("internal job store error")
+)

framework/logstore/hybrid.go

@@ -0,0 +1,613 @@
+package logstore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/bytedance/sonic"
+	"github.com/maximhq/bifrost/core/schemas"
+	"github.com/maximhq/bifrost/framework/objectstore"
+)
+
+const (
+	defaultUploadWorkers         = 10
+	defaultUploadQueueSize       = 5000
+	maxContentSummaryBytes       = 2048
+	defaultMaxUploadQueueBytes   = 1 << 30 // 1 GiB
+)
+
+// uploadWork represents an async S3 upload job.
+type uploadWork struct {
+	logID     string
+	timestamp time.Time
+	payload   []byte // JSON-encoded payload
+	tags      map[string]string
+}
+
+// HybridLogStore wraps an existing LogStore and offloads large payload
+// fields to object storage while keeping a lightweight index in the DB.
+//
+// Method routing:
+//   - Delegated directly (40+ methods): all analytics, search, histogram, ranking,
+//     distinct, MCP, async job methods
+//   - Intercepted: Create, CreateIfNotExists, BatchCreateIfNotExists, FindByID,
+//     Update, DeleteLog, DeleteLogs, DeleteLogsBatch, Close
+type HybridLogStore struct {
+	inner         LogStore
+	objects       objectstore.ObjectStore
+	prefix        string
+	logger        schemas.Logger
+	uploadQueue   chan *uploadWork
+	wg            sync.WaitGroup
+	closed        atomic.Bool
+	droppedUploads atomic.Int64
+	pendingBytes   atomic.Int64
+}
+
+// newHybridLogStore creates a HybridLogStore wrapping the given inner store.
+func newHybridLogStore(inner LogStore, objects objectstore.ObjectStore, prefix string, logger schemas.Logger) *HybridLogStore {
+	h := &HybridLogStore{
+		inner:       inner,
+		objects:     objects,
+		prefix:      prefix,
+		logger:      logger,
+		uploadQueue: make(chan *uploadWork, defaultUploadQueueSize),
+	}
+	// Start upload workers.
+	for i := 0; i < defaultUploadWorkers; i++ {
+		h.wg.Add(1)
+		go h.uploadWorker()
+	}
+	return h
+}
+
+// uploadWorker processes async S3 upload jobs from the queue.
+func (h *HybridLogStore) uploadWorker() {
+	defer h.wg.Done()
+	for work := range h.uploadQueue {
+		h.processUpload(work)
+	}
+}
+
+// processUpload uploads a single payload to object storage.
+// This is fire-and-forget by design: on Put failure the upload is dropped and
+// counted in droppedUploads. The DB row retains has_object=false, so FindByID
+// falls back to whatever data the DB holds. Retries are intentionally omitted
+// to keep S3 latency from cascading into the write path.
+func (h *HybridLogStore) processUpload(work *uploadWork) {
+	payloadSize := int64(len(work.payload))
+	defer h.pendingBytes.Add(-payloadSize)
+
+	defer func() {
+		if r := recover(); r != nil {
+			h.logger.Error("objectstore: panic in upload worker (recovered): %v", r)
+			h.droppedUploads.Add(1)
+		}
+	}()
+
+	key := ObjectKey(h.prefix, work.timestamp, work.logID)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := h.objects.Put(ctx, key, work.payload, work.tags); err != nil {
+		h.logger.Warn("objectstore: failed to upload log %s: %v", work.logID, err)
+		h.droppedUploads.Add(1)
+		return
+	}
+
+	// Mark the DB row as having an object. Use a fresh context so that a slow
+	// Put doesn't starve the DB update of its deadline. Retry up to 3 times
+	// with exponential backoff to avoid orphaning the uploaded object.
+	for attempt := 0; attempt < 3; attempt++ {
+		dbCtx, dbCancel := context.WithTimeout(context.Background(), 10*time.Second)
+		err := h.inner.Update(dbCtx, work.logID, map[string]interface{}{"has_object": true})
+		dbCancel()
+		if err == nil {
+			return
+		}
+		h.logger.Warn("objectstore: failed to set has_object for log %s (attempt %d/3): %v", work.logID, attempt+1, err)
+		if attempt < 2 {
+			time.Sleep(time.Duration(1<<attempt) * time.Second) // 1s, 2s backoff
+		}
+	}
+	h.logger.Error("objectstore: failed to set has_object for log %s after 3 attempts; payload orphaned in object store", work.logID)
+	h.droppedUploads.Add(1)
+}
+
+// isPayloadEmpty returns true when every value in the payload map is empty.
+// Skipping uploads for empty payloads avoids wasted S3 PUTs (e.g. initial
+// "processing" entries that carry no input/output data yet).
+func isPayloadEmpty(payload map[string]string) bool {
+	for _, v := range payload {
+		if v != "" {
+			return false
+		}
+	}
+	return true
+}
+
+// enqueueUpload pushes an upload job onto the queue. If the queue is full,
+// the job is dropped to prevent S3 slowness from cascading.
+func (h *HybridLogStore) enqueueUpload(logID string, timestamp time.Time, payload map[string]string, tags map[string]string) {
+	if h.closed.Load() || isPayloadEmpty(payload) {
+		return
+	}
+	// Recover from send-on-closed-channel panic: Close() may interleave
+	// between the closed check above and the channel send below.
+	// Same pattern as plugins/logging/writer.go enqueueLogEntry.
+	defer func() {
+		if r := recover(); r != nil {
+			h.droppedUploads.Add(1)
+		}
+	}()
+	data, err := sonic.Marshal(payload)
+	if err != nil {
+		h.logger.Warn("objectstore: failed to marshal payload for log %s: %v", logID, err)
+		h.droppedUploads.Add(1)
+		return
+	}
+	if h.pendingBytes.Load()+int64(len(data)) > defaultMaxUploadQueueBytes {
+		h.droppedUploads.Add(1)
+		h.logger.Warn("objectstore: upload queue memory limit reached, dropping upload for log %s", logID)
+		return
+	}
+	select {
+	case h.uploadQueue <- &uploadWork{
+		logID:     logID,
+		timestamp: timestamp,
+		payload:   data,
+		tags:      tags,
+	}:
+		h.pendingBytes.Add(int64(len(data)))
+	default:
+		h.droppedUploads.Add(1)
+		h.logger.Warn("objectstore: upload queue full, dropping upload for log %s", logID)
+	}
+}
+
+// --- Intercepted methods ---
+
+// prepareDBEntry builds the lightweight DB entry by extracting the content
+// summary, trimming input history to the last user message, and clearing
+// payload fields. Must be called after SerializeFields() populates the
+// Parsed fields.
+func prepareDBEntry(dbEntry *Log) {
+	idx := findLastUserMessageIndex(dbEntry.InputHistoryParsed)
+
+	// Content summary: extract text from the found user message.
+	// Falls back to BuildInputContentSummary for non-chat inputs (speech, image, etc.).
+	if idx >= 0 {
+		dbEntry.ContentSummary = extractChatMessageText(&dbEntry.InputHistoryParsed[idx])
+	} else {
+		dbEntry.ContentSummary = dbEntry.BuildInputContentSummary()
+	}
+	// Bound content summary to prevent large prompts from bloating the DB row.
+	dbEntry.ContentSummary = truncateTag(dbEntry.ContentSummary, maxContentSummaryBytes)
+
+	// Serialize last user message before ClearPayload zeros everything.
+	// msgs[idx:idx+1] reuses the backing array — no heap alloc, no struct copy.
+	var lastUserMessage string
+	if idx >= 0 {
+		lastUserMessage, _ = sonic.MarshalString(dbEntry.InputHistoryParsed[idx : idx+1])
+	}
+
+	ClearPayload(dbEntry)
+
+	// Restore last user message so list queries can display it without S3.
+	dbEntry.InputHistory = lastUserMessage
+}
+
+func (h *HybridLogStore) Create(ctx context.Context, entry *Log) error {
+	if err := entry.SerializeFields(); err != nil {
+		return fmt.Errorf("logstore: serialize before extract: %w", err)
+	}
+	payload := ExtractPayload(entry)
+	tags := BuildTags(entry)
+	// Work on a shallow copy so the caller's entry is preserved on DB failure.
+	dbEntry := *entry
+	prepareDBEntry(&dbEntry)
+	if err := h.inner.Create(ctx, &dbEntry); err != nil {
+		return err
+	}
+	entry.ContentSummary = dbEntry.ContentSummary
+	h.enqueueUpload(entry.ID, entry.Timestamp, payload, tags)
+	return nil
+}
+
+func (h *HybridLogStore) CreateIfNotExists(ctx context.Context, entry *Log) error {
+	if err := entry.SerializeFields(); err != nil {
+		return fmt.Errorf("logstore: serialize before extract: %w", err)
+	}
+	payload := ExtractPayload(entry)
+	tags := BuildTags(entry)
+	// Work on a shallow copy so the caller's entry is preserved on DB failure.
+	dbEntry := *entry
+	prepareDBEntry(&dbEntry)
+	if err := h.inner.CreateIfNotExists(ctx, &dbEntry); err != nil {
+		return err
+	}
+	entry.ContentSummary = dbEntry.ContentSummary
+	h.enqueueUpload(entry.ID, entry.Timestamp, payload, tags)
+	return nil
+}
+
+func (h *HybridLogStore) BatchCreateIfNotExists(ctx context.Context, entries []*Log) error {
+	type pendingUpload struct {
+		logID     string
+		timestamp time.Time
+		payload   map[string]string
+		tags      map[string]string
+	}
+	var uploads []pendingUpload
+
+	dbEntries := make([]*Log, len(entries))
+	for i, entry := range entries {
+		if err := entry.SerializeFields(); err != nil {
+			return fmt.Errorf("logstore: serialize before extract: %w", err)
+		}
+		payload := ExtractPayload(entry)
+		tags := BuildTags(entry)
+		// Work on a shallow copy so the caller's entries are preserved on DB failure.
+		dbEntry := *entry
+		prepareDBEntry(&dbEntry)
+		dbEntries[i] = &dbEntry
+		uploads = append(uploads, pendingUpload{
+			logID:     entry.ID,
+			timestamp: entry.Timestamp,
+			payload:   payload,
+			tags:      tags,
+		})
+	}
+
+	if err := h.inner.BatchCreateIfNotExists(ctx, dbEntries); err != nil {
+		return err
+	}
+
+	for i, entry := range entries {
+		entry.ContentSummary = dbEntries[i].ContentSummary
+	}
+
+	for _, u := range uploads {
+		h.enqueueUpload(u.logID, u.timestamp, u.payload, u.tags)
+	}
+	return nil
+}
+
+func (h *HybridLogStore) FindByID(ctx context.Context, id string) (*Log, error) {
+	log, err := h.inner.FindByID(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+	h.hydrateLog(ctx, log)
+	return log, nil
+}
+
+// hydrateLog fetches the offloaded payload from object storage and merges it
+// back into the Log struct. It is a no-op when HasObject is false.
+//
+// When requestedFields is non-empty, only the payload fields present in that
+// projection are kept after merge — unrequested payload fields are cleared to
+// honour projection semantics and avoid pulling large blobs unnecessarily.
+func (h *HybridLogStore) hydrateLog(ctx context.Context, log *Log, requestedFields ...string) {
+	if log == nil || !log.HasObject {
+		return
+	}
+	key := ObjectKey(h.prefix, log.Timestamp, log.ID)
+	data, err := h.objects.Get(ctx, key)
+	if err != nil {
+		h.logger.Warn("objectstore: failed to fetch payload for log %s: %v", log.ID, err)
+		return // Graceful degradation
+	}
+	if mergeErr := MergePayloadFromJSON(log, data); mergeErr != nil {
+		h.logger.Warn("objectstore: failed to merge payload for log %s: %v", log.ID, mergeErr)
+		return
+	}
+	pruneUnrequestedPayloadFields(log, requestedFields)
+}
+
+func (h *HybridLogStore) Update(ctx context.Context, id string, entry any) error {
+	// Pass through to inner store for index field updates.
+	// Payload fields in the update map are handled separately by the logging plugin.
+	return h.inner.Update(ctx, id, entry)
+}
+
+func (h *HybridLogStore) DeleteLog(ctx context.Context, id string) error {
+	log, findErr := h.inner.FindByID(ctx, id)
+	if findErr != nil && !errors.Is(findErr, ErrNotFound) {
+		return findErr
+	}
+	if err := h.inner.DeleteLog(ctx, id); err != nil {
+		return err
+	}
+	if log != nil && log.HasObject {
+		key := ObjectKey(h.prefix, log.Timestamp, log.ID)
+		if delErr := h.objects.Delete(ctx, key); delErr != nil {
+			h.logger.Warn("objectstore: failed to delete object for log %s: %v", id, delErr)
+		}
+	}
+	return nil
+}
+
+func (h *HybridLogStore) DeleteLogs(ctx context.Context, ids []string) error {
+	// Collect keys for S3 deletion before removing from DB.
+	var keys []string
+	for _, id := range ids {
+		log, findErr := h.inner.FindByID(ctx, id)
+		if findErr != nil && !errors.Is(findErr, ErrNotFound) {
+			return findErr
+		}
+		if log != nil && log.HasObject {
+			keys = append(keys, ObjectKey(h.prefix, log.Timestamp, log.ID))
+		}
+	}
+	if err := h.inner.DeleteLogs(ctx, ids); err != nil {
+		return err
+	}
+	if len(keys) > 0 {
+		if delErr := h.objects.DeleteBatch(ctx, keys); delErr != nil {
+			h.logger.Warn("objectstore: failed to batch delete %d objects: %v", len(keys), delErr)
+		}
+	}
+	return nil
+}
+
+func (h *HybridLogStore) DeleteLogsBatch(ctx context.Context, cutoff time.Time, batchSize int) (int64, error) {
+	// Delegate to inner — S3 objects will be cleaned up by lifecycle policies.
+	return h.inner.DeleteLogsBatch(ctx, cutoff, batchSize)
+}
+
+func (h *HybridLogStore) Close(ctx context.Context) error {
+	h.closed.Store(true)
+	close(h.uploadQueue)
+	done := make(chan struct{})
+	go func() {
+		h.wg.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-ctx.Done():
+		h.logger.Warn("objectstore: shutdown cancelled before upload queue drained: %v", ctx.Err())
+		// Still wait for workers to finish so we don't close dependencies mid-flight.
+		<-done
+	}
+	if err := h.objects.Close(); err != nil {
+		h.logger.Warn("objectstore: error closing object store: %v", err)
+	}
+	return h.inner.Close(ctx)
+}
+
+// DroppedUploads returns the number of S3 uploads that were dropped.
+func (h *HybridLogStore) DroppedUploads() int64 {
+	return h.droppedUploads.Load()
+}
+
+// --- Delegated methods (pass through to inner store unchanged) ---
+
+func (h *HybridLogStore) Ping(ctx context.Context) error {
+	return h.inner.Ping(ctx)
+}
+
+func (h *HybridLogStore) FindFirst(ctx context.Context, query any, fields ...string) (*Log, error) {
+	needsHydration := len(fields) == 0 || fieldsNeedHydration(fields)
+	if needsHydration && len(fields) > 0 {
+		fields = ensureHydrationFields(fields)
+	}
+	log, err := h.inner.FindFirst(ctx, query, fields...)
+	if err != nil {
+		return nil, err
+	}
+	if needsHydration {
+		h.hydrateLog(ctx, log, fields...)
+	}
+	return log, nil
+}
+
+func (h *HybridLogStore) FindAll(ctx context.Context, query any, fields ...string) ([]*Log, error) {
+	needsHydration := len(fields) == 0 || fieldsNeedHydration(fields)
+	if needsHydration && len(fields) > 0 {
+		fields = ensureHydrationFields(fields)
+	}
+	logs, err := h.inner.FindAll(ctx, query, fields...)
+	if err != nil {
+		return nil, err
+	}
+	if needsHydration {
+		for _, log := range logs {
+			h.hydrateLog(ctx, log, fields...)
+		}
+	}
+	return logs, nil
+}
+
+func (h *HybridLogStore) FindAllDistinct(ctx context.Context, query any, fields ...string) ([]*Log, error) {
+	return h.inner.FindAllDistinct(ctx, query, fields...)
+}
+
+func (h *HybridLogStore) HasLogs(ctx context.Context) (bool, error) {
+	return h.inner.HasLogs(ctx)
+}
+
+func (h *HybridLogStore) SearchLogs(ctx context.Context, filters SearchFilters, pagination PaginationOptions) (*SearchResult, error) {
+	return h.inner.SearchLogs(ctx, filters, pagination)
+}
+
+func (h *HybridLogStore) GetSessionLogs(ctx context.Context, sessionID string, pagination PaginationOptions) (*SessionDetailResult, error) {
+	return h.inner.GetSessionLogs(ctx, sessionID, pagination)
+}
+
+func (h *HybridLogStore) GetSessionSummary(ctx context.Context, sessionID string) (*SessionSummaryResult, error) {
+	return h.inner.GetSessionSummary(ctx, sessionID)
+}
+
+func (h *HybridLogStore) GetStats(ctx context.Context, filters SearchFilters) (*SearchStats, error) {
+	return h.inner.GetStats(ctx, filters)
+}
+
+func (h *HybridLogStore) GetHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*HistogramResult, error) {
+	return h.inner.GetHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetTokenHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*TokenHistogramResult, error) {
+	return h.inner.GetTokenHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetCostHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*CostHistogramResult, error) {
+	return h.inner.GetCostHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetModelHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*ModelHistogramResult, error) {
+	return h.inner.GetModelHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetLatencyHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*LatencyHistogramResult, error) {
+	return h.inner.GetLatencyHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetProviderCostHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*ProviderCostHistogramResult, error) {
+	return h.inner.GetProviderCostHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetProviderTokenHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*ProviderTokenHistogramResult, error) {
+	return h.inner.GetProviderTokenHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetProviderLatencyHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*ProviderLatencyHistogramResult, error) {
+	return h.inner.GetProviderLatencyHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetModelRankings(ctx context.Context, filters SearchFilters) (*ModelRankingResult, error) {
+	return h.inner.GetModelRankings(ctx, filters)
+}
+
+func (h *HybridLogStore) GetUserRankings(ctx context.Context, filters SearchFilters) (*UserRankingResult, error) {
+	return h.inner.GetUserRankings(ctx, filters)
+}
+
+func (h *HybridLogStore) GetDimensionCostHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64, dimension HistogramDimension) (*DimensionCostHistogramResult, error) {
+	return h.inner.GetDimensionCostHistogram(ctx, filters, bucketSizeSeconds, dimension)
+}
+
+func (h *HybridLogStore) GetDimensionTokenHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64, dimension HistogramDimension) (*DimensionTokenHistogramResult, error) {
+	return h.inner.GetDimensionTokenHistogram(ctx, filters, bucketSizeSeconds, dimension)
+}
+
+func (h *HybridLogStore) GetDimensionLatencyHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64, dimension HistogramDimension) (*DimensionLatencyHistogramResult, error) {
+	return h.inner.GetDimensionLatencyHistogram(ctx, filters, bucketSizeSeconds, dimension)
+}
+
+func (h *HybridLogStore) BulkUpdateCost(ctx context.Context, updates map[string]float64) error {
+	return h.inner.BulkUpdateCost(ctx, updates)
+}
+
+func (h *HybridLogStore) Flush(ctx context.Context, since time.Time) error {
+	return h.inner.Flush(ctx, since)
+}
+
+func (h *HybridLogStore) IsLogEntryPresent(ctx context.Context, id string) (bool, error) {
+	return h.inner.IsLogEntryPresent(ctx, id)
+}
+
+func (h *HybridLogStore) GetDistinctAliases(ctx context.Context) ([]string, error) {
+	return h.inner.GetDistinctAliases(ctx)
+}
+
+func (h *HybridLogStore) GetDistinctModels(ctx context.Context) ([]string, error) {
+	return h.inner.GetDistinctModels(ctx)
+}
+
+func (h *HybridLogStore) GetDistinctKeyPairs(ctx context.Context, idCol, nameCol string) ([]KeyPairResult, error) {
+	return h.inner.GetDistinctKeyPairs(ctx, idCol, nameCol)
+}
+
+func (h *HybridLogStore) GetDistinctRoutingEngines(ctx context.Context) ([]string, error) {
+	return h.inner.GetDistinctRoutingEngines(ctx)
+}
+
+func (h *HybridLogStore) GetDistinctMetadataKeys(ctx context.Context) (map[string][]string, error) {
+	return h.inner.GetDistinctMetadataKeys(ctx)
+}
+
+// MCP Tool Log methods — delegated directly.
+
+func (h *HybridLogStore) GetMCPHistogram(ctx context.Context, filters MCPToolLogSearchFilters, bucketSizeSeconds int64) (*MCPHistogramResult, error) {
+	return h.inner.GetMCPHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetMCPCostHistogram(ctx context.Context, filters MCPToolLogSearchFilters, bucketSizeSeconds int64) (*MCPCostHistogramResult, error) {
+	return h.inner.GetMCPCostHistogram(ctx, filters, bucketSizeSeconds)
+}
+
+func (h *HybridLogStore) GetMCPTopTools(ctx context.Context, filters MCPToolLogSearchFilters, limit int) (*MCPTopToolsResult, error) {
+	return h.inner.GetMCPTopTools(ctx, filters, limit)
+}
+
+func (h *HybridLogStore) CreateMCPToolLog(ctx context.Context, entry *MCPToolLog) error {
+	return h.inner.CreateMCPToolLog(ctx, entry)
+}
+
+func (h *HybridLogStore) FindMCPToolLog(ctx context.Context, id string) (*MCPToolLog, error) {
+	return h.inner.FindMCPToolLog(ctx, id)
+}
+
+func (h *HybridLogStore) UpdateMCPToolLog(ctx context.Context, id string, entry any) error {
+	return h.inner.UpdateMCPToolLog(ctx, id, entry)
+}
+
+func (h *HybridLogStore) SearchMCPToolLogs(ctx context.Context, filters MCPToolLogSearchFilters, pagination PaginationOptions) (*MCPToolLogSearchResult, error) {
+	return h.inner.SearchMCPToolLogs(ctx, filters, pagination)
+}
+
+func (h *HybridLogStore) GetMCPToolLogStats(ctx context.Context, filters MCPToolLogSearchFilters) (*MCPToolLogStats, error) {
+	return h.inner.GetMCPToolLogStats(ctx, filters)
+}
+
+func (h *HybridLogStore) HasMCPToolLogs(ctx context.Context) (bool, error) {
+	return h.inner.HasMCPToolLogs(ctx)
+}
+
+func (h *HybridLogStore) DeleteMCPToolLogs(ctx context.Context, ids []string) error {
+	return h.inner.DeleteMCPToolLogs(ctx, ids)
+}
+
+func (h *HybridLogStore) FlushMCPToolLogs(ctx context.Context, since time.Time) error {
+	return h.inner.FlushMCPToolLogs(ctx, since)
+}
+
+func (h *HybridLogStore) GetAvailableToolNames(ctx context.Context) ([]string, error) {
+	return h.inner.GetAvailableToolNames(ctx)
+}
+
+func (h *HybridLogStore) GetAvailableServerLabels(ctx context.Context) ([]string, error) {
+	return h.inner.GetAvailableServerLabels(ctx)
+}
+
+func (h *HybridLogStore) GetAvailableMCPVirtualKeys(ctx context.Context) ([]MCPToolLog, error) {
+	return h.inner.GetAvailableMCPVirtualKeys(ctx)
+}
+
+// Async Job methods — delegated directly.
+
+func (h *HybridLogStore) CreateAsyncJob(ctx context.Context, job *AsyncJob) error {
+	return h.inner.CreateAsyncJob(ctx, job)
+}
+
+func (h *HybridLogStore) FindAsyncJobByID(ctx context.Context, id string) (*AsyncJob, error) {
+	return h.inner.FindAsyncJobByID(ctx, id)
+}
+
+func (h *HybridLogStore) UpdateAsyncJob(ctx context.Context, id string, updates map[string]interface{}) error {
+	return h.inner.UpdateAsyncJob(ctx, id, updates)
+}
+
+func (h *HybridLogStore) DeleteExpiredAsyncJobs(ctx context.Context) (int64, error) {
+	return h.inner.DeleteExpiredAsyncJobs(ctx)
+}
+
+func (h *HybridLogStore) DeleteStaleAsyncJobs(ctx context.Context, staleSince time.Time) (int64, error) {
+	return h.inner.DeleteStaleAsyncJobs(ctx, staleSince)
+}

framework/logstore/hybrid_test.go

@@ -0,0 +1,332 @@
+package logstore
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/maximhq/bifrost/core/schemas"
+	"github.com/maximhq/bifrost/framework/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type hybridTestLogger struct{}
+
+func (hybridTestLogger) Debug(string, ...any)                                  {}
+func (hybridTestLogger) Info(string, ...any)                                   {}
+func (hybridTestLogger) Warn(string, ...any)                                   {}
+func (hybridTestLogger) Error(string, ...any)                                  {}
+func (hybridTestLogger) Fatal(string, ...any)                                  {}
+func (hybridTestLogger) SetLevel(schemas.LogLevel)                             {}
+func (hybridTestLogger) SetOutputType(schemas.LoggerOutputType)                {}
+func (hybridTestLogger) LogHTTPRequest(schemas.LogLevel, string) schemas.LogEventBuilder {
+	return schemas.NoopLogEvent
+}
+
+func newTestHybrid(t *testing.T) (*HybridLogStore, LogStore, *objectstore.InMemoryObjectStore) {
+	t.Helper()
+	ctx := context.Background()
+
+	// Create SQLite inner store.
+	inner, err := newSqliteLogStore(ctx, &SQLiteConfig{Path: ":memory:"}, hybridTestLogger{})
+	require.NoError(t, err)
+
+	objStore := objectstore.NewInMemoryObjectStore()
+	hybrid := newHybridLogStore(inner, objStore, "test", hybridTestLogger{})
+	return hybrid, inner, objStore
+}
+
+func waitForUploads(t *testing.T, done func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if done() {
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatal("timed out waiting for upload state")
+}
+
+func TestHybrid_CreateAndFindByID(t *testing.T) {
+	hybrid, _, objStore := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	inputContent := "Hello, how are you?"
+	entry := &Log{
+		ID:        "log-1",
+		Timestamp: time.Now().UTC(),
+		Provider:  "anthropic",
+		Model:     "claude-3-sonnet",
+		Status:    "success",
+		Object:    "chat.completion",
+		InputHistoryParsed: []schemas.ChatMessage{
+			{Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: &inputContent}},
+		},
+		OutputMessageParsed: &schemas.ChatMessage{
+			Content: &schemas.ChatMessageContent{ContentStr: strPtr("I'm fine, thanks!")},
+		},
+	}
+
+	// Serialize fields so TEXT columns are populated (simulating what GORM BeforeCreate does).
+	require.NoError(t, entry.SerializeFields())
+
+	err := hybrid.CreateIfNotExists(ctx, entry)
+	require.NoError(t, err)
+
+	waitForUploads(t, func() bool { return objStore.Len() == 1 })
+
+	// Verify object was uploaded.
+	assert.Equal(t, 1, objStore.Len(), "expected 1 object in store")
+
+	// FindByID should return hydrated log with payload.
+	found, err := hybrid.FindByID(ctx, "log-1")
+	require.NoError(t, err)
+	assert.Equal(t, "log-1", found.ID)
+	assert.True(t, found.HasObject)
+	assert.NotEmpty(t, found.InputHistory, "InputHistory should be hydrated from S3")
+	assert.NotEmpty(t, found.OutputMessage, "OutputMessage should be hydrated from S3")
+
+	// Content summary should contain input text but the output should be in the payload.
+	assert.Contains(t, found.ContentSummary, "Hello, how are you?")
+}
+
+func TestHybrid_EmptyPayloadSkipsUpload(t *testing.T) {
+	hybrid, _, objStore := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	entry := &Log{
+		ID:        "log-processing",
+		Timestamp: time.Now().UTC(),
+		Provider:  "openai",
+		Model:     "gpt-4",
+		Status:    "processing",
+		Object:    "chat.completion",
+	}
+
+	err := hybrid.CreateIfNotExists(ctx, entry)
+	require.NoError(t, err)
+
+	waitForUploads(t, func() bool { return len(hybrid.uploadQueue) == 0 })
+
+	// No upload when all payload fields are empty (e.g. initial "processing" entries).
+	assert.Equal(t, 0, objStore.Len(), "empty-payload entries should not be uploaded")
+}
+
+func TestHybrid_BatchCreateIfNotExists(t *testing.T) {
+	hybrid, _, objStore := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	entries := make([]*Log, 3)
+	for i := 0; i < 3; i++ {
+		content := "input message"
+		entries[i] = &Log{
+			ID:        "batch-" + string(rune('a'+i)),
+			Timestamp: time.Now().UTC(),
+			Provider:  "anthropic",
+			Model:     "claude-3",
+			Status:    "success",
+			Object:    "chat.completion",
+			InputHistoryParsed: []schemas.ChatMessage{
+				{Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: &content}},
+			},
+		}
+		require.NoError(t, entries[i].SerializeFields())
+	}
+
+	err := hybrid.BatchCreateIfNotExists(ctx, entries)
+	require.NoError(t, err)
+
+	waitForUploads(t, func() bool { return objStore.Len() == 3 })
+	assert.Equal(t, 3, objStore.Len())
+}
+
+func TestHybrid_FindByID_NoObject(t *testing.T) {
+	hybrid, inner, _ := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	// Insert directly into inner store (simulating legacy data without object).
+	entry := &Log{
+		ID:           "legacy-1",
+		Timestamp:    time.Now().UTC(),
+		Provider:     "openai",
+		Model:        "gpt-4",
+		Status:       "success",
+		Object:       "chat.completion",
+		InputHistory: `[{"role":"user","content":"legacy input"}]`,
+		HasObject:    false,
+	}
+	require.NoError(t, inner.CreateIfNotExists(ctx, entry))
+
+	found, err := hybrid.FindByID(ctx, "legacy-1")
+	require.NoError(t, err)
+	assert.False(t, found.HasObject)
+	// Legacy data: payload is in DB.
+	assert.NotEmpty(t, found.InputHistory)
+}
+
+func TestHybrid_FindByID_GracefulDegradation(t *testing.T) {
+	hybrid, _, objStore := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	content := "test input"
+	entry := &Log{
+		ID:        "degrade-1",
+		Timestamp: time.Now().UTC(),
+		Provider:  "anthropic",
+		Model:     "claude-3",
+		Status:    "success",
+		Object:    "chat.completion",
+		InputHistoryParsed: []schemas.ChatMessage{
+			{Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: &content}},
+		},
+	}
+	require.NoError(t, entry.SerializeFields())
+	require.NoError(t, hybrid.CreateIfNotExists(ctx, entry))
+	waitForUploads(t, func() bool { return objStore.Len() == 1 })
+
+	// Simulate S3 failure.
+	objStore.GetErr = assert.AnError
+
+	found, err := hybrid.FindByID(ctx, "degrade-1")
+	require.NoError(t, err, "FindByID should succeed even when S3 fails")
+	assert.True(t, found.HasObject)
+	// When S3 fails, the DB data is returned. The DB retains the last message
+	// in input_history for list views, so it won't be empty.
+	assert.NotEmpty(t, found.InputHistory, "last message should be retained in DB")
+	// But other payload fields (output_message, params, etc.) should be empty.
+	assert.Empty(t, found.OutputMessage, "output should be empty when S3 fails")
+}
+
+func TestHybrid_PutFailureDropsUpload(t *testing.T) {
+	hybrid, _, objStore := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	// Simulate S3 write failure.
+	objStore.PutErr = assert.AnError
+
+	content := "important input"
+	entry := &Log{
+		ID:        "put-fail-1",
+		Timestamp: time.Now().UTC(),
+		Provider:  "anthropic",
+		Model:     "claude-3",
+		Status:    "success",
+		Object:    "chat.completion",
+		InputHistoryParsed: []schemas.ChatMessage{
+			{Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: &content}},
+		},
+	}
+	require.NoError(t, entry.SerializeFields())
+	require.NoError(t, hybrid.CreateIfNotExists(ctx, entry))
+	waitForUploads(t, func() bool { return hybrid.DroppedUploads() == 1 })
+
+	// Upload should have been dropped.
+	assert.Equal(t, 0, objStore.Len(), "no object should be stored when Put fails")
+	assert.Equal(t, int64(1), hybrid.DroppedUploads(), "dropped upload should be counted")
+
+	// DB row exists but has_object remains false since the upload failed.
+	found, err := hybrid.FindByID(ctx, "put-fail-1")
+	require.NoError(t, err)
+	assert.False(t, found.HasObject, "has_object should remain false when upload fails")
+}
+
+func TestHybrid_DeleteLog(t *testing.T) {
+	hybrid, _, objStore := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	entry := &Log{
+		ID:           "del-1",
+		Timestamp:    time.Now().UTC(),
+		Provider:     "anthropic",
+		Model:        "claude-3",
+		Status:       "success",
+		Object:       "chat.completion",
+		InputHistory: `[{"role":"user","content":"delete me"}]`,
+	}
+	require.NoError(t, entry.SerializeFields())
+	require.NoError(t, hybrid.CreateIfNotExists(ctx, entry))
+	waitForUploads(t, func() bool { return objStore.Len() == 1 })
+	assert.Equal(t, 1, objStore.Len())
+
+	err := hybrid.DeleteLog(ctx, "del-1")
+	require.NoError(t, err)
+
+	// Object should be deleted from S3.
+	assert.Equal(t, 0, objStore.Len())
+
+	// DB should also be empty.
+	_, err = hybrid.FindByID(ctx, "del-1")
+	assert.Error(t, err)
+}
+
+func TestHybrid_Tags(t *testing.T) {
+	hybrid, _, objStore := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	ts := time.Date(2026, 4, 3, 14, 30, 0, 0, time.UTC)
+	vkID := "vk_test"
+	entry := &Log{
+		ID:           "tag-1",
+		Timestamp:    ts,
+		Provider:     "anthropic",
+		Model:        "claude-3",
+		Status:       "error",
+		Object:       "chat.completion",
+		VirtualKeyID: &vkID,
+		Stream:       true,
+		InputHistory: `[{"role":"user","content":"test"}]`,
+	}
+	require.NoError(t, entry.SerializeFields())
+	require.NoError(t, hybrid.CreateIfNotExists(ctx, entry))
+	waitForUploads(t, func() bool { return objStore.Len() == 1 })
+
+	key := ObjectKey("test", ts, "tag-1")
+	tags := objStore.GetTags(key)
+	assert.Equal(t, "anthropic", tags["provider"])
+	assert.Equal(t, "error", tags["status"])
+	assert.Equal(t, "true", tags["has_error"])
+	assert.Equal(t, "true", tags["stream"])
+	assert.Equal(t, "vk_test", tags["virtual_key_id"])
+	assert.Equal(t, "2026-04-03", tags["date"])
+}
+
+func TestHybrid_ContentSummaryIsInputOnly(t *testing.T) {
+	hybrid, inner, _ := newTestHybrid(t)
+	defer hybrid.Close(context.Background())
+	ctx := context.Background()
+
+	inputText := "What is the capital of France?"
+	outputText := "The capital of France is Paris."
+	entry := &Log{
+		ID:        "summary-1",
+		Timestamp: time.Now().UTC(),
+		Provider:  "anthropic",
+		Model:     "claude-3",
+		Status:    "success",
+		Object:    "chat.completion",
+		InputHistoryParsed: []schemas.ChatMessage{
+			{Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: &inputText}},
+		},
+		OutputMessageParsed: &schemas.ChatMessage{
+			Content: &schemas.ChatMessageContent{ContentStr: &outputText},
+		},
+	}
+	require.NoError(t, entry.SerializeFields())
+	require.NoError(t, hybrid.CreateIfNotExists(ctx, entry))
+
+	// Read from inner DB to check content_summary.
+	dbLog, err := inner.FindByID(ctx, "summary-1")
+	require.NoError(t, err)
+	assert.Contains(t, dbLog.ContentSummary, "capital of France")
+	assert.NotContains(t, dbLog.ContentSummary, "Paris", "content_summary should not contain output text")
+}

framework/logstore/logger.go

@@ -0,0 +1,45 @@
+package logstore
+
+import (
+	"context"
+	"time"
+
+	"github.com/maximhq/bifrost/core/schemas"
+	gormLibLogger "gorm.io/gorm/logger"
+)
+
+// GormLogger is a logger for GORM.
+type gormLogger struct {
+	logger schemas.Logger
+}
+
+// LogMode sets the log mode for the logger.
+func (l *gormLogger) LogMode(level gormLibLogger.LogLevel) gormLibLogger.Interface {
+	// NOOP
+	return l
+}
+
+// Info logs an info message.
+func (l *gormLogger) Info(ctx context.Context, msg string, data ...interface{}) {
+	l.logger.Info(msg, data...)
+}
+
+// Warn logs a warning message.
+func (l *gormLogger) Warn(ctx context.Context, msg string, data ...interface{}) {
+	l.logger.Warn(msg, data...)
+}
+
+// Error logs an error message.
+func (l *gormLogger) Error(ctx context.Context, msg string, data ...interface{}) {
+	l.logger.Error(msg, data...)
+}
+
+// Trace logs a trace message.
+func (l *gormLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
+	// NOOP
+}
+
+// newGormLogger creates a new GormLogger.
+func newGormLogger(l schemas.Logger) *gormLogger {
+	return &gormLogger{logger: l}
+}

framework/logstore/migrations_test.go

@@ -0,0 +1,437 @@
+package logstore
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+// postgresDSN matches the postgres service in tests/docker-compose.yml and
+// framework/docker-compose.yml.
+const postgresDSN = "host=localhost user=bifrost password=bifrost_password dbname=bifrost port=5432 sslmode=disable"
+
+// trySetupPostgresDB attempts to connect to Postgres and returns the connection.
+// Returns nil if Postgres is unavailable.
+func trySetupPostgresDB(t *testing.T) *gorm.DB {
+	t.Helper()
+	db, err := gorm.Open(postgres.Open(postgresDSN), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		return nil
+	}
+
+	// Verify the connection is actually live before proceeding.
+	sqlDB, err := db.DB()
+	if err != nil {
+		return nil
+	}
+	if err := sqlDB.Ping(); err != nil {
+		return nil
+	}
+
+	return db
+}
+
+// setupLogsTableForGINIndexTest creates the logs table in a pre-migration state
+// (with metadata column but without the GIN index) for testing the GIN index migration.
+func setupLogsTableForGINIndexTest(t *testing.T, db *gorm.DB) {
+	t.Helper()
+
+	// Drop existing tables and migration tracking in the correct order.
+	// Preserve the shared migrations table — only clear its rows.
+	db.Exec("DROP INDEX IF EXISTS idx_logs_metadata_gin")
+	db.Exec("DROP TABLE IF EXISTS logs")
+	db.Exec("CREATE TABLE IF NOT EXISTS migrations (id VARCHAR(255) PRIMARY KEY)")
+	db.Exec("DELETE FROM migrations")
+
+	// Create a minimal logs table with only the columns needed for the test
+	err := db.Exec(`
+		CREATE TABLE logs (
+			id VARCHAR(255) PRIMARY KEY,
+			timestamp TIMESTAMP NOT NULL,
+			object_type VARCHAR(255) NOT NULL,
+			provider VARCHAR(255) NOT NULL,
+			model VARCHAR(255) NOT NULL,
+			status VARCHAR(50) NOT NULL,
+			metadata TEXT,
+			created_at TIMESTAMP NOT NULL
+		)
+	`).Error
+	require.NoError(t, err, "Failed to create logs table")
+
+	// The migrator will create the migrations table automatically when it runs
+
+	// Clean up tables after the test
+	t.Cleanup(func() {
+		db.Exec("DROP INDEX IF EXISTS idx_logs_metadata_gin")
+		db.Exec("DROP TABLE IF EXISTS logs")
+		db.Exec("DELETE FROM migrations")
+	})
+}
+
+// insertTestLog inserts a test log entry with the given metadata value.
+func insertTestLog(t *testing.T, db *gorm.DB, id string, metadata *string) {
+	t.Helper()
+	now := time.Now()
+
+	var metadataVal interface{}
+	if metadata != nil {
+		metadataVal = *metadata
+	}
+
+	err := db.Exec(`
+		INSERT INTO logs (id, timestamp, object_type, provider, model, status, metadata, created_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+	`, id, now, "chat_completion", "openai", "gpt-4", "success", metadataVal, now).Error
+	require.NoError(t, err, "Failed to insert test log %s", id)
+}
+
+// getMetadataValue retrieves the metadata value for a given log ID.
+func getMetadataValue(t *testing.T, db *gorm.DB, id string) *string {
+	t.Helper()
+	var result struct {
+		Metadata *string
+	}
+	err := db.Table("logs").Select("metadata").Where("id = ?", id).Scan(&result).Error
+	require.NoError(t, err, "Failed to get metadata for log %s", id)
+	return result.Metadata
+}
+
+// indexExists checks if the GIN index exists on the logs table.
+func indexExists(t *testing.T, db *gorm.DB, indexName string) bool {
+	t.Helper()
+	var count int64
+	err := db.Raw(`
+		SELECT COUNT(*) FROM pg_indexes 
+		WHERE tablename = 'logs' AND indexname = ?
+	`, indexName).Scan(&count).Error
+	require.NoError(t, err, "Failed to check index existence")
+	return count > 0
+}
+
+func TestMigrationAddMetadataGINIndex_ValidJSON(t *testing.T) {
+	db := trySetupPostgresDB(t)
+	if db == nil {
+		t.Skip("Postgres not available, skipping test")
+	}
+
+	setupLogsTableForGINIndexTest(t, db)
+	ctx := context.Background()
+
+	// Insert logs with valid JSON object metadata (arrays are not supported)
+	validJSON1 := `{"key": "value"}`
+	validJSON2 := `{"nested": {"foo": "bar"}, "array": [1, 2, 3]}`
+	validJSON3 := `{"empty": {}}`
+	validJSON4 := `{"number": 42, "bool": true, "null": null}`
+
+	insertTestLog(t, db, "log-valid-1", &validJSON1)
+	insertTestLog(t, db, "log-valid-2", &validJSON2)
+	insertTestLog(t, db, "log-valid-3", &validJSON3)
+	insertTestLog(t, db, "log-valid-4", &validJSON4)
+
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("Failed to get SQL DB: %v", err)
+	}
+	conn, err := sqlDB.Conn(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to get SQL connection: %v", err)
+	}
+	t.Cleanup(func() { _ = conn.Close() })
+
+	// Run the migration (cleanup only) then ensure the index is built.
+	err = migrationAddMetadataGINIndex(ctx, db)
+	require.NoError(t, err, "Migration should succeed")
+	err = ensureMetadataGINIndex(ctx, conn)
+	require.NoError(t, err, "GIN index creation should succeed")
+
+	// Verify all valid JSON object values are preserved
+	meta1 := getMetadataValue(t, db, "log-valid-1")
+	assert.NotNil(t, meta1, "Valid JSON object should be preserved")
+	assert.Equal(t, validJSON1, *meta1)
+
+	meta2 := getMetadataValue(t, db, "log-valid-2")
+	assert.NotNil(t, meta2, "Valid JSON object should be preserved")
+	assert.Equal(t, validJSON2, *meta2)
+
+	meta3 := getMetadataValue(t, db, "log-valid-3")
+	assert.NotNil(t, meta3, "Valid JSON object with nested empty object should be preserved")
+	assert.Equal(t, validJSON3, *meta3)
+
+	meta4 := getMetadataValue(t, db, "log-valid-4")
+	assert.NotNil(t, meta4, "Valid JSON object with various types should be preserved")
+	assert.Equal(t, validJSON4, *meta4)
+
+	// Verify the GIN index was created
+	assert.True(t, indexExists(t, db, "idx_logs_metadata_gin"), "GIN index should be created")
+}
+
+func TestMigrationAddMetadataGINIndex_InvalidJSON(t *testing.T) {
+	db := trySetupPostgresDB(t)
+	if db == nil {
+		t.Skip("Postgres not available, skipping test")
+	}
+
+	setupLogsTableForGINIndexTest(t, db)
+	ctx := context.Background()
+
+	// Insert logs with invalid JSON metadata (not valid JSON objects)
+	invalid1 := `{"key": invalid}`     // Unquoted value
+	invalid2 := `{key: "value"}`       // Unquoted key
+	invalid3 := `{"key": "value",}`    // Trailing comma
+	invalid4 := `just a string`        // Plain text
+	invalid5 := ``                     // Empty string
+	invalid6 := `{"unclosed": "brace"` // Unclosed brace
+	invalid7 := `{"key": undefined}`   // JavaScript undefined
+	invalid8 := `{'single': 'quotes'}` // Single quotes
+	invalid9 := `[NULL]`               // Literal string [NULL] (not valid JSON)
+	invalid10 := `NULL`                // Literal string NULL (not valid JSON)
+	invalid11 := `null`                // Valid JSON but not a JSON object
+	invalid12 := `[1, 2, 3]`           // Valid JSON array but not a JSON object
+
+	insertTestLog(t, db, "log-invalid-1", &invalid1)
+	insertTestLog(t, db, "log-invalid-2", &invalid2)
+	insertTestLog(t, db, "log-invalid-3", &invalid3)
+	insertTestLog(t, db, "log-invalid-4", &invalid4)
+	insertTestLog(t, db, "log-invalid-5", &invalid5)
+	insertTestLog(t, db, "log-invalid-6", &invalid6)
+	insertTestLog(t, db, "log-invalid-7", &invalid7)
+	insertTestLog(t, db, "log-invalid-8", &invalid8)
+	insertTestLog(t, db, "log-invalid-9", &invalid9)
+	insertTestLog(t, db, "log-invalid-10", &invalid10)
+	insertTestLog(t, db, "log-invalid-11", &invalid11)
+	insertTestLog(t, db, "log-invalid-12", &invalid12)
+	insertTestLog(t, db, "log-actual-null", nil) // Actual SQL NULL
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("Failed to get SQL DB: %v", err)
+	}
+	conn, err := sqlDB.Conn(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to get SQL connection: %v", err)
+	}
+	t.Cleanup(func() { _ = conn.Close() })
+	// Run the migration (cleanup only) then ensure the index is built.
+	err = migrationAddMetadataGINIndex(ctx, db)
+	require.NoError(t, err, "Migration should succeed even with invalid JSON")
+	err = ensureMetadataGINIndex(ctx, conn)
+	require.NoError(t, err, "GIN index creation should succeed after invalid JSON cleanup")
+
+	// Verify all non-object values were set to NULL (only JSON objects are supported)
+	for i := 1; i <= 12; i++ {
+		id := fmt.Sprintf("log-invalid-%d", i)
+		meta := getMetadataValue(t, db, id)
+		assert.Nil(t, meta, "Non-object JSON for %s should be set to NULL", id)
+	}
+
+	// Verify actual SQL NULL remains NULL
+	metaActualNull := getMetadataValue(t, db, "log-actual-null")
+	assert.Nil(t, metaActualNull, "Actual NULL should remain NULL")
+
+	// Verify the GIN index was created
+	assert.True(t, indexExists(t, db, "idx_logs_metadata_gin"), "GIN index should be created")
+}
+
+func TestMigrationAddMetadataGINIndex_MixedData(t *testing.T) {
+	db := trySetupPostgresDB(t)
+	if db == nil {
+		t.Skip("Postgres not available, skipping test")
+	}
+
+	setupLogsTableForGINIndexTest(t, db)
+	ctx := context.Background()
+
+	// Insert a mix of valid JSON, invalid JSON, and NULL metadata
+	validJSON := `{"environment": "production", "version": "1.0.0"}`
+	invalidJSON := `{"broken": invalid_value}`
+
+	insertTestLog(t, db, "log-mixed-valid", &validJSON)
+	insertTestLog(t, db, "log-mixed-invalid", &invalidJSON)
+	insertTestLog(t, db, "log-mixed-null", nil)
+
+	// Run the migration (cleanup only) then ensure the index is built.
+	err := migrationAddMetadataGINIndex(ctx, db)
+	require.NoError(t, err, "Migration should succeed")
+
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("Failed to get SQL DB: %v", err)
+	}
+	conn, err := sqlDB.Conn(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to get SQL connection: %v", err)
+	}
+	t.Cleanup(func() { _ = conn.Close() })
+
+	err = ensureMetadataGINIndex(ctx, conn)
+	require.NoError(t, err, "GIN index creation should succeed")
+
+	// Verify valid JSON is preserved
+	metaValid := getMetadataValue(t, db, "log-mixed-valid")
+	assert.NotNil(t, metaValid, "Valid JSON should be preserved")
+	assert.Equal(t, validJSON, *metaValid)
+
+	// Verify invalid JSON is cleaned to NULL
+	metaInvalid := getMetadataValue(t, db, "log-mixed-invalid")
+	assert.Nil(t, metaInvalid, "Invalid JSON should be set to NULL")
+
+	// Verify NULL remains NULL
+	metaNull := getMetadataValue(t, db, "log-mixed-null")
+	assert.Nil(t, metaNull, "NULL metadata should remain NULL")
+
+	// Verify the GIN index was created
+	assert.True(t, indexExists(t, db, "idx_logs_metadata_gin"), "GIN index should be created")
+}
+
+func TestMigrationAddMetadataGINIndex_Idempotent(t *testing.T) {
+	db := trySetupPostgresDB(t)
+	if db == nil {
+		t.Skip("Postgres not available, skipping test")
+	}
+
+	setupLogsTableForGINIndexTest(t, db)
+	ctx := context.Background()
+
+	// Insert a log with valid JSON
+	validJSON := `{"test": "idempotent"}`
+	insertTestLog(t, db, "log-idempotent", &validJSON)
+
+	// Run the migration (cleanup only) then ensure the index is built.
+	err := migrationAddMetadataGINIndex(ctx, db)
+	require.NoError(t, err, "First migration should succeed")
+
+
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("Failed to get SQL DB: %v", err)
+	}
+	conn, err := sqlDB.Conn(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to get SQL connection: %v", err)
+	}
+	t.Cleanup(func() { _ = conn.Close() })
+
+	err = ensureMetadataGINIndex(ctx, conn)
+	require.NoError(t, err, "GIN index creation should succeed")
+
+	// Verify index exists
+	assert.True(t, indexExists(t, db, "idx_logs_metadata_gin"), "GIN index should exist after first migration")
+
+	// Verify metadata is preserved
+	meta1 := getMetadataValue(t, db, "log-idempotent")
+	assert.NotNil(t, meta1)
+	assert.Equal(t, validJSON, *meta1)
+
+	// Run the migration second time (should be idempotent due to gomigrate tracking)
+	err = migrationAddMetadataGINIndex(ctx, db)
+	require.NoError(t, err, "Second migration should succeed (idempotent)")
+	err = ensureMetadataGINIndex(ctx, conn)
+	require.NoError(t, err, "ensureMetadataGINIndex should be a no-op when index already exists")
+
+	// Verify index still exists
+	assert.True(t, indexExists(t, db, "idx_logs_metadata_gin"), "GIN index should exist after second migration")
+
+	// Verify metadata is still preserved
+	meta2 := getMetadataValue(t, db, "log-idempotent")
+	assert.NotNil(t, meta2)
+	assert.Equal(t, validJSON, *meta2)
+}
+
+func TestMigrationAddMetadataGINIndex_EmptyTable(t *testing.T) {
+	db := trySetupPostgresDB(t)
+	if db == nil {
+		t.Skip("Postgres not available, skipping test")
+	}
+
+	setupLogsTableForGINIndexTest(t, db)
+	ctx := context.Background()
+
+	// Run the migration (cleanup only) then ensure the index is built.
+	err := migrationAddMetadataGINIndex(ctx, db)
+	require.NoError(t, err, "Migration should succeed on empty table")
+
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("Failed to get SQL DB: %v", err)
+	}
+	conn, err := sqlDB.Conn(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to get SQL connection: %v", err)
+	}
+	t.Cleanup(func() { _ = conn.Close() })
+
+	err = ensureMetadataGINIndex(ctx, conn)
+	require.NoError(t, err, "GIN index creation should succeed on empty table")
+
+	// Verify the GIN index was created
+	assert.True(t, indexExists(t, db, "idx_logs_metadata_gin"), "GIN index should be created even on empty table")
+}
+
+func TestMigrationAddMetadataGINIndex_EdgeCases(t *testing.T) {
+	db := trySetupPostgresDB(t)
+	if db == nil {
+		t.Skip("Postgres not available, skipping test")
+	}
+
+	setupLogsTableForGINIndexTest(t, db)
+	ctx := context.Background()
+
+	// Test edge cases that might be tricky (only JSON objects are supported)
+	emptyObject := `{}`
+	emptyArray := `[]`                       // Not a JSON object, should be nullified
+	whitespaceJSON := `  {"key": "value"}  ` // Valid JSON with surrounding whitespace
+	unicodeJSON := `{"emoji": "🎉", "chinese": "中文"}`
+	largeNumber := `{"bignum": 99999999999999999999}`
+	scientificNotation := `{"sci": 1.23e10}`
+
+	insertTestLog(t, db, "log-edge-empty-obj", &emptyObject)
+	insertTestLog(t, db, "log-edge-empty-arr", &emptyArray)
+	insertTestLog(t, db, "log-edge-whitespace", &whitespaceJSON)
+	insertTestLog(t, db, "log-edge-unicode", &unicodeJSON)
+	insertTestLog(t, db, "log-edge-large-num", &largeNumber)
+	insertTestLog(t, db, "log-edge-scientific", &scientificNotation)
+
+	// Run the migration (cleanup only) then ensure the index is built.
+	err := migrationAddMetadataGINIndex(ctx, db)
+	require.NoError(t, err, "Migration should succeed")
+
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("Failed to get SQL DB: %v", err)
+	}
+	conn, err := sqlDB.Conn(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to get SQL connection: %v", err)
+	}
+	t.Cleanup(func() { _ = conn.Close() })
+
+	err = ensureMetadataGINIndex(ctx, conn)
+	require.NoError(t, err, "GIN index creation should succeed")
+
+	// Verify all edge cases are handled correctly
+	// Empty object should be preserved, but empty array is not a JSON object
+	assert.NotNil(t, getMetadataValue(t, db, "log-edge-empty-obj"), "Empty object should be preserved")
+	assert.Nil(t, getMetadataValue(t, db, "log-edge-empty-arr"), "Empty array should be nullified (not a JSON object)")
+
+	// Whitespace JSON should be preserved (Postgres handles it)
+	meta := getMetadataValue(t, db, "log-edge-whitespace")
+	assert.NotNil(t, meta, "Whitespace JSON object should be preserved")
+
+	// Unicode should be preserved
+	assert.NotNil(t, getMetadataValue(t, db, "log-edge-unicode"), "Unicode JSON object should be preserved")
+
+	// Large numbers and scientific notation should be preserved
+	assert.NotNil(t, getMetadataValue(t, db, "log-edge-large-num"), "Large number JSON object should be preserved")
+	assert.NotNil(t, getMetadataValue(t, db, "log-edge-scientific"), "Scientific notation JSON object should be preserved")
+
+	// Verify the GIN index was created
+	assert.True(t, indexExists(t, db, "idx_logs_metadata_gin"), "GIN index should be created")
+}

framework/logstore/payload.go

@@ -0,0 +1,618 @@
+package logstore
+
+import (
+	"fmt"
+	"time"
+	"unicode/utf8"
+
+	"github.com/bytedance/sonic"
+	"github.com/maximhq/bifrost/core/schemas"
+)
+
+// payloadFields lists the DB column names of large TEXT fields that are
+// offloaded to object storage in hybrid mode. These fields are never needed
+// for analytics queries (histograms, search, rankings) — only for individual
+// log detail views (FindByID).
+var payloadFields = []string{
+	"input_history",
+	"responses_input_history",
+	"output_message",
+	"responses_output",
+	"embedding_output",
+	"rerank_output",
+	"ocr_input",
+	"ocr_output",
+	"params",
+	"tools",
+	"tool_calls",
+	"speech_input",
+	"transcription_input",
+	"image_generation_input",
+	"image_edit_input",
+	"image_variation_input",
+	"video_generation_input",
+	"speech_output",
+	"transcription_output",
+	"image_generation_output",
+	"list_models_output",
+	"video_generation_output",
+	"video_retrieve_output",
+	"video_download_output",
+	"video_list_output",
+	"video_delete_output",
+	"cache_debug",
+	"token_usage",
+	"error_details",
+	"raw_request",
+	"raw_response",
+	"passthrough_request_body",
+	"passthrough_response_body",
+	"routing_engine_logs",
+}
+
+// ExtractPayload reads the serialized TEXT payload fields from a Log into a map.
+// The map keys are the DB column names.
+func ExtractPayload(l *Log) map[string]string {
+	m := make(map[string]string, len(payloadFields))
+	m["input_history"] = l.InputHistory
+	m["responses_input_history"] = l.ResponsesInputHistory
+	m["output_message"] = l.OutputMessage
+	m["responses_output"] = l.ResponsesOutput
+	m["embedding_output"] = l.EmbeddingOutput
+	m["rerank_output"] = l.RerankOutput
+	m["ocr_input"] = l.OCRInput
+	m["ocr_output"] = l.OCROutput
+	m["params"] = l.Params
+	m["tools"] = l.Tools
+	m["tool_calls"] = l.ToolCalls
+	m["speech_input"] = l.SpeechInput
+	m["transcription_input"] = l.TranscriptionInput
+	m["image_generation_input"] = l.ImageGenerationInput
+	m["image_edit_input"] = l.ImageEditInput
+	m["image_variation_input"] = l.ImageVariationInput
+	m["video_generation_input"] = l.VideoGenerationInput
+	m["speech_output"] = l.SpeechOutput
+	m["transcription_output"] = l.TranscriptionOutput
+	m["image_generation_output"] = l.ImageGenerationOutput
+	m["list_models_output"] = l.ListModelsOutput
+	m["video_generation_output"] = l.VideoGenerationOutput
+	m["video_retrieve_output"] = l.VideoRetrieveOutput
+	m["video_download_output"] = l.VideoDownloadOutput
+	m["video_list_output"] = l.VideoListOutput
+	m["video_delete_output"] = l.VideoDeleteOutput
+	m["cache_debug"] = l.CacheDebug
+	m["token_usage"] = l.TokenUsage
+	m["error_details"] = l.ErrorDetails
+	m["raw_request"] = l.RawRequest
+	m["raw_response"] = l.RawResponse
+	m["passthrough_request_body"] = l.PassthroughRequestBody
+	m["passthrough_response_body"] = l.PassthroughResponseBody
+	m["routing_engine_logs"] = l.RoutingEngineLogs
+	return m
+}
+
+// ClearPayload zeros out both the TEXT payload columns and the Parsed virtual
+// fields on a Log struct. Clearing the Parsed fields is necessary to prevent
+// GORM's BeforeCreate/SerializeFields from re-populating TEXT columns.
+// After calling this, the struct only contains index-weight data suitable
+// for a lightweight DB INSERT.
+func ClearPayload(l *Log) {
+	// Clear serialized TEXT columns.
+	l.InputHistory = ""
+	l.ResponsesInputHistory = ""
+	l.OutputMessage = ""
+	l.ResponsesOutput = ""
+	l.EmbeddingOutput = ""
+	l.RerankOutput = ""
+	l.OCRInput = ""
+	l.OCROutput = ""
+	l.Params = ""
+	l.Tools = ""
+	l.ToolCalls = ""
+	l.SpeechInput = ""
+	l.TranscriptionInput = ""
+	l.ImageGenerationInput = ""
+	l.ImageEditInput = ""
+	l.ImageVariationInput = ""
+	l.VideoGenerationInput = ""
+	l.SpeechOutput = ""
+	l.TranscriptionOutput = ""
+	l.ImageGenerationOutput = ""
+	l.ListModelsOutput = ""
+	l.VideoGenerationOutput = ""
+	l.VideoRetrieveOutput = ""
+	l.VideoDownloadOutput = ""
+	l.VideoListOutput = ""
+	l.VideoDeleteOutput = ""
+	l.CacheDebug = ""
+	l.TokenUsage = ""
+	l.ErrorDetails = ""
+	l.RawRequest = ""
+	l.RawResponse = ""
+	l.PassthroughRequestBody = ""
+	l.PassthroughResponseBody = ""
+	l.RoutingEngineLogs = ""
+
+	// Clear Parsed virtual fields so GORM's SerializeFields won't re-serialize them.
+	l.InputHistoryParsed = nil
+	l.ResponsesInputHistoryParsed = nil
+	l.OutputMessageParsed = nil
+	l.ResponsesOutputParsed = nil
+	l.EmbeddingOutputParsed = nil
+	l.RerankOutputParsed = nil
+	l.OCRInputParsed = nil
+	l.OCROutputParsed = nil
+	l.ParamsParsed = nil
+	l.ToolsParsed = nil
+	l.ToolCallsParsed = nil
+	l.SpeechInputParsed = nil
+	l.TranscriptionInputParsed = nil
+	l.ImageGenerationInputParsed = nil
+	l.ImageEditInputParsed = nil
+	l.ImageVariationInputParsed = nil
+	l.VideoGenerationInputParsed = nil
+	l.SpeechOutputParsed = nil
+	l.TranscriptionOutputParsed = nil
+	l.ImageGenerationOutputParsed = nil
+	l.ListModelsOutputParsed = nil
+	l.VideoGenerationOutputParsed = nil
+	l.VideoRetrieveOutputParsed = nil
+	l.VideoDownloadOutputParsed = nil
+	l.VideoListOutputParsed = nil
+	l.VideoDeleteOutputParsed = nil
+	l.CacheDebugParsed = nil
+	l.TokenUsageParsed = nil
+	l.ErrorDetailsParsed = nil
+}
+
+// MergePayloadFromJSON takes a JSON payload (as marshaled by MarshalPayload)
+// and merges the fields back into the Log struct's serialized TEXT columns,
+// then calls DeserializeFields to populate the Parsed virtual fields.
+func MergePayloadFromJSON(l *Log, data []byte) error {
+	var m map[string]string
+	if err := sonic.Unmarshal(data, &m); err != nil {
+		return fmt.Errorf("logstore: unmarshal payload: %w", err)
+	}
+	if v, ok := m["input_history"]; ok && v != "" {
+		l.InputHistory = v
+	}
+	if v, ok := m["responses_input_history"]; ok && v != "" {
+		l.ResponsesInputHistory = v
+	}
+	if v, ok := m["output_message"]; ok && v != "" {
+		l.OutputMessage = v
+	}
+	if v, ok := m["responses_output"]; ok && v != "" {
+		l.ResponsesOutput = v
+	}
+	if v, ok := m["embedding_output"]; ok && v != "" {
+		l.EmbeddingOutput = v
+	}
+	if v, ok := m["rerank_output"]; ok && v != "" {
+		l.RerankOutput = v
+	}
+	if v, ok := m["ocr_input"]; ok && v != "" {
+		l.OCRInput = v
+	}
+	if v, ok := m["ocr_output"]; ok && v != "" {
+		l.OCROutput = v
+	}
+	if v, ok := m["params"]; ok && v != "" {
+		l.Params = v
+	}
+	if v, ok := m["tools"]; ok && v != "" {
+		l.Tools = v
+	}
+	if v, ok := m["tool_calls"]; ok && v != "" {
+		l.ToolCalls = v
+	}
+	if v, ok := m["speech_input"]; ok && v != "" {
+		l.SpeechInput = v
+	}
+	if v, ok := m["transcription_input"]; ok && v != "" {
+		l.TranscriptionInput = v
+	}
+	if v, ok := m["image_generation_input"]; ok && v != "" {
+		l.ImageGenerationInput = v
+	}
+	if v, ok := m["image_edit_input"]; ok && v != "" {
+		l.ImageEditInput = v
+	}
+	if v, ok := m["image_variation_input"]; ok && v != "" {
+		l.ImageVariationInput = v
+	}
+	if v, ok := m["video_generation_input"]; ok && v != "" {
+		l.VideoGenerationInput = v
+	}
+	if v, ok := m["speech_output"]; ok && v != "" {
+		l.SpeechOutput = v
+	}
+	if v, ok := m["transcription_output"]; ok && v != "" {
+		l.TranscriptionOutput = v
+	}
+	if v, ok := m["image_generation_output"]; ok && v != "" {
+		l.ImageGenerationOutput = v
+	}
+	if v, ok := m["list_models_output"]; ok && v != "" {
+		l.ListModelsOutput = v
+	}
+	if v, ok := m["video_generation_output"]; ok && v != "" {
+		l.VideoGenerationOutput = v
+	}
+	if v, ok := m["video_retrieve_output"]; ok && v != "" {
+		l.VideoRetrieveOutput = v
+	}
+	if v, ok := m["video_download_output"]; ok && v != "" {
+		l.VideoDownloadOutput = v
+	}
+	if v, ok := m["video_list_output"]; ok && v != "" {
+		l.VideoListOutput = v
+	}
+	if v, ok := m["video_delete_output"]; ok && v != "" {
+		l.VideoDeleteOutput = v
+	}
+	if v, ok := m["cache_debug"]; ok && v != "" {
+		l.CacheDebug = v
+	}
+	if v, ok := m["token_usage"]; ok && v != "" {
+		l.TokenUsage = v
+	}
+	if v, ok := m["error_details"]; ok && v != "" {
+		l.ErrorDetails = v
+	}
+	if v, ok := m["raw_request"]; ok && v != "" {
+		l.RawRequest = v
+	}
+	if v, ok := m["raw_response"]; ok && v != "" {
+		l.RawResponse = v
+	}
+	if v, ok := m["passthrough_request_body"]; ok && v != "" {
+		l.PassthroughRequestBody = v
+	}
+	if v, ok := m["passthrough_response_body"]; ok && v != "" {
+		l.PassthroughResponseBody = v
+	}
+	if v, ok := m["routing_engine_logs"]; ok && v != "" {
+		l.RoutingEngineLogs = v
+	}
+	return l.DeserializeFields()
+}
+
+// MarshalPayload serializes the payload map (from ExtractPayload) to JSON.
+func MarshalPayload(payload map[string]string) ([]byte, error) {
+	return sonic.Marshal(payload)
+}
+
+// BuildInputContentSummary extracts the last user message text from input fields.
+// This is used in hybrid mode for the content_summary column, which powers
+// full-text search and serves as a display fallback in the log list table.
+// Only the last message is kept — the full conversation history lives in
+// object storage and is merged back on FindByID.
+func (l *Log) BuildInputContentSummary() string {
+	// Chat completions: last user message
+	if idx := findLastUserMessageIndex(l.InputHistoryParsed); idx >= 0 {
+		if text := extractChatMessageText(&l.InputHistoryParsed[idx]); text != "" {
+			return text
+		}
+	}
+
+	// Responses API: last user message
+	for i := len(l.ResponsesInputHistoryParsed) - 1; i >= 0; i-- {
+		if l.ResponsesInputHistoryParsed[i].Role != nil && *l.ResponsesInputHistoryParsed[i].Role == schemas.ResponsesInputMessageRoleUser {
+			if text := extractResponsesMessageText(&l.ResponsesInputHistoryParsed[i]); text != "" {
+				return text
+			}
+		}
+	}
+
+	// Speech input
+	if l.SpeechInputParsed != nil && l.SpeechInputParsed.Input != "" {
+		return l.SpeechInputParsed.Input
+	}
+
+	// Image generation input prompt
+	if l.ImageGenerationInputParsed != nil && l.ImageGenerationInputParsed.Prompt != "" {
+		return l.ImageGenerationInputParsed.Prompt
+	}
+
+	// Image edit input prompt
+	if l.ImageEditInputParsed != nil && l.ImageEditInputParsed.Prompt != "" {
+		return l.ImageEditInputParsed.Prompt
+	}
+
+	// Video generation input prompt
+	if l.VideoGenerationInputParsed != nil && l.VideoGenerationInputParsed.Prompt != "" {
+		return l.VideoGenerationInputParsed.Prompt
+	}
+
+	return ""
+}
+
+// extractChatMessageText returns the text content from a ChatMessage.
+// It prefers ContentStr; falls back to the last text ContentBlock.
+func extractChatMessageText(msg *schemas.ChatMessage) string {
+	if msg.Content == nil {
+		return ""
+	}
+	if msg.Content.ContentStr != nil && *msg.Content.ContentStr != "" {
+		return *msg.Content.ContentStr
+	}
+	if msg.Content.ContentBlocks != nil {
+		var lastText string
+		for _, block := range msg.Content.ContentBlocks {
+			if block.Text != nil && *block.Text != "" {
+				lastText = *block.Text
+			}
+		}
+		return lastText
+	}
+	return ""
+}
+
+// extractResponsesMessageText returns the text content from a ResponsesMessage.
+// It prefers ContentStr; falls back to the last text ContentBlock.
+func extractResponsesMessageText(msg *schemas.ResponsesMessage) string {
+	if msg.Content == nil {
+		return ""
+	}
+	if msg.Content.ContentStr != nil && *msg.Content.ContentStr != "" {
+		return *msg.Content.ContentStr
+	}
+	if msg.Content.ContentBlocks != nil {
+		var lastText string
+		for _, block := range msg.Content.ContentBlocks {
+			if block.Text != nil && *block.Text != "" {
+				lastText = *block.Text
+			}
+		}
+		return lastText
+	}
+	return ""
+}
+
+// findLastUserMessageIndex returns the index of the last ChatMessage with
+// role "user", or -1 if none exists. Used by both BuildInputContentSummary
+// and prepareDBEntry to avoid scanning the slice twice.
+func findLastUserMessageIndex(msgs []schemas.ChatMessage) int {
+	for i := len(msgs) - 1; i >= 0; i-- {
+		if msgs[i].Role == schemas.ChatMessageRoleUser {
+			return i
+		}
+	}
+	return -1
+}
+
+// BuildTags creates the S3 object tag map from a Log's index fields.
+// S3 allows max 10 tags per object; chosen for lifecycle rules and
+// S3 Metadata Tables queryability.
+func BuildTags(l *Log) map[string]string {
+	tags := make(map[string]string, 10)
+	if l.Provider != "" {
+		tags["provider"] = l.Provider
+	}
+	if l.Model != "" {
+		tags["model"] = truncateTag(l.Model, 256)
+	}
+	if l.Status != "" {
+		tags["status"] = l.Status
+	}
+	if l.Object != "" {
+		tags["object_type"] = l.Object
+	}
+	if l.VirtualKeyID != nil && *l.VirtualKeyID != "" {
+		tags["virtual_key_id"] = truncateTag(*l.VirtualKeyID, 256)
+	}
+	if l.SelectedKeyID != "" {
+		tags["selected_key_id"] = truncateTag(l.SelectedKeyID, 256)
+	}
+	if l.RoutingRuleID != nil && *l.RoutingRuleID != "" {
+		tags["routing_rule_id"] = truncateTag(*l.RoutingRuleID, 256)
+	}
+	if l.Stream {
+		tags["stream"] = "true"
+	} else {
+		tags["stream"] = "false"
+	}
+	tags["has_error"] = "false"
+	if l.Status == "error" {
+		tags["has_error"] = "true"
+	}
+	tags["date"] = l.Timestamp.UTC().Format("2006-01-02")
+	return tags
+}
+
+// ObjectKey constructs the S3 object key for a log entry.
+func ObjectKey(prefix string, timestamp time.Time, logID string) string {
+	ts := timestamp.UTC()
+	return fmt.Sprintf("%s/logs/%04d/%02d/%02d/%02d/%s.json.gz",
+		prefix,
+		ts.Year(), ts.Month(), ts.Day(), ts.Hour(),
+		logID,
+	)
+}
+
+// PayloadFieldNames returns the list of DB column names that are payload fields.
+func PayloadFieldNames() []string {
+	cp := make([]string, len(payloadFields))
+	copy(cp, payloadFields)
+	return cp
+}
+
+// payloadFieldSet is a set for O(1) lookup of payload field names.
+var payloadFieldSet = func() map[string]struct{} {
+	s := make(map[string]struct{}, len(payloadFields))
+	for _, f := range payloadFields {
+		s[f] = struct{}{}
+	}
+	return s
+}()
+
+// fieldsNeedHydration returns true if any of the requested fields are
+// payload fields that have been offloaded to object storage.
+func fieldsNeedHydration(fields []string) bool {
+	if len(fields) == 0 {
+		return true
+	}
+	for _, f := range fields {
+		if _, ok := payloadFieldSet[f]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+// ensureHydrationFields appends id, timestamp, and has_object to the
+// projection if not already present, so hydrateLog can function correctly.
+func ensureHydrationFields(fields []string) []string {
+	required := [3]string{"id", "timestamp", "has_object"}
+	have := make(map[string]struct{}, len(fields))
+	for _, f := range fields {
+		have[f] = struct{}{}
+	}
+	for _, r := range required {
+		if _, ok := have[r]; !ok {
+			fields = append(fields, r)
+		}
+	}
+	return fields
+}
+
+// pruneUnrequestedPayloadFields clears payload fields that were not in the
+// caller's field projection. This ensures hydration doesn't break projection
+// semantics by populating unrequested fields with large blobs.
+// A nil/empty requestedFields means "no projection" — everything is kept.
+func pruneUnrequestedPayloadFields(l *Log, requestedFields []string) {
+	if len(requestedFields) == 0 {
+		return
+	}
+	requested := make(map[string]struct{}, len(requestedFields))
+	for _, f := range requestedFields {
+		requested[f] = struct{}{}
+	}
+	for _, pf := range payloadFields {
+		if _, ok := requested[pf]; !ok {
+			clearPayloadField(l, pf)
+		}
+	}
+}
+
+// clearPayloadField zeros a single payload field (serialized TEXT column and
+// its Parsed counterpart, if any) by column name.
+func clearPayloadField(l *Log, name string) {
+	switch name {
+	case "input_history":
+		l.InputHistory = ""
+		l.InputHistoryParsed = nil
+	case "responses_input_history":
+		l.ResponsesInputHistory = ""
+		l.ResponsesInputHistoryParsed = nil
+	case "output_message":
+		l.OutputMessage = ""
+		l.OutputMessageParsed = nil
+	case "responses_output":
+		l.ResponsesOutput = ""
+		l.ResponsesOutputParsed = nil
+	case "embedding_output":
+		l.EmbeddingOutput = ""
+		l.EmbeddingOutputParsed = nil
+	case "rerank_output":
+		l.RerankOutput = ""
+		l.RerankOutputParsed = nil
+	case "ocr_input":
+		l.OCRInput = ""
+		l.OCRInputParsed = nil
+	case "ocr_output":
+		l.OCROutput = ""
+		l.OCROutputParsed = nil
+	case "params":
+		l.Params = ""
+		l.ParamsParsed = nil
+	case "tools":
+		l.Tools = ""
+		l.ToolsParsed = nil
+	case "tool_calls":
+		l.ToolCalls = ""
+		l.ToolCallsParsed = nil
+	case "speech_input":
+		l.SpeechInput = ""
+		l.SpeechInputParsed = nil
+	case "transcription_input":
+		l.TranscriptionInput = ""
+		l.TranscriptionInputParsed = nil
+	case "image_generation_input":
+		l.ImageGenerationInput = ""
+		l.ImageGenerationInputParsed = nil
+	case "image_edit_input":
+		l.ImageEditInput = ""
+		l.ImageEditInputParsed = nil
+	case "image_variation_input":
+		l.ImageVariationInput = ""
+		l.ImageVariationInputParsed = nil
+	case "video_generation_input":
+		l.VideoGenerationInput = ""
+		l.VideoGenerationInputParsed = nil
+	case "speech_output":
+		l.SpeechOutput = ""
+		l.SpeechOutputParsed = nil
+	case "transcription_output":
+		l.TranscriptionOutput = ""
+		l.TranscriptionOutputParsed = nil
+	case "image_generation_output":
+		l.ImageGenerationOutput = ""
+		l.ImageGenerationOutputParsed = nil
+	case "list_models_output":
+		l.ListModelsOutput = ""
+		l.ListModelsOutputParsed = nil
+	case "video_generation_output":
+		l.VideoGenerationOutput = ""
+		l.VideoGenerationOutputParsed = nil
+	case "video_retrieve_output":
+		l.VideoRetrieveOutput = ""
+		l.VideoRetrieveOutputParsed = nil
+	case "video_download_output":
+		l.VideoDownloadOutput = ""
+		l.VideoDownloadOutputParsed = nil
+	case "video_list_output":
+		l.VideoListOutput = ""
+		l.VideoListOutputParsed = nil
+	case "video_delete_output":
+		l.VideoDeleteOutput = ""
+		l.VideoDeleteOutputParsed = nil
+	case "cache_debug":
+		l.CacheDebug = ""
+		l.CacheDebugParsed = nil
+	case "token_usage":
+		l.TokenUsage = ""
+		l.TokenUsageParsed = nil
+	case "error_details":
+		l.ErrorDetails = ""
+		l.ErrorDetailsParsed = nil
+	case "raw_request":
+		l.RawRequest = ""
+	case "raw_response":
+		l.RawResponse = ""
+	case "passthrough_request_body":
+		l.PassthroughRequestBody = ""
+	case "passthrough_response_body":
+		l.PassthroughResponseBody = ""
+	case "routing_engine_logs":
+		l.RoutingEngineLogs = ""
+	}
+}
+
+// truncateTag ensures a tag value doesn't exceed the given max length.
+func truncateTag(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
+	}
+	// Truncate at a rune boundary without exceeding maxLen bytes.
+	byteLen := 0
+	for _, r := range s {
+		rl := utf8.RuneLen(r)
+		if byteLen+rl > maxLen {
+			break
+		}
+		byteLen += rl
+	}
+	return s[:byteLen]
+}

framework/logstore/payload_test.go

@@ -0,0 +1,156 @@
+package logstore
+
+import (
+	"testing"
+	"time"
+
+	"github.com/maximhq/bifrost/core/schemas"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExtractPayload_RoundTrip(t *testing.T) {
+	log := &Log{
+		ID:                      "test-1",
+		InputHistory:            `[{"role":"user","content":"hello"}]`,
+		ResponsesInputHistory:   `[{"role":"user","content":"hi"}]`,
+		OutputMessage:           `{"role":"assistant","content":"world"}`,
+		ResponsesOutput:         `[{"role":"assistant","content":"there"}]`,
+		EmbeddingOutput:         `[{"embedding":[0.1]}]`,
+		RerankOutput:            `[{"score":0.9}]`,
+		Params:                  `{"temperature":0.7}`,
+		Tools:                   `[{"name":"tool1"}]`,
+		ToolCalls:               `[{"id":"tc1"}]`,
+		SpeechInput:             `{"input":"text"}`,
+		TranscriptionInput:      `{"file":"test.mp3"}`,
+		ImageGenerationInput:    `{"prompt":"cat"}`,
+		ImageEditInput:          `{"prompt":"edit cat"}`,
+		ImageVariationInput:     `{"image":"base64img"}`,
+		VideoGenerationInput:    `{"prompt":"dog"}`,
+		SpeechOutput:            `{"audio":"base64"}`,
+		TranscriptionOutput:     `{"text":"hello"}`,
+		ImageGenerationOutput:   `{"url":"http://img"}`,
+		ListModelsOutput:        `[{"id":"model1"}]`,
+		VideoGenerationOutput:   `{"id":"vid1"}`,
+		VideoRetrieveOutput:     `{"status":"ready"}`,
+		VideoDownloadOutput:     `{"url":"http://vid"}`,
+		VideoListOutput:         `{"videos":[]}`,
+		VideoDeleteOutput:       `{"deleted":true}`,
+		CacheDebug:              `{"hit":true}`,
+		TokenUsage:              `{"total_tokens":100}`,
+		ErrorDetails:            `{"error":"bad"}`,
+		RawRequest:              `{"method":"POST"}`,
+		RawResponse:             `{"status":200}`,
+		PassthroughRequestBody:  `body-req`,
+		PassthroughResponseBody: `body-resp`,
+		RoutingEngineLogs:       `routing log`,
+	}
+
+	payload := ExtractPayload(log)
+	assert.Equal(t, len(payloadFields), len(payload), "payload map should have all payload fields")
+	assert.Equal(t, `[{"role":"user","content":"hello"}]`, payload["input_history"])
+	assert.Equal(t, `{"role":"assistant","content":"world"}`, payload["output_message"])
+	assert.Equal(t, `routing log`, payload["routing_engine_logs"])
+
+	// Clear and verify.
+	ClearPayload(log)
+	assert.Empty(t, log.InputHistory)
+	assert.Empty(t, log.OutputMessage)
+	assert.Empty(t, log.RawRequest)
+	assert.Empty(t, log.RoutingEngineLogs)
+
+	// Marshal and merge back.
+	data, err := MarshalPayload(payload)
+	require.NoError(t, err)
+
+	err = MergePayloadFromJSON(log, data)
+	require.NoError(t, err)
+	assert.Equal(t, `[{"role":"user","content":"hello"}]`, log.InputHistory)
+	assert.Equal(t, `{"role":"assistant","content":"world"}`, log.OutputMessage)
+	assert.Equal(t, `routing log`, log.RoutingEngineLogs)
+}
+
+func TestClearPayload_DoesNotTouchIndexFields(t *testing.T) {
+	log := &Log{
+		ID:           "test-1",
+		Provider:     "anthropic",
+		Model:        "claude-3",
+		Status:       "success",
+		InputHistory: `[{"role":"user","content":"hello"}]`,
+	}
+	ClearPayload(log)
+	assert.Equal(t, "test-1", log.ID)
+	assert.Equal(t, "anthropic", log.Provider)
+	assert.Equal(t, "claude-3", log.Model)
+	assert.Equal(t, "success", log.Status)
+	assert.Empty(t, log.InputHistory)
+}
+
+func TestBuildInputContentSummary(t *testing.T) {
+	content := "What is the weather?"
+	log := &Log{
+		InputHistoryParsed: []schemas.ChatMessage{
+			{Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: &content}},
+		},
+		OutputMessageParsed: &schemas.ChatMessage{
+			Content: &schemas.ChatMessageContent{ContentStr: strPtr("It's sunny")},
+		},
+	}
+
+	summary := log.BuildInputContentSummary()
+	assert.Contains(t, summary, "What is the weather?")
+	assert.NotContains(t, summary, "It's sunny", "BuildInputContentSummary should not include output")
+}
+
+func TestBuildTags(t *testing.T) {
+	vkID := "vk_123"
+	rrID := "rr_456"
+	log := &Log{
+		Provider:      "anthropic",
+		Model:         "claude-3-sonnet",
+		Status:        "success",
+		Object:        "chat.completion",
+		VirtualKeyID:  &vkID,
+		SelectedKeyID: "sk_789",
+		RoutingRuleID: &rrID,
+		Stream:        true,
+		Timestamp:     time.Date(2026, 4, 3, 14, 0, 0, 0, time.UTC),
+	}
+
+	tags := BuildTags(log)
+	assert.Equal(t, "anthropic", tags["provider"])
+	assert.Equal(t, "claude-3-sonnet", tags["model"])
+	assert.Equal(t, "success", tags["status"])
+	assert.Equal(t, "chat.completion", tags["object_type"])
+	assert.Equal(t, "vk_123", tags["virtual_key_id"])
+	assert.Equal(t, "sk_789", tags["selected_key_id"])
+	assert.Equal(t, "rr_456", tags["routing_rule_id"])
+	assert.Equal(t, "true", tags["stream"])
+	assert.Equal(t, "false", tags["has_error"])
+	assert.Equal(t, "2026-04-03", tags["date"])
+	assert.LessOrEqual(t, len(tags), 10, "S3 allows max 10 tags")
+}
+
+func TestBuildTags_ErrorStatus(t *testing.T) {
+	log := &Log{Status: "error", Timestamp: time.Now()}
+	tags := BuildTags(log)
+	assert.Equal(t, "true", tags["has_error"])
+}
+
+func TestObjectKey(t *testing.T) {
+	ts := time.Date(2026, 4, 3, 14, 0, 0, 0, time.UTC)
+	key := ObjectKey("bifrost", ts, "req_abc123")
+	assert.Equal(t, "bifrost/logs/2026/04/03/14/req_abc123.json.gz", key)
+}
+
+func TestPayloadFieldNames(t *testing.T) {
+	fields := PayloadFieldNames()
+	assert.True(t, len(fields) > 0)
+	// Verify it's a copy.
+	fields[0] = "modified"
+	assert.NotEqual(t, "modified", payloadFields[0])
+}
+
+func strPtr(s string) *string {
+	return &s
+}

framework/logstore/postgres.go

@@ -0,0 +1,189 @@
+package logstore
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/maximhq/bifrost/core/schemas"
+
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+// PostgresConfig represents the configuration for a Postgres database.
+type PostgresConfig struct {
+	Host         *schemas.EnvVar `json:"host"`
+	Port         *schemas.EnvVar `json:"port"`
+	User         *schemas.EnvVar `json:"user"`
+	Password     *schemas.EnvVar `json:"password"`
+	DBName       *schemas.EnvVar `json:"db_name"`
+	SSLMode      *schemas.EnvVar `json:"ssl_mode"`
+	MaxIdleConns int             `json:"max_idle_conns"`
+	MaxOpenConns int             `json:"max_open_conns"`
+}
+
+// newPostgresLogStore creates a new Postgres log store.
+//
+// Uses a two-pool lifecycle to avoid SQLSTATE 0A000 ("cached plan must not
+// change result type"): a throwaway pool runs the version check and schema
+// migrations and is closed immediately, then a fresh runtime pool is opened
+// for query traffic and the async index / matview builders. The runtime
+// pool's connections never see pre-migration schema, so their cached
+// prepared-plans stay valid for the life of the process.
+func newPostgresLogStore(ctx context.Context, config *PostgresConfig, logger schemas.Logger) (LogStore, error) {
+	if config == nil {
+		return nil, fmt.Errorf("config is required")
+	}
+	// Validate required config
+	if config.Host == nil || config.Host.GetValue() == "" {
+		return nil, fmt.Errorf("postgres host is required")
+	}
+	if config.Port == nil || config.Port.GetValue() == "" {
+		return nil, fmt.Errorf("postgres port is required")
+	}
+	if config.User == nil || config.User.GetValue() == "" {
+		return nil, fmt.Errorf("postgres user is required")
+	}
+	if config.Password == nil || config.Password.GetValue() == "" {
+		return nil, fmt.Errorf("postgres password is required")
+	}
+	if config.DBName == nil || config.DBName.GetValue() == "" {
+		return nil, fmt.Errorf("postgres db name is required")
+	}
+	if config.SSLMode == nil || config.SSLMode.GetValue() == "" {
+		return nil, fmt.Errorf("postgres ssl mode is required")
+	}
+	dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", config.Host.GetValue(), config.Port.GetValue(), config.User.GetValue(), config.Password.GetValue(), config.DBName.GetValue(), config.SSLMode.GetValue())
+
+	openPool := func() (*gorm.DB, error) {
+		return gorm.Open(postgres.New(postgres.Config{DSN: dsn}), &gorm.Config{
+			Logger: newGormLogger(logger),
+		})
+	}
+
+	// closePoolStrict returns the close error so callers can abort startup
+	// when the throwaway migration pool doesn't tear down cleanly — a half-
+	// closed pool weakens the guarantee that no cached plans survive DDL.
+	closePool := func(db *gorm.DB) error {
+		if db == nil {
+			return nil
+		}
+		sqlDB, err := db.DB()
+		if err != nil {
+			return err
+		}
+		return sqlDB.Close()
+	}
+
+	// Throwaway pool for the version gate and schema migrations. Closing it
+	// before the runtime pool opens guarantees no cached plan survives DDL.
+	mDb, err := openPool()
+	if err != nil {
+		return nil, err
+	}
+
+	// Postgres version gate: refuse to start below 16 (matviews, partitioning,
+	// and some JSON operators we rely on depend on 16+).
+	var pgVersionNum int
+	if err := mDb.Raw("SELECT current_setting('server_version_num')::int").Scan(&pgVersionNum).Error; err != nil {
+		_ = closePool(mDb)
+		return nil, err
+	}
+	if pgVersionNum < 160000 {
+		_ = closePool(mDb)
+		return nil, fmt.Errorf("postgres version is lower than 16, please upgrade to 16 or higher")
+	}
+
+	if err := triggerMigrations(ctx, mDb); err != nil {
+		_ = closePool(mDb)
+		return nil, err
+	}
+	if err := closePool(mDb); err != nil {
+		return nil, fmt.Errorf("close migration db connection: %w", err)
+	}
+
+	// Runtime pool. Opens against post-migration schema.
+	db, err := openPool()
+	if err != nil {
+		return nil, err
+	}
+
+	// Configure connection pool
+	sqlDB, err := db.DB()
+	if err != nil {
+		closePool(db)
+		return nil, err
+	}
+	// Set MaxIdleConns (default: 5)
+	maxIdleConns := config.MaxIdleConns
+	if maxIdleConns == 0 {
+		maxIdleConns = 5
+	}
+	sqlDB.SetMaxIdleConns(maxIdleConns)
+
+	// Set MaxOpenConns (default: 50)
+	maxOpenConns := config.MaxOpenConns
+	if maxOpenConns == 0 {
+		maxOpenConns = 50
+	}
+	sqlDB.SetMaxOpenConns(maxOpenConns)
+	d := &RDBLogStore{db: db, logger: logger}
+
+	// Run all index builds sequentially in a single goroutine to prevent
+	// deadlocks from concurrent CREATE INDEX CONCURRENTLY on the same table.
+	// Each function is idempotent and acquires its own advisory lock for
+	// cross-node serialization. Running in a goroutine avoids blocking pod startup.
+	go func() {
+		if db.Dialector.Name() != "postgres" {
+			return
+		}
+		// Acquire advisory lock to serialize GIN index builds across cluster nodes.
+		lock, err := acquireIndexLock(context.Background(), db)
+		if err != nil {
+			// Lock is taken by another node, so we will skip the index build
+			return
+		}
+		defer lock.release(context.Background())
+
+		if err := ensureMetadataGINIndex(context.Background(), lock.conn); err != nil {
+			logger.Warn(fmt.Sprintf("logstore: metadata GIN index build failed: %s (queries will still work without the index)", err))
+		} else {
+			logger.Info("logstore: metadata GIN index is ready")
+		}
+
+		if err := ensureDashboardEnhancements(context.Background(), lock.conn); err != nil {
+			logger.Warn(fmt.Sprintf("logstore: dashboard enhancements failed: %s (dashboard will still work with partial data)", err))
+		} else {
+			logger.Info("logstore: dashboard enhancements completed")
+		}
+
+		if err := ensurePerformanceIndexes(context.Background(), lock.conn); err != nil {
+			logger.Warn(fmt.Sprintf("logstore: performance index build failed: %s (queries will still work without the indexes)", err))
+		} else {
+			logger.Info("logstore: performance indexes are ready")
+		}
+	}()
+
+	// Create materialized views and start periodic refresh for dashboard queries.
+	go func() {
+		if db.Dialector.Name() != "postgres" {
+			return
+		}
+		if err := ensureMatViews(context.Background(), db); err != nil {
+			logger.Warn(fmt.Sprintf("logstore: matview creation failed: %s (dashboard queries will use raw tables)", err))
+			return
+		}
+		if err := refreshMatViews(context.Background(), db); err != nil {
+			logger.Warn(fmt.Sprintf("logstore: initial matview refresh failed: %s", err))
+		} else {
+			logger.Info("logstore: materialized views are ready")
+			// Signal that matviews are ready for query use. Until this point,
+			// canUseMatView() returns false so all queries use raw tables.
+			d.matViewsReady.Store(true)
+		}
+		startMatViewRefresher(context.Background(), db, 30*time.Second, logger, &d.matViewsReady)
+	}()
+
+	return d, nil
+}

framework/logstore/rdb_perf_test.go

@@ -0,0 +1,260 @@
+package logstore
+
+import (
+	"context"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/maximhq/bifrost/core/schemas"
+)
+
+type testLogger struct{}
+
+func (testLogger) Debug(string, ...any)                   {}
+func (testLogger) Info(string, ...any)                    {}
+func (testLogger) Warn(string, ...any)                    {}
+func (testLogger) Error(string, ...any)                   {}
+func (testLogger) Fatal(string, ...any)                   {}
+func (testLogger) SetLevel(schemas.LogLevel)              {}
+func (testLogger) SetOutputType(schemas.LoggerOutputType) {}
+func (testLogger) LogHTTPRequest(schemas.LogLevel, string) schemas.LogEventBuilder {
+	return schemas.NoopLogEvent
+}
+
+func newTestSQLiteStore(t *testing.T) *RDBLogStore {
+	t.Helper()
+
+	store, err := newSqliteLogStore(context.Background(), &SQLiteConfig{
+		Path: filepath.Join(t.TempDir(), "logs.db"),
+	}, testLogger{})
+	if err != nil {
+		t.Fatalf("newSqliteLogStore() error = %v", err)
+	}
+	return store
+}
+
+func TestLogCreateSerializesFields(t *testing.T) {
+	store := newTestSQLiteStore(t)
+	prompt := "hello"
+	reply := "world"
+
+	entry := &Log{
+		ID:        "log-1",
+		Timestamp: time.Now().UTC(),
+		Object:    "chat_completion",
+		Provider:  "openai",
+		Model:     "gpt-4o-mini",
+		Status:    "success",
+		InputHistoryParsed: []schemas.ChatMessage{{
+			Role: schemas.ChatMessageRoleUser,
+			Content: &schemas.ChatMessageContent{
+				ContentStr: &prompt,
+			},
+		}},
+		OutputMessageParsed: &schemas.ChatMessage{
+			Role: schemas.ChatMessageRoleAssistant,
+			Content: &schemas.ChatMessageContent{
+				ContentStr: &reply,
+			},
+		},
+	}
+
+	if err := store.Create(context.Background(), entry); err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+
+	logEntry, err := store.FindByID(context.Background(), entry.ID)
+	if err != nil {
+		t.Fatalf("FindByID() error = %v", err)
+	}
+	if logEntry.InputHistory == "" {
+		t.Fatalf("expected InputHistory to be serialized")
+	}
+	if logEntry.OutputMessage == "" {
+		t.Fatalf("expected OutputMessage to be serialized")
+	}
+	if logEntry.ContentSummary == "" {
+		t.Fatalf("expected ContentSummary to be populated")
+	}
+	if logEntry.CreatedAt.IsZero() {
+		t.Fatalf("expected CreatedAt to be populated")
+	}
+}
+
+func TestMCPToolLogCreateSerializesFields(t *testing.T) {
+	store := newTestSQLiteStore(t)
+
+	entry := &MCPToolLog{
+		ID:        "mcp-1",
+		Timestamp: time.Now().UTC(),
+		ToolName:  "echo",
+		Status:    "success",
+		ArgumentsParsed: map[string]any{
+			"message": "hello",
+		},
+		ResultParsed: map[string]any{
+			"ok": true,
+		},
+	}
+
+	if err := store.CreateMCPToolLog(context.Background(), entry); err != nil {
+		t.Fatalf("CreateMCPToolLog() error = %v", err)
+	}
+
+	logEntry, err := store.FindMCPToolLog(context.Background(), entry.ID)
+	if err != nil {
+		t.Fatalf("FindMCPToolLog() error = %v", err)
+	}
+	if logEntry.Arguments == "" {
+		t.Fatalf("expected Arguments to be serialized")
+	}
+	if logEntry.Result == "" {
+		t.Fatalf("expected Result to be serialized")
+	}
+}
+
+func TestBuildBulkUpdateCostPostgresSQL(t *testing.T) {
+	updates := map[string]float64{
+		"log-a": 1.25,
+		"log-b": 2.5,
+	}
+
+	query, args := buildBulkUpdateCostPostgresSQL([]string{"log-a", "log-b"}, updates)
+	wantQuery := "UPDATE logs SET cost = v.cost FROM (VALUES ($1::text,$2::float8),($3::text,$4::float8)) AS v(id, cost) WHERE logs.id = v.id"
+	wantArgs := []interface{}{"log-a", 1.25, "log-b", 2.5}
+
+	if query != wantQuery {
+		t.Fatalf("query mismatch\n got: %s\nwant: %s", query, wantQuery)
+	}
+	if !reflect.DeepEqual(args, wantArgs) {
+		t.Fatalf("args mismatch\n got: %#v\nwant: %#v", args, wantArgs)
+	}
+}
+
+func TestUpdateSerializesStructEntry(t *testing.T) {
+	store := newTestSQLiteStore(t)
+	now := time.Now().UTC()
+	entry := &Log{
+		ID:        "log-update",
+		Timestamp: now,
+		Object:    "chat_completion",
+		Provider:  "openai",
+		Model:     "gpt-4o-mini",
+		Status:    "processing",
+	}
+
+	if err := store.Create(context.Background(), entry); err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+
+	reply := "updated response"
+	if err := store.Update(context.Background(), entry.ID, Log{
+		Status: "success",
+		OutputMessageParsed: &schemas.ChatMessage{
+			Role: schemas.ChatMessageRoleAssistant,
+			Content: &schemas.ChatMessageContent{
+				ContentStr: &reply,
+			},
+		},
+		TokenUsageParsed: &schemas.BifrostLLMUsage{
+			PromptTokens:     3,
+			CompletionTokens: 7,
+			TotalTokens:      10,
+		},
+	}); err != nil {
+		t.Fatalf("Update() error = %v", err)
+	}
+
+	logEntry, err := store.FindByID(context.Background(), entry.ID)
+	if err != nil {
+		t.Fatalf("FindByID() error = %v", err)
+	}
+	if logEntry.OutputMessage == "" {
+		t.Fatalf("expected OutputMessage to be serialized on Update")
+	}
+	if logEntry.TokenUsage == "" {
+		t.Fatalf("expected TokenUsage to be serialized on Update")
+	}
+	if logEntry.TotalTokens != 10 {
+		t.Fatalf("expected TotalTokens to be updated, got %d", logEntry.TotalTokens)
+	}
+}
+
+func TestUpdateMCPToolLogSerializesStructEntry(t *testing.T) {
+	store := newTestSQLiteStore(t)
+	now := time.Now().UTC()
+	entry := &MCPToolLog{
+		ID:        "mcp-update",
+		Timestamp: now,
+		ToolName:  "echo",
+		Status:    "processing",
+	}
+
+	if err := store.CreateMCPToolLog(context.Background(), entry); err != nil {
+		t.Fatalf("CreateMCPToolLog() error = %v", err)
+	}
+
+	if err := store.UpdateMCPToolLog(context.Background(), entry.ID, MCPToolLog{
+		Status: "success",
+		ResultParsed: map[string]any{
+			"message": "done",
+		},
+	}); err != nil {
+		t.Fatalf("UpdateMCPToolLog() error = %v", err)
+	}
+
+	logEntry, err := store.FindMCPToolLog(context.Background(), entry.ID)
+	if err != nil {
+		t.Fatalf("FindMCPToolLog() error = %v", err)
+	}
+	if logEntry.Result == "" {
+		t.Fatalf("expected Result to be serialized on UpdateMCPToolLog")
+	}
+}
+
+func TestBulkUpdateCostSQLiteFallback(t *testing.T) {
+	store := newTestSQLiteStore(t)
+	now := time.Now().UTC()
+	entries := []*Log{
+		{
+			ID:        "log-a",
+			Timestamp: now,
+			Object:    "chat_completion",
+			Provider:  "openai",
+			Model:     "gpt-4o-mini",
+			Status:    "success",
+		},
+		{
+			ID:        "log-b",
+			Timestamp: now,
+			Object:    "chat_completion",
+			Provider:  "openai",
+			Model:     "gpt-4o-mini",
+			Status:    "success",
+		},
+	}
+	for _, entry := range entries {
+		if err := store.Create(context.Background(), entry); err != nil {
+			t.Fatalf("Create(%s) error = %v", entry.ID, err)
+		}
+	}
+
+	if err := store.BulkUpdateCost(context.Background(), map[string]float64{
+		"log-a": 1.5,
+		"log-b": 2.5,
+	}); err != nil {
+		t.Fatalf("BulkUpdateCost() error = %v", err)
+	}
+
+	for id, wantCost := range map[string]float64{"log-a": 1.5, "log-b": 2.5} {
+		logEntry, err := store.FindByID(context.Background(), id)
+		if err != nil {
+			t.Fatalf("FindByID(%s) error = %v", id, err)
+		}
+		if logEntry.Cost == nil || *logEntry.Cost != wantCost {
+			t.Fatalf("cost mismatch for %s: got %v want %v", id, logEntry.Cost, wantCost)
+		}
+	}
+}

framework/logstore/rdb_postgres_perf_test.go

@@ -0,0 +1,585 @@
+package logstore
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
+)
+
+// setupPerfTestDB connects to Postgres, runs migrations, and returns the store.
+func setupPerfTestDB(t *testing.T) (*RDBLogStore, *gorm.DB) {
+	t.Helper()
+	db := trySetupPostgresDB(t)
+	if db == nil {
+		t.Skip("Postgres not available, skipping test")
+	}
+
+	// Clean slate — drop test-owned tables but preserve the shared migrations
+	// table so concurrent test packages (e.g. configstore) are not disrupted.
+	db.Exec("DROP MATERIALIZED VIEW IF EXISTS mv_logs_hourly CASCADE")
+	db.Exec("DROP MATERIALIZED VIEW IF EXISTS mv_logs_filterdata CASCADE")
+	db.Exec("DROP TABLE IF EXISTS mcp_tool_logs CASCADE")
+	db.Exec("DROP TABLE IF EXISTS async_jobs CASCADE")
+	db.Exec("DROP TABLE IF EXISTS logs CASCADE")
+	db.Exec("CREATE TABLE IF NOT EXISTS migrations (id VARCHAR(255) PRIMARY KEY)")
+	db.Exec("DELETE FROM migrations")
+
+	ctx := context.Background()
+	err := triggerMigrations(ctx, db)
+	require.NoError(t, err, "migrations should succeed")
+
+	err = ensureMatViews(ctx, db)
+	require.NoError(t, err, "matview creation should succeed")
+
+	store := &RDBLogStore{db: db}
+
+	t.Cleanup(func() {
+		for _, idx := range performanceIndexes {
+			db.Exec("DROP INDEX IF EXISTS " + idx.name)
+		}
+		db.Exec("DROP MATERIALIZED VIEW IF EXISTS mv_logs_hourly CASCADE")
+		db.Exec("DROP MATERIALIZED VIEW IF EXISTS mv_logs_filterdata CASCADE")
+		db.Exec("DROP TABLE IF EXISTS mcp_tool_logs CASCADE")
+		db.Exec("DROP TABLE IF EXISTS async_jobs CASCADE")
+		db.Exec("DROP TABLE IF EXISTS logs CASCADE")
+		db.Exec("DELETE FROM migrations")
+	})
+
+	return store, db
+}
+
+// acquirePerfTestSQLConn returns a dedicated connection for ensurePerformanceIndexes (CONCURRENTLY + session SET).
+func acquirePerfTestSQLConn(t *testing.T, ctx context.Context, db *gorm.DB) *sql.Conn {
+	t.Helper()
+	sqlDB, err := db.DB()
+	require.NoError(t, err)
+	conn, err := sqlDB.Conn(ctx)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = conn.Close() })
+	return conn
+}
+
+type logOpts struct {
+	Model              string
+	Provider           string
+	Status             string
+	Timestamp          time.Time
+	RoutingEnginesUsed string
+	Metadata           string
+	ContentSummary     string
+	VirtualKeyID       string
+	VirtualKeyName     string
+	SelectedKeyID      string
+	SelectedKeyName    string
+	RoutingRuleID      string
+	RoutingRuleName    string
+}
+
+func insertPerfLog(t *testing.T, db *gorm.DB, opts logOpts) {
+	t.Helper()
+	if opts.Provider == "" {
+		opts.Provider = "openai"
+	}
+	if opts.Status == "" {
+		opts.Status = "success"
+	}
+	if opts.Model == "" {
+		opts.Model = "gpt-4"
+	}
+	id := uuid.New().String()
+	err := db.Exec(`
+		INSERT INTO logs (id, timestamp, object_type, provider, model, status,
+			routing_engines_used, metadata, content_summary,
+			virtual_key_id, virtual_key_name, selected_key_id, selected_key_name,
+			routing_rule_id, routing_rule_name, created_at, latency, cost,
+			prompt_tokens, completion_tokens, total_tokens)
+		VALUES (?, ?, 'chat_completion', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 100, 0.01, 10, 5, 15)
+	`, id, opts.Timestamp, opts.Provider, opts.Model, opts.Status,
+		opts.RoutingEnginesUsed, opts.Metadata, opts.ContentSummary,
+		opts.VirtualKeyID, opts.VirtualKeyName, opts.SelectedKeyID, opts.SelectedKeyName,
+		opts.RoutingRuleID, opts.RoutingRuleName, opts.Timestamp).Error
+	require.NoError(t, err, "Failed to insert test log")
+}
+
+type mcpLogOpts struct {
+	ToolName       string
+	ServerLabel    string
+	Timestamp      time.Time
+	VirtualKeyID   string
+	VirtualKeyName string
+	Arguments      string
+	Result         string
+}
+
+func insertPerfMCPLog(t *testing.T, db *gorm.DB, opts mcpLogOpts) {
+	t.Helper()
+	id := uuid.New().String()
+	err := db.Exec(`
+		INSERT INTO mcp_tool_logs (id, llm_request_id, tool_name, server_label,
+			timestamp, status, latency, cost,
+			virtual_key_id, virtual_key_name, arguments, result, created_at)
+		VALUES (?, ?, ?, ?, ?, 'success', 50, 0.001, ?, ?, ?, ?, ?)
+	`, id, uuid.New().String(), opts.ToolName, opts.ServerLabel,
+		opts.Timestamp, opts.VirtualKeyID, opts.VirtualKeyName,
+		opts.Arguments, opts.Result, opts.Timestamp).Error
+	require.NoError(t, err, "Failed to insert MCP test log")
+}
+
+// refreshTestMatViews refreshes materialized views after inserting test data.
+// This is needed because matviews are populated at creation time and don't
+// automatically reflect new inserts until explicitly refreshed.
+func refreshTestMatViews(t *testing.T, db *gorm.DB) {
+	t.Helper()
+	ctx := context.Background()
+	err := refreshMatViews(ctx, db)
+	require.NoError(t, err, "Failed to refresh materialized views")
+}
+
+// ---------- Phase 1: Defensive Limits ----------
+
+func TestSearchLogs_LimitClamping(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+
+	for i := 0; i < 5; i++ {
+		insertPerfLog(t, db, logOpts{Timestamp: now})
+	}
+	refreshTestMatViews(t, db)
+
+	// Limit=0 should be clamped (not return 0 results)
+	result, err := store.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: 0})
+	require.NoError(t, err)
+	assert.Equal(t, 5, len(result.Logs), "Limit=0 should be clamped")
+
+	// Limit=2 should return 2
+	result, err = store.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: 2})
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(result.Logs))
+
+	// Limit=-1 should be clamped
+	result, err = store.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: -1})
+	require.NoError(t, err)
+	assert.Equal(t, 5, len(result.Logs), "Limit=-1 should be clamped")
+
+	// Limit=2000 should be clamped to 1000
+	result, err = store.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: 2000})
+	require.NoError(t, err)
+	assert.Equal(t, 5, len(result.Logs))
+}
+
+func TestSearchMCPToolLogs_LimitClamping(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+
+	for i := 0; i < 5; i++ {
+		insertPerfMCPLog(t, db, mcpLogOpts{
+			ToolName: "search", ServerLabel: "s1", Timestamp: now,
+			VirtualKeyID: "vk-1", VirtualKeyName: "key-1",
+		})
+	}
+
+	result, err := store.SearchMCPToolLogs(ctx, MCPToolLogSearchFilters{}, PaginationOptions{Limit: 0})
+	require.NoError(t, err)
+	assert.Equal(t, 5, len(result.Logs), "Limit=0 should be clamped")
+
+	result, err = store.SearchMCPToolLogs(ctx, MCPToolLogSearchFilters{}, PaginationOptions{Limit: 3})
+	require.NoError(t, err)
+	assert.Equal(t, 3, len(result.Logs))
+}
+
+func TestGetModelRankings_HasLimit(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	start := now.Add(-1 * time.Hour)
+
+	for i := 0; i < 5; i++ {
+		insertPerfLog(t, db, logOpts{
+			Model: fmt.Sprintf("model-%d", i), Timestamp: now,
+		})
+	}
+	refreshTestMatViews(t, db)
+
+	result, err := store.GetModelRankings(ctx, SearchFilters{StartTime: &start, EndTime: &now})
+	require.NoError(t, err)
+	assert.LessOrEqual(t, len(result.Rankings), defaultMaxRankingsLimit)
+	assert.Equal(t, 5, len(result.Rankings))
+}
+
+func TestDeleteExpiredAsyncJobs_BatchDeletes(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	past := time.Now().UTC().Add(-1 * time.Hour)
+
+	for i := 0; i < 5; i++ {
+		err := db.Exec(`
+			INSERT INTO async_jobs (id, status, request_type, virtual_key_id, expires_at, created_at)
+			VALUES (?, 'completed', 'chat_completion', 'vk-1', ?, ?)
+		`, uuid.New().String(), past, past).Error
+		require.NoError(t, err)
+	}
+
+	deleted, err := store.DeleteExpiredAsyncJobs(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, int64(5), deleted)
+
+	var count int64
+	db.Model(&AsyncJob{}).Count(&count)
+	assert.Equal(t, int64(0), count)
+}
+
+// ---------- Phase 2: Time-scoped filter data ----------
+
+func TestGetDistinctModels_TimeCutoff(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	recent := now.Add(-7 * 24 * time.Hour)
+	old := now.Add(-60 * 24 * time.Hour)
+
+	insertPerfLog(t, db, logOpts{Model: "recent-model", Timestamp: recent})
+	insertPerfLog(t, db, logOpts{Model: "old-model", Timestamp: old})
+	refreshTestMatViews(t, db)
+
+	models, err := store.GetDistinctModels(ctx)
+	require.NoError(t, err)
+	assert.Contains(t, models, "recent-model")
+	assert.NotContains(t, models, "old-model")
+}
+
+func TestGetDistinctKeyPairs_TimeCutoff(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	recent := now.Add(-7 * 24 * time.Hour)
+	old := now.Add(-60 * 24 * time.Hour)
+
+	insertPerfLog(t, db, logOpts{
+		Timestamp: recent, VirtualKeyID: "vk-recent", VirtualKeyName: "Recent Key",
+	})
+	insertPerfLog(t, db, logOpts{
+		Timestamp: old, VirtualKeyID: "vk-old", VirtualKeyName: "Old Key",
+	})
+	refreshTestMatViews(t, db)
+
+	pairs, err := store.GetDistinctKeyPairs(ctx, "virtual_key_id", "virtual_key_name")
+	require.NoError(t, err)
+
+	var ids []string
+	for _, p := range pairs {
+		ids = append(ids, p.ID)
+	}
+	assert.Contains(t, ids, "vk-recent")
+	assert.NotContains(t, ids, "vk-old")
+}
+
+func TestGetDistinctRoutingEngines_TimeCutoff(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	recent := now.Add(-7 * 24 * time.Hour)
+	old := now.Add(-60 * 24 * time.Hour)
+
+	insertPerfLog(t, db, logOpts{
+		Timestamp: recent, RoutingEnginesUsed: "loadbalancing,governance",
+	})
+	insertPerfLog(t, db, logOpts{
+		Timestamp: old, RoutingEnginesUsed: "routing-rule",
+	})
+	refreshTestMatViews(t, db)
+
+	engines, err := store.GetDistinctRoutingEngines(ctx)
+	require.NoError(t, err)
+	assert.Contains(t, engines, "loadbalancing")
+	assert.Contains(t, engines, "governance")
+	assert.NotContains(t, engines, "routing-rule")
+}
+
+func TestGetDistinctMetadataKeys_TimeCutoff(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	recent := now.Add(-7 * 24 * time.Hour)
+	old := now.Add(-60 * 24 * time.Hour)
+
+	insertPerfLog(t, db, logOpts{
+		Timestamp: recent, Metadata: `{"env": "production"}`,
+	})
+	insertPerfLog(t, db, logOpts{
+		Timestamp: old, Metadata: `{"old_key": "old_value"}`,
+	})
+
+	keys, err := store.GetDistinctMetadataKeys(ctx)
+	require.NoError(t, err)
+	assert.Contains(t, keys, "env")
+	assert.NotContains(t, keys, "old_key")
+}
+
+func TestGetAvailableToolNames_TimeCutoff(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	recent := now.Add(-7 * 24 * time.Hour)
+	old := now.Add(-60 * 24 * time.Hour)
+
+	insertPerfMCPLog(t, db, mcpLogOpts{
+		ToolName: "recent-tool", ServerLabel: "s1", Timestamp: recent,
+		VirtualKeyID: "vk-1", VirtualKeyName: "k1",
+	})
+	insertPerfMCPLog(t, db, mcpLogOpts{
+		ToolName: "old-tool", ServerLabel: "s1", Timestamp: old,
+		VirtualKeyID: "vk-1", VirtualKeyName: "k1",
+	})
+
+	tools, err := store.GetAvailableToolNames(ctx)
+	require.NoError(t, err)
+	assert.Contains(t, tools, "recent-tool")
+	assert.NotContains(t, tools, "old-tool")
+}
+
+func TestGetAvailableServerLabels_TimeCutoff(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	recent := now.Add(-7 * 24 * time.Hour)
+	old := now.Add(-60 * 24 * time.Hour)
+
+	insertPerfMCPLog(t, db, mcpLogOpts{
+		ToolName: "t1", ServerLabel: "recent-server", Timestamp: recent,
+		VirtualKeyID: "vk-1", VirtualKeyName: "k1",
+	})
+	insertPerfMCPLog(t, db, mcpLogOpts{
+		ToolName: "t2", ServerLabel: "old-server", Timestamp: old,
+		VirtualKeyID: "vk-1", VirtualKeyName: "k1",
+	})
+
+	labels, err := store.GetAvailableServerLabels(ctx)
+	require.NoError(t, err)
+	assert.Contains(t, labels, "recent-server")
+	assert.NotContains(t, labels, "old-server")
+}
+
+func TestGetAvailableMCPVirtualKeys_TimeCutoff(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	recent := now.Add(-7 * 24 * time.Hour)
+	old := now.Add(-60 * 24 * time.Hour)
+
+	insertPerfMCPLog(t, db, mcpLogOpts{
+		ToolName: "t1", ServerLabel: "s1", Timestamp: recent,
+		VirtualKeyID: "vk-recent", VirtualKeyName: "Recent VK",
+	})
+	insertPerfMCPLog(t, db, mcpLogOpts{
+		ToolName: "t2", ServerLabel: "s1", Timestamp: old,
+		VirtualKeyID: "vk-old", VirtualKeyName: "Old VK",
+	})
+
+	keys, err := store.GetAvailableMCPVirtualKeys(ctx)
+	require.NoError(t, err)
+
+	var ids []string
+	for _, k := range keys {
+		if k.VirtualKeyID != nil {
+			ids = append(ids, *k.VirtualKeyID)
+		}
+	}
+	assert.Contains(t, ids, "vk-recent")
+	assert.NotContains(t, ids, "vk-old")
+}
+
+// ---------- Phase 3: Routing engine filter + indexes ----------
+
+func TestRoutingEngineFilter_Postgres(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	start := now.Add(-1 * time.Hour)
+
+	insertPerfLog(t, db, logOpts{
+		Model: "m1", Timestamp: now, RoutingEnginesUsed: "loadbalancing,governance",
+	})
+	insertPerfLog(t, db, logOpts{
+		Model: "m2", Timestamp: now, RoutingEnginesUsed: "routing-rule",
+	})
+	insertPerfLog(t, db, logOpts{
+		Model: "m3", Timestamp: now, RoutingEnginesUsed: "loadbalancing",
+	})
+
+	// Single engine filter
+	result, err := store.SearchLogs(ctx, SearchFilters{
+		RoutingEngineUsed: []string{"loadbalancing"},
+		StartTime:         &start, EndTime: &now,
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(result.Logs), "Should find 2 logs with loadbalancing")
+
+	result, err = store.SearchLogs(ctx, SearchFilters{
+		RoutingEngineUsed: []string{"governance"},
+		StartTime:         &start, EndTime: &now,
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(result.Logs), "Should find 1 log with governance")
+
+	result, err = store.SearchLogs(ctx, SearchFilters{
+		RoutingEngineUsed: []string{"routing-rule"},
+		StartTime:         &start, EndTime: &now,
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(result.Logs), "Should find 1 log with routing-rule")
+
+	// Multiple engines (OR)
+	result, err = store.SearchLogs(ctx, SearchFilters{
+		RoutingEngineUsed: []string{"loadbalancing", "routing-rule"},
+		StartTime:         &start, EndTime: &now,
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 3, len(result.Logs), "Should find all 3 with loadbalancing OR routing-rule")
+
+	// Non-existent engine
+	result, err = store.SearchLogs(ctx, SearchFilters{
+		RoutingEngineUsed: []string{"nonexistent"},
+		StartTime:         &start, EndTime: &now,
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(result.Logs))
+}
+
+func TestEnsurePerformanceIndexes(t *testing.T) {
+	db := trySetupPostgresDB(t)
+	if db == nil {
+		t.Skip("Postgres not available, skipping test")
+	}
+
+	db.Exec("DROP TABLE IF EXISTS mcp_tool_logs CASCADE")
+	db.Exec("DROP TABLE IF EXISTS async_jobs CASCADE")
+	db.Exec("DROP TABLE IF EXISTS logs CASCADE")
+	db.Exec("CREATE TABLE IF NOT EXISTS migrations (id VARCHAR(255) PRIMARY KEY)")
+	db.Exec("DELETE FROM migrations")
+
+	ctx := context.Background()
+	err := triggerMigrations(ctx, db)
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		for _, idx := range performanceIndexes {
+			db.Exec("DROP INDEX IF EXISTS " + idx.name)
+		}
+		db.Exec("DROP TABLE IF EXISTS mcp_tool_logs CASCADE")
+		db.Exec("DROP TABLE IF EXISTS async_jobs CASCADE")
+		db.Exec("DROP TABLE IF EXISTS logs CASCADE")
+		db.Exec("DELETE FROM migrations")
+	})
+
+	conn := acquirePerfTestSQLConn(t, ctx, db)
+	// First run
+	err = ensurePerformanceIndexes(ctx, conn)
+	require.NoError(t, err, "ensurePerformanceIndexes should succeed")
+
+	// Verify all indexes exist and are valid
+	for _, idx := range performanceIndexes {
+		var indexValid bool
+		err := db.Raw(`
+			SELECT COALESCE(bool_and(pi.indisvalid), false)
+			FROM pg_class pc
+			JOIN pg_index pi ON pi.indrelid = pc.oid
+			JOIN pg_class ic ON ic.oid = pi.indexrelid
+			WHERE pc.relname = ?
+			  AND ic.relname = ?
+		`, idx.table, idx.name).Scan(&indexValid).Error
+		require.NoError(t, err)
+		assert.True(t, indexValid, "Index %s should be valid", idx.name)
+	}
+
+	// Idempotent — second run should be a no-op
+	err = ensurePerformanceIndexes(ctx, conn)
+	require.NoError(t, err, "ensurePerformanceIndexes should be idempotent")
+}
+
+func TestContentSearch_Postgres(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+	start := now.Add(-1 * time.Hour)
+
+	// Build indexes
+	conn := acquirePerfTestSQLConn(t, ctx, db)
+
+	err := ensurePerformanceIndexes(ctx, conn)
+	require.NoError(t, err)
+
+	insertPerfLog(t, db, logOpts{
+		Timestamp:      now,
+		ContentSummary: "The quick brown fox jumps over the lazy dog",
+	})
+	insertPerfLog(t, db, logOpts{
+		Timestamp:      now,
+		ContentSummary: "Hello world this is a test message",
+	})
+
+	result, err := store.SearchLogs(ctx, SearchFilters{
+		ContentSearch: "brown fox",
+		StartTime:     &start, EndTime: &now,
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(result.Logs), "Should find 1 log matching 'brown fox'")
+
+	result, err = store.SearchLogs(ctx, SearchFilters{
+		ContentSearch: "test message",
+		StartTime:     &start, EndTime: &now,
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(result.Logs), "Should find 1 log matching 'test message'")
+
+	result, err = store.SearchLogs(ctx, SearchFilters{
+		ContentSearch: "nonexistent phrase",
+		StartTime:     &start, EndTime: &now,
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(result.Logs))
+}
+
+func TestMCPContentSearch_Postgres(t *testing.T) {
+	store, db := setupPerfTestDB(t)
+	ctx := context.Background()
+	now := time.Now().UTC()
+
+	// Build indexes
+	conn := acquirePerfTestSQLConn(t, ctx, db)
+	err := ensurePerformanceIndexes(ctx, conn)
+	require.NoError(t, err)
+
+	insertPerfMCPLog(t, db, mcpLogOpts{
+		ToolName: "search", ServerLabel: "s1", Timestamp: now,
+		VirtualKeyID: "vk-1", VirtualKeyName: "k1",
+		Arguments: `{"query": "weather in london"}`,
+		Result:    `{"temperature": 15}`,
+	})
+	insertPerfMCPLog(t, db, mcpLogOpts{
+		ToolName: "calc", ServerLabel: "s1", Timestamp: now,
+		VirtualKeyID: "vk-1", VirtualKeyName: "k1",
+		Arguments: `{"expression": "2+2"}`,
+		Result:    `{"answer": 4}`,
+	})
+
+	result, err := store.SearchMCPToolLogs(ctx, MCPToolLogSearchFilters{
+		ContentSearch: "london",
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(result.Logs), "Should find 1 MCP log matching 'london'")
+
+	result, err = store.SearchMCPToolLogs(ctx, MCPToolLogSearchFilters{
+		ContentSearch: "temperature",
+	}, PaginationOptions{Limit: 100})
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(result.Logs), "Should find 1 MCP log matching 'temperature' in result")
+}

framework/logstore/sqlite.go

@@ -0,0 +1,47 @@
+package logstore
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/maximhq/bifrost/core/schemas"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+// SQLiteConfig represents the configuration for a SQLite database.
+type SQLiteConfig struct {
+	Path string `json:"path"`
+}
+
+// newSqliteLogStore creates a new SQLite log store.
+func newSqliteLogStore(ctx context.Context, config *SQLiteConfig, logger schemas.Logger) (*RDBLogStore, error) {
+	if _, err := os.Stat(config.Path); os.IsNotExist(err) {
+		// Create DB file
+		f, err := os.Create(config.Path)
+		if err != nil {
+			return nil, err
+		}
+		_ = f.Close()
+	}
+	// Configure SQLite with proper settings to handle concurrent access
+	dsn := fmt.Sprintf("%s?_journal_mode=WAL&_synchronous=NORMAL&_cache_size=10000&_busy_timeout=60000&_wal_autocheckpoint=1000&_foreign_keys=1", config.Path)
+	logger.Debug("opening DB with dsn: %s", dsn)
+	db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
+		Logger: newGormLogger(logger),
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	logger.Debug("db opened for logstore")
+
+	s := &RDBLogStore{db: db, logger: logger}
+	// Run migrations
+	if err := triggerMigrations(ctx, db); err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}

framework/logstore/store.go

@@ -0,0 +1,140 @@
+package logstore
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/maximhq/bifrost/core/schemas"
+	"github.com/maximhq/bifrost/framework/objectstore"
+)
+
+// LogStoreType represents the type of log store.
+type LogStoreType string
+
+// LogStoreTypeSQLite is the type of log store for SQLite.
+const (
+	LogStoreTypeSQLite   LogStoreType = "sqlite"
+	LogStoreTypePostgres LogStoreType = "postgres"
+)
+
+// LogStore is the interface for the log store.
+type LogStore interface {
+	Ping(ctx context.Context) error
+	Create(ctx context.Context, entry *Log) error
+	CreateIfNotExists(ctx context.Context, entry *Log) error
+	BatchCreateIfNotExists(ctx context.Context, entries []*Log) error
+	FindByID(ctx context.Context, id string) (*Log, error)
+	IsLogEntryPresent(ctx context.Context, id string) (bool, error)
+	FindFirst(ctx context.Context, query any, fields ...string) (*Log, error)
+	FindAll(ctx context.Context, query any, fields ...string) ([]*Log, error)
+	FindAllDistinct(ctx context.Context, query any, fields ...string) ([]*Log, error)
+	HasLogs(ctx context.Context) (bool, error)
+	SearchLogs(ctx context.Context, filters SearchFilters, pagination PaginationOptions) (*SearchResult, error)
+	GetSessionLogs(ctx context.Context, sessionID string, pagination PaginationOptions) (*SessionDetailResult, error)
+	GetSessionSummary(ctx context.Context, sessionID string) (*SessionSummaryResult, error)
+	GetStats(ctx context.Context, filters SearchFilters) (*SearchStats, error)
+	GetHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*HistogramResult, error)
+	GetTokenHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*TokenHistogramResult, error)
+	GetCostHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*CostHistogramResult, error)
+	GetModelHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*ModelHistogramResult, error)
+	GetLatencyHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*LatencyHistogramResult, error)
+	GetProviderCostHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*ProviderCostHistogramResult, error)
+	GetProviderTokenHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*ProviderTokenHistogramResult, error)
+	GetProviderLatencyHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64) (*ProviderLatencyHistogramResult, error)
+	GetModelRankings(ctx context.Context, filters SearchFilters) (*ModelRankingResult, error)
+	GetUserRankings(ctx context.Context, filters SearchFilters) (*UserRankingResult, error)
+	// GetDimensionCostHistogram returns time-bucketed cost data grouped by the specified dimension (e.g., team_id, customer_id).
+	GetDimensionCostHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64, dimension HistogramDimension) (*DimensionCostHistogramResult, error)
+	// GetDimensionTokenHistogram returns time-bucketed token usage grouped by the specified dimension.
+	GetDimensionTokenHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64, dimension HistogramDimension) (*DimensionTokenHistogramResult, error)
+	// GetDimensionLatencyHistogram returns time-bucketed latency percentiles grouped by the specified dimension.
+	GetDimensionLatencyHistogram(ctx context.Context, filters SearchFilters, bucketSizeSeconds int64, dimension HistogramDimension) (*DimensionLatencyHistogramResult, error)
+	Update(ctx context.Context, id string, entry any) error
+	BulkUpdateCost(ctx context.Context, updates map[string]float64) error
+	Flush(ctx context.Context, since time.Time) error
+	Close(ctx context.Context) error
+	DeleteLog(ctx context.Context, id string) error
+	DeleteLogs(ctx context.Context, ids []string) error
+	DeleteLogsBatch(ctx context.Context, cutoff time.Time, batchSize int) (deletedCount int64, err error)
+
+	// Distinct value methods for filter data
+	GetDistinctModels(ctx context.Context) ([]string, error)
+	GetDistinctAliases(ctx context.Context) ([]string, error)
+	GetDistinctKeyPairs(ctx context.Context, idCol, nameCol string) ([]KeyPairResult, error)
+	GetDistinctRoutingEngines(ctx context.Context) ([]string, error)
+	GetDistinctMetadataKeys(ctx context.Context) (map[string][]string, error)
+
+	// MCP Tool Log histogram methods
+	GetMCPHistogram(ctx context.Context, filters MCPToolLogSearchFilters, bucketSizeSeconds int64) (*MCPHistogramResult, error)
+	GetMCPCostHistogram(ctx context.Context, filters MCPToolLogSearchFilters, bucketSizeSeconds int64) (*MCPCostHistogramResult, error)
+	GetMCPTopTools(ctx context.Context, filters MCPToolLogSearchFilters, limit int) (*MCPTopToolsResult, error)
+
+	// MCP Tool Log methods
+	CreateMCPToolLog(ctx context.Context, entry *MCPToolLog) error
+	FindMCPToolLog(ctx context.Context, id string) (*MCPToolLog, error)
+	UpdateMCPToolLog(ctx context.Context, id string, entry any) error
+	SearchMCPToolLogs(ctx context.Context, filters MCPToolLogSearchFilters, pagination PaginationOptions) (*MCPToolLogSearchResult, error)
+	GetMCPToolLogStats(ctx context.Context, filters MCPToolLogSearchFilters) (*MCPToolLogStats, error)
+	HasMCPToolLogs(ctx context.Context) (bool, error)
+	DeleteMCPToolLogs(ctx context.Context, ids []string) error
+	FlushMCPToolLogs(ctx context.Context, since time.Time) error
+	GetAvailableToolNames(ctx context.Context) ([]string, error)
+	GetAvailableServerLabels(ctx context.Context) ([]string, error)
+	GetAvailableMCPVirtualKeys(ctx context.Context) ([]MCPToolLog, error)
+
+	// Async Job methods
+	CreateAsyncJob(ctx context.Context, job *AsyncJob) error
+	FindAsyncJobByID(ctx context.Context, id string) (*AsyncJob, error)
+	UpdateAsyncJob(ctx context.Context, id string, updates map[string]interface{}) error
+	DeleteExpiredAsyncJobs(ctx context.Context) (int64, error)
+	DeleteStaleAsyncJobs(ctx context.Context, staleSince time.Time) (int64, error)
+}
+
+// NewLogStore creates a new log store based on the configuration.
+// When ObjectStorage is configured, the returned store is wrapped with a
+// HybridLogStore that offloads payloads to S3-compatible object storage.
+func NewLogStore(ctx context.Context, config *Config, logger schemas.Logger) (LogStore, error) {
+	if config == nil {
+		return nil, fmt.Errorf("logstore: config is nil")
+	}
+
+	var inner LogStore
+	var err error
+
+	switch config.Type {
+	case LogStoreTypeSQLite:
+		if sqliteConfig, ok := config.Config.(*SQLiteConfig); ok {
+			inner, err = newSqliteLogStore(ctx, sqliteConfig, logger)
+		} else {
+			return nil, fmt.Errorf("invalid sqlite config: %T", config.Config)
+		}
+	case LogStoreTypePostgres:
+		if postgresConfig, ok := config.Config.(*PostgresConfig); ok {
+			inner, err = newPostgresLogStore(ctx, postgresConfig, logger)
+		} else {
+			return nil, fmt.Errorf("invalid postgres config: %T", config.Config)
+		}
+	default:
+		return nil, fmt.Errorf("unsupported log store type: %s", config.Type)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// Optionally wrap with hybrid decorator for object storage offloading.
+	if config.ObjectStorage != nil {
+		objStore, objErr := objectstore.NewObjectStore(ctx, config.ObjectStorage, logger)
+		if objErr != nil {
+			_ = inner.Close(ctx)
+			return nil, fmt.Errorf("failed to create object store: %w", objErr)
+		}
+		if err := objStore.Ping(ctx); err != nil {
+			_ = objStore.Close()
+			_ = inner.Close(ctx)
+			return nil, fmt.Errorf("failed to ping object store: %w", err)
+		}
+		return newHybridLogStore(inner, objStore, config.ObjectStorage.GetPrefix(), logger), nil
+	}
+	return inner, nil
+}