api/lib/metadata.ts

/**
 * ZDR Metadata Tracker
 *
 * Always-on, privacy-first analytics for G0DM0D3.
 * Records EVERYTHING about requests — except the actual content.
 *
 * What IS tracked:
 * - Timestamps, endpoints, modes, tiers
 * - Models used, winner model, scores, latencies
 * - Pipeline flags (godmode, autotune, parseltongue, stm)
 * - AutoTune detected context type + confidence
 * - Per-model success/failure, response lengths, durations
 * - Error types (categorized, never raw error messages)
 *
 * What is NEVER tracked:
 * - Message content, system prompts, responses
 * - API keys, auth tokens, IP addresses
 * - Any PII whatsoever
 *
 * Storage: in-memory ring buffer, resets on restart.
 */

import { randomUUID } from 'crypto'

// ── Types ────────────────────────────────────────────────────────────

export interface MetadataEvent {
  id: string
  timestamp: number

  // Request shape
  endpoint: string
  mode: 'standard' | 'ultraplinian'
  tier?: string // fast | standard | full (ultraplinian only)
  stream: boolean

  // Pipeline config (what was enabled, not what it produced)
  pipeline: {
    godmode: boolean
    autotune: boolean
    parseltongue: boolean
    stm_modules: string[]
    strategy?: string
  }

  // AutoTune metadata (context detection only, no content)
  autotune?: {
    detected_context: string
    confidence: number
  }

  // Model metadata
  model?: string // single-model mode
  models_queried?: number // ultraplinian
  models_succeeded?: number

  // Per-model results (no content, just numbers)
  model_results?: Array<{
    model: string
    score: number
    duration_ms: number
    success: boolean
    content_length: number
    error_type?: string // categorized: 'timeout' | 'rate_limit' | 'auth' | 'model_error' | 'empty' | 'unknown'
  }>

  // Winner metadata
  winner?: {
    model: string
    score: number
    duration_ms: number
    content_length: number
  }

  // Response metadata
  total_duration_ms: number
  response_length: number

  // Liquid Response metadata (streaming upgrades)
  liquid?: {
    upgrades: number // how many leader changes happened
    first_response_ms: number // time to first leader
  }
}

// ── In-Memory Ring Buffer ────────────────────────────────────────────

let events: MetadataEvent[] = []
const MAX_EVENTS = 50000 // ~50k events before eviction

// ── Recording ────────────────────────────────────────────────────────

export function recordEvent(event: Omit<MetadataEvent, 'id' | 'timestamp'>): string {
  const id = randomUUID()
  const record: MetadataEvent = {
    ...event,
    id,
    timestamp: Date.now(),
  }

  events.push(record)

  // Evict oldest if over cap
  if (events.length > MAX_EVENTS) {
    events = events.slice(events.length - MAX_EVENTS)
  }

  return id
}

// ── Error Categorization ─────────────────────────────────────────────

export function categorizeError(error?: string): string | undefined {
  if (!error) return undefined
  const lower = error.toLowerCase()
  if (lower.includes('timeout') || lower.includes('abort')) return 'timeout'
  if (lower.includes('rate') || lower.includes('429')) return 'rate_limit'
  if (lower.includes('auth') || lower.includes('401') || lower.includes('403')) return 'auth'
  if (lower.includes('empty')) return 'empty'
  if (lower.includes('race ended')) return 'early_exit'
  if (lower.includes('api error') || lower.includes('http')) return 'model_error'
  return 'unknown'
}

// ── Query: Raw Events ────────────────────────────────────────────────

export function getEvents(opts?: {
  limit?: number
  offset?: number
  since?: number // unix ms timestamp
  mode?: 'standard' | 'ultraplinian'
}): { events: MetadataEvent[]; total: number } {
  let filtered = events

  if (opts?.since) {
    filtered = filtered.filter(e => e.timestamp >= opts.since!)
  }
  if (opts?.mode) {
    filtered = filtered.filter(e => e.mode === opts.mode)
  }

  const total = filtered.length
  const offset = opts?.offset ?? 0
  const limit = opts?.limit ?? 100

  return {
    events: filtered.slice(offset, offset + limit),
    total,
  }
}

// ── Query: Aggregated Stats ──────────────────────────────────────────

export interface MetadataStats {
  total_requests: number
  since: number | null // oldest event timestamp
  uptime_ms: number

  // Breakdown by mode
  by_mode: Record<string, number>
  by_tier: Record<string, number>
  by_endpoint: Record<string, number>

  // Model usage
  models: {
    total_queries: number // sum of all individual model calls
    unique_models: number
    by_model: Record<string, {
      queries: number
      wins: number
      avg_score: number
      avg_duration_ms: number
      success_rate: number
    }>
  }

  // Pipeline usage
  pipeline: {
    godmode_rate: number // % of requests with godmode enabled
    autotune_rate: number
    parseltongue_rate: number
    stm_usage: Record<string, number> // module -> count
    strategy_usage: Record<string, number>
  }

  // Context detection (from autotune)
  contexts: Record<string, number>

  // Latency
  latency: {
    avg_ms: number
    p50_ms: number
    p95_ms: number
    p99_ms: number
  }

  // Response sizes
  response_lengths: {
    avg: number
    p50: number
    p95: number
  }

  // Streaming
  streaming: {
    stream_rate: number // % of requests that used streaming
    avg_upgrades: number // avg leader changes per stream
    avg_first_response_ms: number // avg time to first leader
  }

  // Errors
  errors: {
    total_model_errors: number
    by_type: Record<string, number>
    model_success_rate: number // across all individual model queries
  }
}

export function getStats(): MetadataStats {
  const total = events.length

  if (total === 0) {
    return {
      total_requests: 0,
      since: null,
      uptime_ms: 0,
      by_mode: {},
      by_tier: {},
      by_endpoint: {},
      models: { total_queries: 0, unique_models: 0, by_model: {} },
      pipeline: { godmode_rate: 0, autotune_rate: 0, parseltongue_rate: 0, stm_usage: {}, strategy_usage: {} },
      contexts: {},
      latency: { avg_ms: 0, p50_ms: 0, p95_ms: 0, p99_ms: 0 },
      response_lengths: { avg: 0, p50: 0, p95: 0 },
      streaming: { stream_rate: 0, avg_upgrades: 0, avg_first_response_ms: 0 },
      errors: { total_model_errors: 0, by_type: {}, model_success_rate: 0 },
    }
  }

  // Breakdowns
  const byMode: Record<string, number> = {}
  const byTier: Record<string, number> = {}
  const byEndpoint: Record<string, number> = {}
  const contexts: Record<string, number> = {}
  const stmUsage: Record<string, number> = {}
  const strategyUsage: Record<string, number> = {}
  const errorTypes: Record<string, number> = {}

  // Model tracking
  const modelData: Record<string, { queries: number; wins: number; scores: number[]; durations: number[]; successes: number }> = {}

  // Accumulators
  let godmodeCount = 0
  let autotuneCount = 0
  let parseltongueCount = 0
  let totalModelQueries = 0
  let totalModelErrors = 0
  let totalModelSuccesses = 0
  const durations: number[] = []
  const responseLengths: number[] = []
  let streamCount = 0
  let upgradeSum = 0
  let firstResponseSum = 0
  let streamWithLiquidCount = 0

  for (const e of events) {
    byMode[e.mode] = (byMode[e.mode] || 0) + 1
    if (e.tier) byTier[e.tier] = (byTier[e.tier] || 0) + 1
    byEndpoint[e.endpoint] = (byEndpoint[e.endpoint] || 0) + 1

    if (e.pipeline.godmode) godmodeCount++
    if (e.pipeline.autotune) autotuneCount++
    if (e.pipeline.parseltongue) parseltongueCount++
    if (e.pipeline.strategy) {
      strategyUsage[e.pipeline.strategy] = (strategyUsage[e.pipeline.strategy] || 0) + 1
    }
    for (const mod of e.pipeline.stm_modules) {
      stmUsage[mod] = (stmUsage[mod] || 0) + 1
    }

    if (e.autotune?.detected_context) {
      const ctx = e.autotune.detected_context
      contexts[ctx] = (contexts[ctx] || 0) + 1
    }

    durations.push(e.total_duration_ms)
    responseLengths.push(e.response_length)

    if (e.stream) streamCount++

    if (e.liquid) {
      upgradeSum += e.liquid.upgrades
      firstResponseSum += e.liquid.first_response_ms
      streamWithLiquidCount++
    }

    // Per-model results
    if (e.model_results) {
      for (const mr of e.model_results) {
        totalModelQueries++
        if (!modelData[mr.model]) {
          modelData[mr.model] = { queries: 0, wins: 0, scores: [], durations: [], successes: 0 }
        }
        const md = modelData[mr.model]
        md.queries++
        if (mr.success) {
          md.successes++
          totalModelSuccesses++
          md.scores.push(mr.score)
          md.durations.push(mr.duration_ms)
        } else {
          totalModelErrors++
          if (mr.error_type) {
            errorTypes[mr.error_type] = (errorTypes[mr.error_type] || 0) + 1
          }
        }
      }
    }

    // Winner tracking
    if (e.winner) {
      if (!modelData[e.winner.model]) {
        modelData[e.winner.model] = { queries: 0, wins: 0, scores: [], durations: [], successes: 0 }
      }
      modelData[e.winner.model].wins++
    }
  }

  // Build per-model stats
  const byModel: Record<string, { queries: number; wins: number; avg_score: number; avg_duration_ms: number; success_rate: number }> = {}
  for (const [model, data] of Object.entries(modelData)) {
    byModel[model] = {
      queries: data.queries,
      wins: data.wins,
      avg_score: data.scores.length > 0 ? Math.round(data.scores.reduce((a, b) => a + b, 0) / data.scores.length) : 0,
      avg_duration_ms: data.durations.length > 0 ? Math.round(data.durations.reduce((a, b) => a + b, 0) / data.durations.length) : 0,
      success_rate: data.queries > 0 ? Math.round((data.successes / data.queries) * 100) / 100 : 0,
    }
  }

  return {
    total_requests: total,
    since: events[0].timestamp,
    uptime_ms: Date.now() - events[0].timestamp,
    by_mode: byMode,
    by_tier: byTier,
    by_endpoint: byEndpoint,
    models: {
      total_queries: totalModelQueries,
      unique_models: Object.keys(modelData).length,
      by_model: byModel,
    },
    pipeline: {
      godmode_rate: Math.round((godmodeCount / total) * 100) / 100,
      autotune_rate: Math.round((autotuneCount / total) * 100) / 100,
      parseltongue_rate: Math.round((parseltongueCount / total) * 100) / 100,
      stm_usage: stmUsage,
      strategy_usage: strategyUsage,
    },
    contexts,
    latency: {
      avg_ms: Math.round(durations.reduce((a, b) => a + b, 0) / durations.length),
      p50_ms: percentile(durations, 50),
      p95_ms: percentile(durations, 95),
      p99_ms: percentile(durations, 99),
    },
    response_lengths: {
      avg: Math.round(responseLengths.reduce((a, b) => a + b, 0) / responseLengths.length),
      p50: percentile(responseLengths, 50),
      p95: percentile(responseLengths, 95),
    },
    streaming: {
      stream_rate: Math.round((streamCount / total) * 100) / 100,
      avg_upgrades: streamWithLiquidCount > 0 ? Math.round((upgradeSum / streamWithLiquidCount) * 100) / 100 : 0,
      avg_first_response_ms: streamWithLiquidCount > 0 ? Math.round(firstResponseSum / streamWithLiquidCount) : 0,
    },
    errors: {
      total_model_errors: totalModelErrors,
      by_type: errorTypes,
      model_success_rate: totalModelQueries > 0 ? Math.round((totalModelSuccesses / totalModelQueries) * 100) / 100 : 0,
    },
  }
}

// ── Helpers ──────────────────────────────────────────────────────────

function percentile(arr: number[], p: number): number {
  if (arr.length === 0) return 0
  const sorted = [...arr].sort((a, b) => a - b)
  const idx = Math.ceil((p / 100) * sorted.length) - 1
  return sorted[Math.max(0, idx)]
}