cad-bench / apps /dashboard /trace-api.mjs
Duplicate from markov-ai/autocad-bench
1866d85 about 1 month ago
41.7 kB
import fs from 'node:fs'
import { createHash } from 'node:crypto'
import path from 'node:path'
import { fileURLToPath } from 'node:url'

const DASHBOARD_ROOT = path.dirname(fileURLToPath(import.meta.url))
const CHECKOUT_ROOT = path.resolve(DASHBOARD_ROOT, '..', '..')
const BENCHMARK_DATA_ROOT = path.resolve(
  process.env.AUTOCAD_BENCH_ROOT || CHECKOUT_ROOT,
)
const DEFAULT_RUNS_ROOT = path.join(CHECKOUT_ROOT, 'runs')
const MANIFEST_PATH = path.join(BENCHMARK_DATA_ROOT, 'tasks', 'manifest.jsonl')
const TRACE_SUMMARY_NAME = 'model_turn_summaries.jsonl'
const MAX_INLINE_LEGACY_TRACE_BYTES = 8 * 1024 * 1024
const MAX_INLINE_USAGE_SUMMARY_BYTES = 512 * 1024
const usageCache = new Map()

function readJson(filePath) {
  try {
    return JSON.parse(fs.readFileSync(filePath, 'utf8').replace(/^\uFEFF/, ''))
  } catch {
    return null
  }
}

function readJsonl(filePath) {
  try {
    return fs
      .readFileSync(filePath, 'utf8')
      .split(/\r?\n/)
      .filter((line) => line.trim())
      .flatMap((line) => {
        try {
          return [JSON.parse(line.replace(/^\uFEFF/, ''))]
        } catch {
          return []
        }
      })
  } catch {
    return []
  }
}

function modelInputArtifacts(value, output = []) {
  if (Array.isArray(value)) {
    for (const item of value) modelInputArtifacts(item, output)
    return output
  }
  if (!value || typeof value !== 'object') return output
  if (typeof value.artifact_path === 'string') {
    output.push(Object.fromEntries(Object.entries(value).filter(([key]) => [
      'type', 'artifact_path', 'sha256', 'bytes', 'width', 'height', 'sequence',
    ].includes(key))))
  }
  for (const item of Object.values(value)) modelInputArtifacts(item, output)
  return output
}

function compactTurn(turn) {
  if (!turn || typeof turn !== 'object') return turn
  const compact = Object.fromEntries(
    Object.entries(turn).filter(([key]) => key !== 'model_input'),
  )
  if (turn.model_input && typeof turn.model_input === 'object') {
    compact.model_input = {
      model: turn.model_input.model || null,
      artifacts: modelInputArtifacts(turn.model_input),
    }
  }
  return compact
}

function traceSummaryPath(runDirectory) {
  return path.join(runDirectory, TRACE_SUMMARY_NAME)
}

function traceTurns(runDirectory) {
  const summaryPath = traceSummaryPath(runDirectory)
  if (fs.existsSync(summaryPath)) return readJsonl(summaryPath)
  const legacyPath = path.join(runDirectory, 'model_turns.jsonl')
  try {
    if (fs.statSync(legacyPath).size <= MAX_INLINE_LEGACY_TRACE_BYTES) {
      return readJsonl(legacyPath).map(compactTurn)
    }
  } catch {
    // Missing or actively-created traces simply have no turns yet.
  }
  return []
}

function manifestByTask() {
  const entries = readJsonl(MANIFEST_PATH)
  return new Map(entries.map((entry) => [entry.task_id, entry]))
}

function safeRunDirectory(runsRoot, runId) {
  if (!runId || runId.includes('\\')) {
    return null
  }
  const segments = runId.split('/')
  if (segments.some((segment) => !segment || segment === '.' || segment === '..')) return null
  const candidate = path.resolve(runsRoot, runId)
  return candidate.startsWith(path.resolve(runsRoot) + path.sep) ? candidate : null
}

function safeArtifact(runDirectory, relativePath) {
  const candidate = path.resolve(runDirectory, relativePath)
  return candidate.startsWith(path.resolve(runDirectory) + path.sep) ? candidate : null
}

function normalizeSha256(value) {
  const sha256 = String(value || '').toLowerCase()
  return /^[a-f0-9]{64}$/.test(sha256) ? sha256 : null
}

function fileSha256(filePath) {
  try {
    if (!fs.statSync(filePath).isFile()) return null
    return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex')
  } catch {
    return null
  }
}

function contentType(filePath) {
  const extension = path.extname(filePath).toLowerCase()
  return {
    '.png': 'image/png',
    '.json': 'application/json; charset=utf-8',
    '.jsonl': 'application/x-ndjson; charset=utf-8',
    '.dwg': 'application/acad',
  }[extension] || 'application/octet-stream'
}

function sendJson(response, status, value) {
  response.statusCode = status
  response.setHeader('Content-Type', 'application/json; charset=utf-8')
  response.setHeader('Cache-Control', 'no-store')
  response.end(JSON.stringify(value))
}

function sendFile(response, filePath, { immutable = false } = {}) {
  if (!filePath || !fs.existsSync(filePath) || !fs.statSync(filePath).isFile()) {
    sendJson(response, 404, { error: 'artifact_not_found' })
    return
  }
  response.statusCode = 200
  response.setHeader('Content-Type', contentType(filePath))
  response.setHeader(
    'Cache-Control',
    immutable ? 'private, max-age=31536000, immutable' : 'no-store',
  )
  fs.createReadStream(filePath).pipe(response)
}

function sendPrivateRedirect(response, location) {
  response.statusCode = 307
  response.setHeader('Location', location)
  // Keep the stable local URL cached only inside this browser and comfortably
  // below the five-minute lifetime of the signed origin URL.
  response.setHeader('Cache-Control', 'private, max-age=240')
  response.end()
}

function readRunScores(runDirectory) {
  const visionArtifact = readJson(path.join(
    runDirectory, 'evaluation', 'vision-judge', 'score.json',
  ))
  const vision = visionArtifact
    ? {
      ...visionArtifact,
      score: visionArtifact.overall_score ?? null,
      valid: true,
      failure_reason: null,
      scoring_method: `vision:${visionArtifact.prompt_version || 'unknown'}`,
      geometry_similarity: visionArtifact.component_scores?.geometry ?? null,
      dimension_similarity: visionArtifact.component_scores?.dimensions ?? null,
      annotation_similarity: visionArtifact.component_scores?.annotations ?? null,
      visual_similarity: visionArtifact.component_scores?.presentation ?? null,
      score_source: 'vision_judge',
    }
    : null
  const deterministicArtifact = readJson(path.join(
    runDirectory, 'evaluation', 'score.json',
  ))
  const deterministic = deterministicArtifact
    ? { ...deterministicArtifact, score_source: 'deterministic' }
    : null
  return {
    vision,
    deterministic,
    displayed: vision || deterministic,
  }
}

function readRunScore(runDirectory) {
  return readRunScores(runDirectory).displayed
}

function scoreSummary(score) {
  if (!score) return null
  return {
    source: score.score_source ?? null,
    score: score.score ?? null,
    valid: score.valid ?? null,
    failureReason: score.failure_reason ?? null,
    method: score.scoring_method ?? null,
    verdict: score.verdict ?? null,
    targetAcquired: score.target_acquired ?? null,
    confidence: score.confidence ?? null,
    needsHumanReview: score.needs_human_review ?? false,
    geometry: score.geometry_similarity ?? null,
    dimensions: score.dimension_similarity ?? null,
    annotations: score.annotation_similarity ?? null,
    presentation: score.visual_similarity ?? null,
  }
}

const ACTIVE_STATES = new Set([
  'starting', 'resetting', 'handoff_ready', 'running', 'finalizing',
])
const TERMINAL_EXECUTION_STATES = new Set([
  'completed', 'recovered', 'budget_exhausted', 'model_failed',
  'infrastructure_failed', 'artifact_missing', 'interrupted',
])
const ACTIVE_EVALUATION_STAGES = new Set([
  'inspecting', 'deterministic_scoring', 'vision_judging',
])

function executionState(result, state = null) {
  if (TERMINAL_EXECUTION_STATES.has(result?.execution_status)) {
    return result.execution_status
  }
  const stopReason = String(result?.stop_reason || '').toLowerCase()
  const failure = String(result?.failure || '').toLowerCase()
  const artifactBytes = Number(result?.artifact_bytes || 0)
  if (result?.completed === true || stopReason === 'done') {
    return artifactBytes > 0 ? 'completed' : 'artifact_missing'
  }
  if (stopReason === 'recovered_finalized_session' && artifactBytes > 0) return 'recovered'
  if (['max_actions', 'wall_timeout'].includes(stopReason)) return 'budget_exhausted'
  if (['operator_stopped', 'interrupted'].includes(stopReason)) return 'interrupted'
  if (stopReason === 'rollout_error' || [
    'brokererror', 'connecterror', 'connection attempts failed',
    'nodename nor servname', 'transport unavailable', 'ssm',
  ].some((marker) => failure.includes(marker))) return 'infrastructure_failed'
  if (result && artifactBytes <= 0 && !failure) return 'artifact_missing'
  if (result) return 'model_failed'
  if (ACTIVE_STATES.has(state?.state)) return state.state
  if (TERMINAL_EXECUTION_STATES.has(state?.state)) return state.state
  return state?.state || 'running'
}

function evaluationState({ status, artifactBytes, score, evaluationStatus = null }) {
  if (status === 'queued') return 'queued'
  if (ACTIVE_STATES.has(status)) return 'running'
  if (ACTIVE_EVALUATION_STAGES.has(evaluationStatus?.stage)) {
    const updatedAt = Date.parse(evaluationStatus?.updated_at || '')
    if (Number.isFinite(updatedAt) && Date.now() - updatedAt > 30 * 60 * 1000) {
      return 'evaluation_interrupted'
    }
    return 'evaluating'
  }
  if (evaluationStatus?.stage === 'retry_pending') return 'evaluation_retrying'
  if (evaluationStatus?.stage === 'failed') return 'evaluation_failed'
  if (score?.valid === false || score?.failure_reason) return 'candidate_invalid'
  if (Number.isFinite(score?.score)) return 'scored'
  if (artifactBytes > 0) return 'awaiting_score'
  if (['model_failed', 'infrastructure_failed', 'artifact_missing', 'interrupted'].includes(status)) return 'run_failed'
  return 'no_artifact'
}

function runSummary(runsRoot, runId) {
  const runDirectory = safeRunDirectory(runsRoot, runId)
  if (!runDirectory) return null
  const result = readJson(path.join(runDirectory, 'result.json'))
  const state = readJson(path.join(runDirectory, 'run-state.json'))
  const evaluationStatus = readJson(path.join(
    runDirectory, 'evaluation', 'status.json',
  ))
  const scores = readRunScores(runDirectory)
  const score = scores.displayed
  const stats = fs.statSync(runDirectory)
  const status = executionState(result, state)
  const artifactBytes = result?.artifact_bytes ?? 0
  const executionCanCarryFailure = [
    'model_failed', 'infrastructure_failed', 'artifact_missing', 'interrupted',
  ].includes(status)
  return {
    id: runId,
    taskId: state?.task_id || result?.task_id || 'unknown',
    attemptId: state?.attempt_id || result?.attempt_id || null,
    modelId: state?.model_id || result?.model_id || 'unknown',
    provider: state?.provider || result?.provider || (String(result?.model_id || '').includes('nova') ? 'bedrock' : 'unknown'),
    reasoningEffort: state?.reasoning_effort || result?.reasoning_effort || null,
    status,
    stopReason: result?.stop_reason || state?.stop_reason || null,
    failure: executionCanCarryFailure
      ? (result?.failure || state?.failure || null)
      : (result?.failure || null),
    acceptedActions: state?.accepted_actions ?? result?.accepted_actions ?? readJsonl(path.join(runDirectory, 'input_receipts.jsonl')).length,
    invalidActions: state?.invalid_actions ?? result?.invalid_actions ?? 0,
    screenshots: state?.screenshots ?? result?.screenshots ?? 0,
    elapsedSeconds: result?.elapsed_s ?? null,
    artifactBytes,
    score: score?.score ?? null,
    valid: score?.valid ?? null,
    scoreFailure: score?.failure_reason ?? null,
    scoreMethod: score?.scoring_method ?? null,
    scoreSource: score?.score_source ?? null,
    verdict: score?.verdict ?? null,
    targetAcquired: score?.target_acquired ?? null,
    judgeConfidence: score?.confidence ?? null,
    needsHumanReview: score?.needs_human_review ?? false,
    visionScore: scoreSummary(scores.vision),
    deterministicScore: scoreSummary(scores.deterministic),
    evaluationState: evaluationState({
      status, artifactBytes, score, evaluationStatus,
    }),
    evaluationStage: evaluationStatus?.stage || null,
    evaluationError: evaluationStatus?.error || null,
    updatedAt: evaluationStatus?.updated_at || state?.updated_at || stats.mtime.toISOString(),
  }
}

function listRuns(runsRoot) {
  if (!fs.existsSync(runsRoot)) return []
  const runIds = []
  const visit = (directory, relativeDirectory, depth) => {
    if (depth > 4) return
    for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
      if (!entry.isDirectory()) continue
      const relativePath = relativeDirectory
        ? `${relativeDirectory}/${entry.name}`
        : entry.name
      const child = path.join(directory, entry.name)
      const isRun = ['run-state.json', 'result.json', 'model_turns.jsonl']
        .some((name) => fs.existsSync(path.join(child, name)))
      if (isRun) runIds.push(relativePath)
      else visit(child, relativePath, depth + 1)
    }
  }
  visit(runsRoot, '', 1)
  return runIds
    .map((runId) => runSummary(runsRoot, runId))
    .filter(Boolean)
    .sort((left, right) => String(right.updatedAt).localeCompare(String(left.updatedAt)))
}

function usageSummary(turns, pricing = null) {
  const totals = {
    inputTokens: 0,
    cachedInputTokens: 0,
    outputTokens: 0,
    reasoningOutputTokens: 0,
    totalTokens: 0,
    modelTurns: turns.length,
    apiCost: pricing ? 0 : null,
  }
  for (const turn of turns) {
    const usage = turn?.usage || {}
    const input = Number(usage.input_tokens ?? usage.inputTokens ?? 0) || 0
    const cached = Math.min(
      input,
      Number(usage.cached_input_tokens ?? usage.cachedInputTokens ?? 0) || 0,
    )
    const output = Number(usage.output_tokens ?? usage.outputTokens ?? 0) || 0
    const reasoning = Number(
      usage.reasoning_output_tokens ?? usage.reasoningOutputTokens ?? 0,
    ) || 0
    totals.inputTokens += input
    totals.cachedInputTokens += cached
    totals.outputTokens += output
    totals.reasoningOutputTokens += reasoning
    totals.totalTokens += Number(usage.total_tokens ?? usage.totalTokens ?? input + output) || 0
    if (pricing) {
      const isLong = pricing.long_context_threshold_tokens != null
        && input > Number(pricing.long_context_threshold_tokens)
      const inputMultiplier = isLong
        ? Number(pricing.long_context_input_multiplier ?? 1)
        : 1
      const outputMultiplier = isLong
        ? Number(pricing.long_context_output_multiplier ?? 1)
        : 1
      const uncached = Math.max(0, input - cached)
      totals.apiCost += inputMultiplier * (
        uncached * Number(pricing.input_per_million ?? 0)
        + cached * Number(pricing.cached_input_per_million ?? 0)
      ) / 1_000_000
      totals.apiCost += outputMultiplier
        * output * Number(pricing.output_per_million ?? 0) / 1_000_000
    }
  }
  return totals
}

function cachedUsageSummary(filePath, pricing) {
  let stats
  try {
    stats = fs.statSync(filePath)
  } catch {
    return null
  }
  const cacheKey = JSON.stringify([
    stats.size, stats.mtimeMs, pricing || null,
  ])
  const cached = usageCache.get(filePath)
  if (cached?.key === cacheKey) return cached.value
  const value = usageSummary(readJsonl(filePath), pricing)
  usageCache.set(filePath, { key: cacheKey, value })
  if (usageCache.size > 64) usageCache.delete(usageCache.keys().next().value)
  return value
}

function runUsage(
  runDirectory,
  pricing,
  runState = null,
  result = null,
  { preferAggregate = false } = {},
) {
  const summaryPath = traceSummaryPath(runDirectory)
  if (!preferAggregate) {
    try {
      if (fs.statSync(summaryPath).size <= MAX_INLINE_USAGE_SUMMARY_BYTES) {
        const compact = cachedUsageSummary(summaryPath, pricing)
        if (compact) return compact
      }
    } catch {
      // Fall through to the legacy trace or live aggregate.
    }
  }

  const legacyPath = path.join(runDirectory, 'model_turns.jsonl')
  if (!preferAggregate) {
    try {
      if (fs.statSync(legacyPath).size <= MAX_INLINE_LEGACY_TRACE_BYTES) {
        return cachedUsageSummary(legacyPath, pricing)
      }
    } catch {
      // Fall through to the live aggregate written by newer pilots.
    }
  }

  const raw = runState?.usage || result?.usage
  const modelTurns = Number(runState?.turn_count ?? 0) || 0
  if (!raw || typeof raw !== 'object') {
    return {
      inputTokens: 0,
      cachedInputTokens: 0,
      outputTokens: 0,
      reasoningOutputTokens: 0,
      totalTokens: 0,
      modelTurns,
      apiCost: modelTurns ? null : (pricing ? 0 : null),
    }
  }
  const inputTokens = Number(raw.input_tokens ?? raw.inputTokens ?? 0) || 0
  const outputTokens = Number(raw.output_tokens ?? raw.outputTokens ?? 0) || 0
  return {
    inputTokens,
    cachedInputTokens: Number(
      raw.cached_input_tokens ?? raw.cache_read_input_tokens
      ?? raw.cachedInputTokens ?? 0,
    ) || 0,
    outputTokens,
    reasoningOutputTokens: Number(
      raw.reasoning_output_tokens ?? raw.reasoningOutputTokens ?? 0,
    ) || 0,
    totalTokens: Number(
      raw.total_tokens ?? raw.totalTokens ?? inputTokens + outputTokens,
    ) || 0,
    modelTurns,
    // Exact cost can depend on per-request long-context thresholds. The compact
    // per-turn sidecar is authoritative whenever pricing is configured.
    apiCost: null,
  }
}

function listBenchmarkIds(runsRoot) {
  if (!fs.existsSync(runsRoot)) return []
  const ids = []
  const visit = (directory, relativeDirectory, depth) => {
    if (depth > 4) return
    if (fs.existsSync(path.join(directory, 'batch-state.json'))) {
      ids.push(relativeDirectory)
      return
    }
    for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
      if (!entry.isDirectory()) continue
      const relativePath = relativeDirectory
        ? `${relativeDirectory}/${entry.name}`
        : entry.name
      visit(path.join(directory, entry.name), relativePath, depth + 1)
    }
  }
  visit(runsRoot, '', 0)
  return ids.filter(Boolean)
}

function benchmarkDetail(
  runsRoot,
  benchmarkId,
  { preferAggregateUsage = false } = {},
) {
  const directory = safeRunDirectory(runsRoot, benchmarkId)
  if (!directory || !fs.existsSync(directory)) return null
  const state = readJson(path.join(directory, 'batch-state.json'))
  if (!state || !state.rollouts || typeof state.rollouts !== 'object') return null
  const manifest = manifestByTask()
  const pricing = state.pricing || null
  const tasks = Object.entries(state.rollouts).map(([rolloutId, row]) => {
    const runId = `${benchmarkId}/rollouts/${rolloutId}`
    const runDirectory = path.join(directory, 'rollouts', rolloutId)
    const hasRun = fs.existsSync(runDirectory)
    const summary = hasRun ? runSummary(runsRoot, runId) : null
    const runState = hasRun ? readJson(path.join(runDirectory, 'run-state.json')) : null
    const result = hasRun ? readJson(path.join(runDirectory, 'result.json')) : null
    const evaluationStatus = hasRun ? readJson(path.join(
      runDirectory, 'evaluation', 'status.json',
    )) : null
    const scores = hasRun
      ? readRunScores(runDirectory)
      : { vision: null, deterministic: null, displayed: null }
    const score = scores.displayed
    const usage = hasRun
      ? runUsage(runDirectory, pricing, runState, result, {
        preferAggregate: preferAggregateUsage,
      })
      : usageSummary([], pricing)
    const taskId = row.task_id || summary?.taskId || state.task_id || 'unknown'
    const task = manifest.get(taskId)
    // Batch state covers the complete sandbox lifecycle, including automatic
    // evaluation. The pilot's local result can say completed while its wrapper
    // is still judging or has failed evaluation.
    const rowStatus = row.state || 'queued'
    const summaryIsTerminal = TERMINAL_EXECUTION_STATES.has(summary?.status)
    const status = !hasRun && rowStatus === 'running'
      ? 'starting'
      : summaryIsTerminal
        ? summary.status
        : summary?.status || rowStatus || 'queued'
    const artifactBytes = summary?.artifactBytes ?? result?.artifact_bytes ?? 0
    const executionCanCarryFailure = [
      'model_failed', 'infrastructure_failed', 'artifact_missing', 'interrupted',
    ].includes(status)
    return {
      rolloutId,
      runId,
      taskId,
      level: task?.level || 'unknown',
      drawingType: task?.drawing_type || '2d',
      split: task?.split || 'unknown',
      displayName: row.display_name || rolloutId,
      modelId: row.model_id || summary?.modelId || 'unknown',
      provider: row.provider || summary?.provider || 'unknown',
      status,
      stopReason: summary?.stopReason || result?.stop_reason || null,
      failure: executionCanCarryFailure
        ? (summary?.failure || result?.failure || row.failure || null)
        : (result?.failure || null),
      acceptedActions: summary?.acceptedActions ?? result?.accepted_actions ?? 0,
      invalidActions: summary?.invalidActions ?? result?.invalid_actions ?? 0,
      elapsedSeconds: summary?.elapsedSeconds ?? result?.elapsed_s ?? null,
      setupElapsedSeconds: result?.setup_elapsed_s ?? null,
      rolloutElapsedSeconds: result?.rollout_elapsed_s ?? null,
      artifactBytes,
      score: score?.score ?? null,
      valid: score?.valid ?? null,
      scoreFailure: score?.failure_reason ?? null,
      scoreMethod: score?.scoring_method ?? null,
      scoreSource: score?.score_source ?? null,
      verdict: score?.verdict ?? null,
      targetAcquired: score?.target_acquired ?? null,
      judgeConfidence: score?.confidence ?? null,
      needsHumanReview: score?.needs_human_review ?? false,
      visionScore: scoreSummary(scores.vision),
      deterministicScore: scoreSummary(scores.deterministic),
      geometrySimilarity: score?.geometry_similarity ?? null,
      dimensionSimilarity: score?.dimension_similarity ?? null,
      annotationSimilarity: score?.annotation_similarity ?? null,
      visualSimilarity: score?.visual_similarity ?? null,
      evaluationState: summary?.evaluationState || evaluationState({
        status, artifactBytes, score, evaluationStatus,
      }),
      evaluationStage: summary?.evaluationStage || evaluationStatus?.stage || null,
      evaluationError: summary?.evaluationError || evaluationStatus?.error || null,
      startedAt: row.started_at || null,
      finishedAt: row.finished_at || null,
      updatedAt: summary?.updatedAt || row.finished_at || row.started_at || state.updated_at,
      usage,
    }
  })
  const numericScores = tasks
    .map((task) => task.score)
    .filter((score) => Number.isFinite(score))
  const visionScores = tasks
    .map((task) => task.visionScore?.score)
    .filter((score) => Number.isFinite(score))
  const deterministicScores = tasks
    .map((task) => task.deterministicScore?.score)
    .filter((score) => Number.isFinite(score))
  const levels = {}
  const drawingTypes = {}
  const evaluationStates = {}
  const executionStates = {}
  const verdicts = {}
  for (const task of tasks) {
    const level = task.level
    levels[level] ||= { total: 0, completed: 0, scored: 0, scoreSum: 0, cost: 0 }
    levels[level].total += 1
    if (task.status === 'completed') levels[level].completed += 1
    if (Number.isFinite(task.score)) {
      levels[level].scored += 1
      levels[level].scoreSum += task.score
    }
    levels[level].cost += task.usage.apiCost || 0
    const drawingType = task.drawingType
    drawingTypes[drawingType] ||= { total: 0, scored: 0, scoreSum: 0 }
    drawingTypes[drawingType].total += 1
    if (Number.isFinite(task.score)) {
      drawingTypes[drawingType].scored += 1
      drawingTypes[drawingType].scoreSum += task.score
    }
    evaluationStates[task.evaluationState] = (
      evaluationStates[task.evaluationState] || 0
    ) + 1
    executionStates[task.status] = (executionStates[task.status] || 0) + 1
    if (task.verdict) verdicts[task.verdict] = (verdicts[task.verdict] || 0) + 1
  }
  for (const level of Object.values(levels)) {
    level.meanScore = level.scored ? level.scoreSum / level.scored : null
    delete level.scoreSum
  }
  for (const drawingType of Object.values(drawingTypes)) {
    drawingType.meanScore = drawingType.scored
      ? drawingType.scoreSum / drawingType.scored
      : null
    delete drawingType.scoreSum
  }
  const aggregateUsage = tasks.reduce((aggregate, task) => {
    for (const key of [
      'inputTokens', 'cachedInputTokens', 'outputTokens',
      'reasoningOutputTokens', 'totalTokens', 'modelTurns',
    ]) aggregate[key] += task.usage[key]
    if (aggregate.apiCost != null) {
      if (task.usage.apiCost == null && task.usage.modelTurns > 0) {
        aggregate.apiCost = null
      } else {
        aggregate.apiCost += task.usage.apiCost || 0
      }
    }
    return aggregate
  }, {
    inputTokens: 0,
    cachedInputTokens: 0,
    outputTokens: 0,
    reasoningOutputTokens: 0,
    totalTokens: 0,
    modelTurns: 0,
    apiCost: pricing ? 0 : null,
  })
  const aggregate = {
    totalTasks: tasks.length,
    queuedTasks: tasks.filter((task) => task.status === 'queued').length,
    activeTasks: tasks.filter((task) => ACTIVE_STATES.has(task.status)).length,
    settledTasks: tasks.filter((task) => TERMINAL_EXECUTION_STATES.has(task.status)).length,
    completedTasks: tasks.filter((task) => task.status === 'completed').length,
    recoveredTasks: tasks.filter((task) => task.status === 'recovered').length,
    budgetExhaustedTasks: tasks.filter((task) => task.status === 'budget_exhausted').length,
    failedTasks: tasks.filter((task) => task.status === 'model_failed').length,
    infrastructureFailedTasks: tasks.filter((task) => task.status === 'infrastructure_failed').length,
    artifactMissingTasks: tasks.filter((task) => task.status === 'artifact_missing').length,
    artifactTasks: tasks.filter((task) => task.artifactBytes > 0).length,
    scoredTasks: numericScores.length,
    meanScore: numericScores.length
      ? numericScores.reduce((sum, score) => sum + score, 0) / numericScores.length
      : null,
    visionScoredTasks: visionScores.length,
    visionMeanScore: visionScores.length
      ? visionScores.reduce((sum, score) => sum + score, 0) / visionScores.length
      : null,
    deterministicScoredTasks: deterministicScores.length,
    deterministicMeanScore: deterministicScores.length
      ? deterministicScores.reduce((sum, score) => sum + score, 0)
        / deterministicScores.length
      : null,
    acceptedActions: tasks.reduce((sum, task) => sum + task.acceptedActions, 0),
    invalidActions: tasks.reduce((sum, task) => sum + task.invalidActions, 0),
    aggregateElapsedSeconds: tasks.reduce(
      (sum, task) => sum + (task.elapsedSeconds || 0), 0,
    ),
    usage: aggregateUsage,
    levels,
    drawingTypes,
    evaluationStates,
    executionStates,
    verdicts,
  }
  const models = [...new Set(tasks.map((task) => task.modelId))].sort()
  const providers = [...new Set(tasks.map((task) => task.provider))].sort()
  const reportedStatus = state.state || 'running'
  const updatedAtMs = Date.parse(state.updated_at || '')
  const controllerIsStale = ['starting', 'running'].includes(reportedStatus)
    && aggregate.activeTasks === 0
    && aggregate.queuedTasks > 0
    && !aggregate.evaluationStates.evaluating
    && Number.isFinite(updatedAtMs)
    && Date.now() - updatedAtMs > 5 * 60 * 1000
  const hasExecutionIssues = aggregate.failedTasks > 0
    || aggregate.infrastructureFailedTasks > 0
    || aggregate.artifactMissingTasks > 0
    || aggregate.evaluationStates.evaluation_failed > 0
    || aggregate.evaluationStates.evaluation_interrupted > 0
  const derivedStatus = aggregate.activeTasks > 0
    ? 'running'
    : aggregate.queuedTasks > 0
      ? (controllerIsStale ? 'interrupted' : reportedStatus)
      : aggregate.settledTasks === aggregate.totalTasks
        ? (hasExecutionIssues ? 'completed_with_issues' : 'completed')
        : reportedStatus
  return {
    id: benchmarkId,
    batchId: state.batch_id || null,
    name: state.benchmark_name || state.batch_id || benchmarkId,
    status: derivedStatus,
    modelId: models.length === 1 ? models[0] : `${models.length} models`,
    provider: providers.length === 1 ? providers[0] : `${providers.length} providers`,
    models,
    providers,
    startedAt: state.started_at || null,
    finishedAt: state.finished_at || null,
    updatedAt: state.updated_at || null,
    maxConcurrency: state.max_concurrency || state.preflight?.quota?.requested_instances || null,
    pricing,
    aggregate,
    tasks,
  }
}

function listBenchmarks(runsRoot) {
  return listBenchmarkIds(runsRoot)
    // Registry refreshes must never synchronously parse every model-turn file
    // across every historical run. Live run-state aggregates are sufficient
    // for registry totals; exact per-turn cost remains available in traces.
    .map((id) => benchmarkDetail(runsRoot, id, { preferAggregateUsage: true }))
    .filter(Boolean)
    .map((benchmark) => ({
      id: benchmark.id,
      batchId: benchmark.batchId,
      name: benchmark.name,
      status: benchmark.status,
      modelId: benchmark.modelId,
      provider: benchmark.provider,
      models: benchmark.models,
      providers: benchmark.providers,
      startedAt: benchmark.startedAt,
      updatedAt: benchmark.updatedAt,
      maxConcurrency: benchmark.maxConcurrency,
      aggregate: benchmark.aggregate,
    }))
    .sort((left, right) => String(right.updatedAt).localeCompare(String(left.updatedAt)))
}

function traceDetail(runsRoot, runId) {
  const runDirectory = safeRunDirectory(runsRoot, runId)
  if (!runDirectory || !fs.existsSync(runDirectory)) return null
  const summary = runSummary(runsRoot, runId)
  const frameDirectory = path.join(runDirectory, 'frames')
  const localFrames = fs.existsSync(frameDirectory)
    ? fs.readdirSync(frameDirectory).filter((name) => /^frame_\d+\.png$/.test(name)).sort()
    : []
  const turns = traceTurns(runDirectory)
  const state = readJson(path.join(runDirectory, 'run-state.json'))
  const frameNames = new Set(localFrames)
  const frameSha256 = new Map()
  let referenceSha256 = null
  for (const turn of turns) {
    for (const artifact of modelInputArtifacts(turn?.model_input)) {
      if (String(artifact.artifact_path || '') === 'reference.png') {
        referenceSha256 ||= normalizeSha256(artifact.sha256)
      }
      const match = String(artifact.artifact_path || '').match(/^frames\/(frame_\d+\.png)$/)
      if (match) {
        frameNames.add(match[1])
        const sha256 = normalizeSha256(artifact.sha256)
        if (sha256) frameSha256.set(match[1], sha256)
      }
    }
  }
  const acceptedActions = Number(state?.accepted_actions)
  if (Number.isInteger(acceptedActions) && acceptedActions >= 0) {
    frameNames.add(`frame_${String(acceptedActions).padStart(4, '0')}.png`)
  }
  const frames = [...frameNames].sort()
  const fullTracePath = path.join(runDirectory, 'model_turns.jsonl')
  const scores = readRunScores(runDirectory)
  const score = scores.displayed
  const hasTrustedRender = Boolean(
    fs.existsSync(path.join(runDirectory, 'evaluation', 'render.png'))
    || fs.existsSync(path.join(runDirectory, 'evaluation', 'candidate-render.png'))
    || score?.score_source === 'vision_judge',
  )
  const traceIndexed = fs.existsSync(traceSummaryPath(runDirectory))
    || !fs.existsSync(fullTracePath)
    || fs.statSync(fullTracePath).size <= MAX_INLINE_LEGACY_TRACE_BYTES
  return {
    summary,
    state,
    result: readJson(path.join(runDirectory, 'result.json')),
    score,
    scores: {
      vision: scores.vision,
      deterministic: scores.deterministic,
    },
    turns,
    receipts: readJsonl(path.join(runDirectory, 'input_receipts.jsonl')),
    traceIndexed,
    traceNotice: traceIndexed
      ? null
      : 'This legacy trace needs a compact viewer index before its turns can be displayed.',
    frames: frames.map((name) => {
      const sha256 = frameSha256.get(name) || null
      const version = sha256 ? `?sha256=${encodeURIComponent(sha256)}` : ''
      return {
        name,
        sha256,
        url: `/api/runs/${encodeURIComponent(runId)}/file/frames/${encodeURIComponent(name)}${version}`,
      }
    }),
    // A rollout always publishes its exact model-visible reference to cloud
    // storage. Keep the URL run-scoped so lightweight dashboard deployments do
    // not need a second copy of the private task corpus. The file route falls
    // back to the local manifest image for local-only development runs.
    referenceUrl: `/api/runs/${encodeURIComponent(runId)}/file/reference.png${
      referenceSha256 ? `?sha256=${encodeURIComponent(referenceSha256)}` : ''
    }`,
    artifacts: {
      dwg: fs.existsSync(path.join(runDirectory, 'attempt.dwg'))
        ? `/api/runs/${encodeURIComponent(runId)}/file/attempt.dwg`
        : null,
      candidateRender: hasTrustedRender
        ? `/api/runs/${encodeURIComponent(runId)}/file/evaluation/render.png`
        : null,
      diffRender: fs.existsSync(path.join(runDirectory, 'evaluation', 'diff.png'))
        ? `/api/runs/${encodeURIComponent(runId)}/file/evaluation/diff.png`
        : null,
    },
  }
}

export function createTraceApi(options = {}) {
  const runsRoot = path.resolve(options.runsRoot || process.env.AUTOCAD_BENCH_RUNS_ROOT || DEFAULT_RUNS_ROOT)
  const manifest = manifestByTask()
  async function refreshCloudMetadata(loader, request) {
    if (typeof loader !== 'function') return
    try {
      await loader(request)
    } catch {
      // Cloud mirroring is best effort. A transient AWS/auth failure must not
      // hide an already-synchronized local trace or take down the registry.
    }
  }
  return async function handleTraceApi(request, response) {
    const url = new URL(request.url || '/', 'http://127.0.0.1')
    if (!url.pathname.startsWith('/api/')) return false
    if (request.method !== 'GET') {
      sendJson(response, 405, { error: 'method_not_allowed' })
      return true
    }
    if (url.pathname === '/api/health') {
      sendJson(response, 200, { status: 'ok', runsRoot })
      return true
    }
    if (url.pathname === '/api/runs') {
      sendJson(response, 200, { runs: listRuns(runsRoot) })
      return true
    }
    if (url.pathname === '/api/benchmarks') {
      sendJson(response, 200, { benchmarks: listBenchmarks(runsRoot) })
      return true
    }
    const benchmarkMatch = url.pathname.match(/^\/api\/benchmarks\/([^/]+)$/)
    if (benchmarkMatch) {
      const benchmarkId = decodeURIComponent(benchmarkMatch[1])
      const benchmarkDirectory = safeRunDirectory(runsRoot, benchmarkId)
      if (benchmarkDirectory) {
        await refreshCloudMetadata(options.cloudBenchmarkLoader, {
          benchmarkId,
          benchmarkDirectory,
        })
      }
      const detail = benchmarkDetail(runsRoot, benchmarkId)
      sendJson(response, detail ? 200 : 404, detail || { error: 'benchmark_not_found' })
      return true
    }
    const runFileMatch = url.pathname.match(/^\/api\/runs\/([^/]+)\/file\/(.+)$/)
    if (runFileMatch) {
      const runId = decodeURIComponent(runFileMatch[1])
      const relativePath = decodeURIComponent(runFileMatch[2])
      const runDirectory = safeRunDirectory(runsRoot, runId)
      const artifact = runDirectory
        ? safeArtifact(runDirectory, relativePath)
        : null
      const requestedSha256 = normalizeSha256(url.searchParams.get('sha256'))
      const artifactExists = Boolean(
        artifact && fs.existsSync(artifact) && fs.statSync(artifact).isFile(),
      )
      const artifactMatches = artifactExists && (
        !requestedSha256 || fileSha256(artifact) === requestedSha256
      )
      if (artifactMatches) {
        sendFile(response, artifact, {
          immutable: Boolean(requestedSha256) || relativePath.toLowerCase().endsWith('.png'),
        })
        return true
      }
      if (relativePath === 'reference.png' && runDirectory) {
        const taskId = readJson(path.join(runDirectory, 'run-state.json'))?.task_id
          || readJson(path.join(runDirectory, 'result.json'))?.task_id
        const entry = manifest.get(taskId)
        const manifestImage = entry ? path.resolve(BENCHMARK_DATA_ROOT, entry.image_path) : null
        if (
          manifestImage
          && manifestImage.startsWith(BENCHMARK_DATA_ROOT + path.sep)
          && fs.existsSync(manifestImage)
          && fs.statSync(manifestImage).isFile()
          && (!requestedSha256 || fileSha256(manifestImage) === requestedSha256)
        ) {
          sendFile(response, manifestImage, { immutable: true })
          return true
        }
      }
      if (relativePath === 'evaluation/render.png' && runDirectory) {
        const legacyRender = safeArtifact(runDirectory, 'evaluation/candidate-render.png')
        if (legacyRender && fs.existsSync(legacyRender) && fs.statSync(legacyRender).isFile()) {
          sendFile(response, legacyRender, { immutable: true })
          return true
        }
      }
      if (
        artifact
        && !artifactExists
        && typeof options.cloudArtifactUrl === 'function'
      ) {
        try {
          const location = await options.cloudArtifactUrl({ runId, relativePath })
          if (location) {
            sendPrivateRedirect(response, location)
            return true
          }
        } catch {
          // Fall through to the local caching proxy if signing is unavailable.
        }
      }
      if (
        artifact
        && !artifactMatches
        && typeof options.cloudArtifactLoader === 'function'
      ) {
        try {
          await options.cloudArtifactLoader({
            runId,
            relativePath,
            artifact,
            expectedSha256: requestedSha256,
            force: artifactExists,
          })
        } catch {
          // The normal 404 below is the stable contract when the cloud mirror
          // also lacks an artifact or AWS is temporarily unavailable.
        }
      }
      const refreshedMatches = Boolean(
        artifact
        && fs.existsSync(artifact)
        && fs.statSync(artifact).isFile()
        && (!requestedSha256 || fileSha256(artifact) === requestedSha256),
      )
      let servedArtifact = refreshedMatches ? artifact : null
      if (!servedArtifact && relativePath === 'evaluation/render.png' && runDirectory) {
        const fallbackRelativePath = 'evaluation/candidate-render.png'
        const fallbackArtifact = safeArtifact(runDirectory, fallbackRelativePath)
        if (
          fallbackArtifact
          && !fs.existsSync(fallbackArtifact)
          && typeof options.cloudArtifactLoader === 'function'
        ) {
          try {
            await options.cloudArtifactLoader({
              runId,
              relativePath: fallbackRelativePath,
              artifact: fallbackArtifact,
            })
          } catch {
            // Older evaluator runs may not have either render filename.
          }
        }
        if (
          fallbackArtifact
          && fs.existsSync(fallbackArtifact)
          && fs.statSync(fallbackArtifact).isFile()
        ) {
          servedArtifact = fallbackArtifact
        }
      }
      if (!servedArtifact && relativePath === 'reference.png') {
        const taskId = runDirectory
          ? readJson(path.join(runDirectory, 'run-state.json'))?.task_id
            || readJson(path.join(runDirectory, 'result.json'))?.task_id
          : null
        const entry = manifest.get(taskId)
        const manifestImage = entry ? path.resolve(BENCHMARK_DATA_ROOT, entry.image_path) : null
        if (
          manifestImage
          && manifestImage.startsWith(BENCHMARK_DATA_ROOT + path.sep)
          && fs.existsSync(manifestImage)
          && fs.statSync(manifestImage).isFile()
        ) {
          servedArtifact = manifestImage
        }
      }
      sendFile(response, servedArtifact, {
        // Run-scoped PNGs are append-only/final artifacts. Browser caching
        // makes reopening a rollout instant and avoids repeated S3 downloads.
        immutable: Boolean(requestedSha256) || relativePath.toLowerCase().endsWith('.png'),
      })
      return true
    }
    const runMatch = url.pathname.match(/^\/api\/runs\/([^/]+)$/)
    if (runMatch) {
      const runId = decodeURIComponent(runMatch[1])
      const runDirectory = safeRunDirectory(runsRoot, runId)
      if (runDirectory) {
        // Live cloud-controller runs may be present in batch-state before their
        // rollout directory has ever been mirrored locally. Bootstrap and
        // refresh the small trace index here; frame PNGs remain lazy-loaded by
        // the /file route below.
        await refreshCloudMetadata(options.cloudTraceLoader, {
          runId,
          runDirectory,
        })
      }
      const detail = traceDetail(runsRoot, runId)
      sendJson(response, detail ? 200 : 404, detail || { error: 'run_not_found' })
      return true
    }
    const referenceMatch = url.pathname.match(/^\/api\/reference\/([^/]+)$/)
    if (referenceMatch) {
      const taskId = decodeURIComponent(referenceMatch[1])
      const entry = manifest.get(taskId)
      const imagePath = entry ? path.resolve(BENCHMARK_DATA_ROOT, entry.image_path) : null
      const safeImage = imagePath && imagePath.startsWith(BENCHMARK_DATA_ROOT + path.sep) ? imagePath : null
      sendFile(response, safeImage)
      return true
    }
    sendJson(response, 404, { error: 'not_found' })
    return true
  }
}

export const _test = {
  readJsonl,
  safeRunDirectory,
  safeArtifact,
  runSummary,
  traceDetail,
  traceTurns,
  listRuns,
  usageSummary,
  runUsage,
  compactTurn,
  normalizeSha256,
  fileSha256,
  benchmarkDetail,
  listBenchmarks,
  executionState,
  readRunScores,
  scoreSummary,
}