import assert from 'node:assert/strict'
import { createHash } from 'node:crypto'
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import { once } from 'node:events'
import { Writable } from 'node:stream'
import test from 'node:test'

import { _test, createTraceApi } from '../trace-api.mjs'

class TestResponse extends Writable {
  constructor() {
    super()
    this.headers = {}
    this.body = []
    this.statusCode = 0
  }

  setHeader(name, value) {
    this.headers[name] = value
  }

  _write(chunk, _encoding, callback) {
    this.body.push(Buffer.from(chunk))
    callback()
  }
}

test('reads complete JSONL rows while a writer is appending a partial row', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-jsonl-'))
  const file = path.join(root, 'events.json')
  fs.writeFileSync(file, '{"actionIndex":1}\n{"actionIndex":2')
  assert.deepEqual(_test.readJsonl(file), [{ actionIndex: 1 }])
})

test('rejects run and artifact path traversal', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-safe-'))
  const run = path.join(root, 'run-001')
  fs.mkdirSync(run)
  assert.equal(_test.safeRunDirectory(root, '../outside'), null)
  assert.equal(_test.safeRunDirectory(root, 'run-001'), run)
  assert.equal(_test.safeRunDirectory(root, 'batch/rollouts/run-001'), path.join(root, 'batch', 'rollouts', 'run-001'))
  assert.equal(_test.safeRunDirectory(root, 'batch//run-001'), null)
  assert.equal(_test.safeArtifact(run, '../../secret'), null)
  assert.equal(_test.safeArtifact(run, 'frames/frame_0000.png'), path.join(run, 'frames', 'frame_0000.png'))
})

test('recursively indexes batch rollout directories', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-batch-'))
  const rollout = path.join(root, 'batch-001', 'rollouts', 'model-a')
  fs.mkdirSync(rollout, { recursive: true })
  fs.writeFileSync(path.join(rollout, 'run-state.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'model-a',
    provider: 'openai',
    state: 'running',
    updated_at: '2026-07-16T00:00:00Z',
  }))

  const runs = _test.listRuns(root)

  assert.equal(runs.length, 1)
  assert.equal(runs[0].id, 'batch-001/rollouts/model-a')
  assert.equal(runs[0].status, 'running')
})

test('indexes an unsuccessful wall-time rollout as budget exhausted', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-run-'))
  const run = path.join(root, 'gpt-run')
  fs.mkdirSync(run)
  fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'gpt-5.6-sol',
    provider: 'openai',
    reasoning_effort: 'xhigh',
    completed: false,
    stop_reason: 'wall_timeout',
    accepted_actions: 40,
    invalid_actions: 0,
    screenshots: 41,
    artifact_bytes: 0,
  }))
  const summary = _test.runSummary(root, 'gpt-run')
  assert.equal(summary.status, 'budget_exhausted')
  assert.equal(summary.stopReason, 'wall_timeout')
  assert.equal(summary.acceptedActions, 40)
  assert.equal(summary.evaluationState, 'no_artifact')
})

test('prefers the versioned vision judgment over the legacy deterministic score', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-vision-score-'))
  const run = path.join(root, 'vision-run')
  fs.mkdirSync(path.join(run, 'evaluation', 'vision-judge'), { recursive: true })
  fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'gpt-5.6-sol',
    provider: 'openai',
    state: 'completed',
  }))
  fs.writeFileSync(path.join(run, 'evaluation', 'score.json'), JSON.stringify({
    score: 0.52,
    valid: true,
    scoring_method: 'vector-semantic-v2',
  }))
  fs.writeFileSync(
    path.join(run, 'evaluation', 'vision-judge', 'score.json'),
    JSON.stringify({
      prompt_version: 'autocad-vision-judge-v1',
      overall_score: 0.93,
      target_acquired: true,
      confidence: 0.91,
      needs_human_review: false,
      component_scores: {
        geometry: 0.98,
        dimensions: 0.96,
        annotations: 0.75,
        presentation: 0.70,
      },
    }),
  )

  const summary = _test.runSummary(root, 'vision-run')
  const detail = _test.traceDetail(root, 'vision-run')

  assert.equal(summary.score, 0.93)
  assert.equal(summary.scoreMethod, 'vision:autocad-vision-judge-v1')
  assert.equal(summary.scoreSource, 'vision_judge')
  assert.equal(summary.visionScore.score, 0.93)
  assert.equal(summary.visionScore.geometry, 0.98)
  assert.equal(summary.deterministicScore.score, 0.52)
  assert.equal(summary.deterministicScore.method, 'vector-semantic-v2')
  assert.equal(summary.targetAcquired, true)
  assert.equal(summary.evaluationState, 'scored')
  assert.equal(detail.score.geometry_similarity, 0.98)
  assert.equal(detail.score.score_source, 'vision_judge')
  assert.equal(detail.scores.vision.overall_score, 0.93)
  assert.equal(detail.scores.deterministic.score, 0.52)
})

test('shows automatic evaluation progress immediately after model completion', () => {
  const heartbeat = new Date().toISOString()
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-evaluating-'))
  const run = path.join(root, 'evaluating-run')
  fs.mkdirSync(path.join(run, 'evaluation'), { recursive: true })
  fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'gpt-test',
    provider: 'openai',
    completed: true,
    artifact_bytes: 1234,
  }))
  fs.writeFileSync(path.join(run, 'evaluation', 'status.json'), JSON.stringify({
    stage: 'vision_judging',
    updated_at: heartbeat,
  }))

  const summary = _test.runSummary(root, 'evaluating-run')

  assert.equal(summary.evaluationState, 'evaluating')
  assert.equal(summary.evaluationStage, 'vision_judging')
  assert.equal(summary.updatedAt, heartbeat)
})

test('reports independent vision and deterministic benchmark means', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-dual-score-'))
  const benchmark = path.join(root, 'benchmarks', 'dual-score')
  const run = path.join(benchmark, 'rollouts', 'task-001')
  fs.mkdirSync(path.join(run, 'evaluation', 'vision-judge'), { recursive: true })
  fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({
    state: 'completed',
    rollouts: {
      'task-001': {
        task_id: 'task-001', model_id: 'model', provider: 'openai', state: 'completed',
      },
    },
  }))
  fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({
    task_id: 'task-001', model_id: 'model', provider: 'openai',
    completed: true, artifact_bytes: 100,
  }))
  fs.writeFileSync(path.join(run, 'evaluation', 'score.json'), JSON.stringify({
    score: 0.4,
    valid: true,
    scoring_method: 'vector-semantic-v2',
    geometry_similarity: 0.5,
  }))
  fs.writeFileSync(
    path.join(run, 'evaluation', 'vision-judge', 'score.json'),
    JSON.stringify({
      overall_score: 0.8,
      verdict: 'mostly_acquired',
      component_scores: { geometry: 0.9, dimensions: 0.7, annotations: 0.6, presentation: 0.5 },
    }),
  )

  const detail = _test.benchmarkDetail(root, 'benchmarks/dual-score')

  assert.equal(detail.aggregate.meanScore, 0.8)
  assert.equal(detail.aggregate.visionMeanScore, 0.8)
  assert.equal(detail.aggregate.visionScoredTasks, 1)
  assert.equal(detail.aggregate.deterministicMeanScore, 0.4)
  assert.equal(detail.aggregate.deterministicScoredTasks, 1)
  assert.equal(detail.tasks[0].visionScore.score, 0.8)
  assert.equal(detail.tasks[0].deterministicScore.score, 0.4)
})

test('does not show a stale evaluator heartbeat as live forever', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-stale-evaluation-'))
  const run = path.join(root, 'stale-evaluation')
  fs.mkdirSync(path.join(run, 'evaluation'), { recursive: true })
  fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({
    task_id: 'task-001',
    completed: true,
    stop_reason: 'done',
    artifact_bytes: 100,
  }))
  fs.writeFileSync(path.join(run, 'evaluation', 'status.json'), JSON.stringify({
    stage: 'vision_judging',
    updated_at: '2020-01-01T00:00:00Z',
  }))

  assert.equal(_test.runSummary(root, 'stale-evaluation').evaluationState, 'evaluation_interrupted')
})

test('labels a batch process without a pilot trace as startup, not handoff', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-startup-'))
  const benchmark = path.join(root, 'benchmark-starting')
  fs.mkdirSync(benchmark, { recursive: true })
  fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({
    batch_id: 'batch-starting',
    state: 'running',
    rollouts: {
      'task-002': {
        task_id: 'task-002',
        model_id: 'model-test',
        provider: 'openrouter',
        state: 'running',
      },
    },
  }))

  const detail = _test.benchmarkDetail(root, 'benchmark-starting')

  assert.equal(detail.tasks[0].status, 'starting')
  assert.equal(detail.tasks[0].evaluationState, 'running')
})

test('retains batch grouping receipts and intermediate audit frames', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-batch-turn-'))
  const run = path.join(root, 'batched-run')
  const frames = path.join(run, 'frames')
  fs.mkdirSync(frames, { recursive: true })
  fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'gpt-test',
    provider: 'openai',
    state: 'running',
  }))
  fs.writeFileSync(path.join(run, 'model_turns.jsonl'), `${JSON.stringify({
    turn: 1,
    tool_use_id: 'call-batch',
    action: {
      actions: [
        { action: 'type', text: 'LINE', intent: 'Type LINE.' },
        { action: 'key', keys: ['ENTER'], intent: 'Submit LINE.' },
      ],
    },
  })}\n`)
  fs.writeFileSync(path.join(run, 'input_receipts.jsonl'), [
    { action_index: 1, tool_use_id: 'call-batch', batch_index: 1, batch_size: 2 },
    { action_index: 2, tool_use_id: 'call-batch', batch_index: 2, batch_size: 2 },
  ].map(JSON.stringify).join('\n') + '\n')
  for (const name of ['frame_0000.png', 'frame_0001.png', 'frame_0002.png']) {
    fs.writeFileSync(path.join(frames, name), 'png')
  }

  const detail = _test.traceDetail(root, 'batched-run')

  assert.equal(detail.turns[0].action.actions.length, 2)
  assert.deepEqual(detail.receipts.map((receipt) => receipt.batch_index), [1, 2])
  assert.deepEqual(detail.frames.map((frame) => frame.name), [
    'frame_0000.png',
    'frame_0001.png',
    'frame_0002.png',
  ])
})

test('indexes cloud-backed frames referenced by compact resumed traces', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-frames-'))
  const run = path.join(root, 'cloud-frame-run')
  const frames = path.join(run, 'frames')
  fs.mkdirSync(frames, { recursive: true })
  fs.writeFileSync(path.join(frames, 'frame_0000.png'), 'png')
  fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'gpt-test',
    provider: 'openai',
    state: 'completed',
    accepted_actions: 12,
  }))
  fs.writeFileSync(path.join(run, 'model_turn_summaries.jsonl'), `${JSON.stringify({
    turn: 2,
    model_input: {
      artifacts: [{ artifact_path: 'frames/frame_0010.png', sequence: 10 }],
    },
  })}\n`)

  const detail = _test.traceDetail(root, 'cloud-frame-run')

  assert.deepEqual(detail.frames.map((frame) => frame.name), [
    'frame_0000.png',
    'frame_0010.png',
    'frame_0012.png',
  ])
})

test('advertises and lazily hydrates cloud reference and trusted render images', async () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-images-'))
  const runId = 'benchmarks/sol/rollouts/task-022'
  const run = path.join(root, runId)
  fs.mkdirSync(path.join(run, 'evaluation', 'vision-judge'), { recursive: true })
  fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({
    task_id: 'cloud-only-task',
    model_id: 'gpt-test',
    provider: 'openai',
    state: 'completed',
  }))
  fs.writeFileSync(
    path.join(run, 'evaluation', 'vision-judge', 'score.json'),
    JSON.stringify({
      prompt_version: 'autocad-vision-judge-v3',
      overall_score: 0.9,
      component_scores: { geometry: 0.9 },
    }),
  )

  const detail = _test.traceDetail(root, runId)
  assert.equal(
    detail.referenceUrl,
    `/api/runs/${encodeURIComponent(runId)}/file/reference.png`,
  )
  assert.equal(
    detail.artifacts.candidateRender,
    `/api/runs/${encodeURIComponent(runId)}/file/evaluation/render.png`,
  )

  const hydrated = []
  const api = createTraceApi({
    runsRoot: root,
    cloudArtifactLoader: async (request) => {
      hydrated.push(request.relativePath)
      fs.mkdirSync(path.dirname(request.artifact), { recursive: true })
      fs.writeFileSync(request.artifact, `cloud:${request.relativePath}`)
    },
  })
  for (const relativePath of ['reference.png', 'evaluation/render.png']) {
    const response = new TestResponse()
    const finished = once(response, 'finish')
    await api({
      method: 'GET',
      url: `/api/runs/${encodeURIComponent(runId)}/file/${relativePath}`,
    }, response)
    await finished
    assert.equal(response.statusCode, 200)
    assert.equal(response.headers['Content-Type'], 'image/png')
    assert.equal(response.headers['Cache-Control'], 'private, max-age=31536000, immutable')
    assert.equal(Buffer.concat(response.body).toString(), `cloud:${relativePath}`)
  }
  assert.deepEqual(hydrated, ['reference.png', 'evaluation/render.png'])
})

test('hydrates a missing frame from the cloud artifact loader on demand', async () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-fetch-'))
  const run = path.join(root, 'batch', 'rollouts', 'task-001')
  fs.mkdirSync(run, { recursive: true })
  let hydrated = null
  const api = createTraceApi({
    runsRoot: root,
    cloudArtifactLoader: async (request) => {
      hydrated = request
      fs.mkdirSync(path.dirname(request.artifact), { recursive: true })
      fs.writeFileSync(request.artifact, 'cloud-png')
    },
  })
  const response = new TestResponse()
  const finished = once(response, 'finish')

  const handled = await api({
    method: 'GET',
    url: '/api/runs/batch%2Frollouts%2Ftask-001/file/frames/frame_0042.png',
  }, response)
  await finished

  assert.equal(handled, true)
  assert.equal(response.statusCode, 200)
  assert.equal(response.headers['Content-Type'], 'image/png')
  assert.equal(Buffer.concat(response.body).toString(), 'cloud-png')
  assert.equal(hydrated.runId, 'batch/rollouts/task-001')
  assert.equal(hydrated.relativePath, 'frames/frame_0042.png')
})

test('redirects a cold cloud image directly to a short-lived private URL', async () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-redirect-'))
  const runId = 'batch/rollouts/task-001'
  fs.mkdirSync(path.join(root, runId), { recursive: true })
  let signed = null
  const api = createTraceApi({
    runsRoot: root,
    cloudArtifactUrl: async (request) => {
      signed = request
      return 'https://private-origin.example/signed-frame'
    },
    cloudArtifactLoader: async () => {
      assert.fail('the proxy loader must not run after a successful signed redirect')
    },
  })
  const response = new TestResponse()
  const finished = once(response, 'finish')

  await api({
    method: 'GET',
    url: `/api/runs/${encodeURIComponent(runId)}/file/frames/frame_0042.png`,
  }, response)
  await finished

  assert.equal(response.statusCode, 307)
  assert.equal(response.headers.Location, 'https://private-origin.example/signed-frame')
  assert.equal(response.headers['Cache-Control'], 'private, max-age=240')
  assert.deepEqual(signed, {
    runId,
    relativePath: 'frames/frame_0042.png',
  })
})

test('bootstraps a missing live rollout trace from cloud metadata', async () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-index-'))
  const runId = 'benchmarks/live-suite/rollouts/task-004'
  const run = path.join(root, runId)
  let hydration = null
  const api = createTraceApi({
    runsRoot: root,
    cloudTraceLoader: async (request) => {
      hydration = request
      fs.mkdirSync(request.runDirectory, { recursive: true })
      fs.writeFileSync(path.join(request.runDirectory, 'run-state.json'), JSON.stringify({
        task_id: 'task-004',
        model_id: 'claude-fable-5',
        provider: 'anthropic',
        state: 'running',
        accepted_actions: 42,
      }))
      fs.writeFileSync(path.join(request.runDirectory, 'model_turn_summaries.jsonl'), `${JSON.stringify({
        turn: 1,
        tool_use_id: 'tool-1',
        action: { actions: [{ action: 'click', x: 1, y: 2 }] },
        model_input: {
          artifacts: [{ artifact_path: 'frames/frame_0041.png' }],
        },
      })}\n`)
    },
  })
  const response = new TestResponse()
  const finished = once(response, 'finish')

  await api({
    method: 'GET',
    url: `/api/runs/${encodeURIComponent(runId)}`,
  }, response)
  await finished

  assert.equal(response.statusCode, 200)
  assert.equal(hydration.runId, runId)
  assert.equal(hydration.runDirectory, run)
  const detail = JSON.parse(Buffer.concat(response.body).toString())
  assert.equal(detail.summary.acceptedActions, 42)
  assert.equal(detail.turns.length, 1)
  assert.deepEqual(detail.frames.map((frame) => frame.name), [
    'frame_0041.png',
    'frame_0042.png',
  ])
})

test('refreshes stale live benchmark metadata before returning its detail', async () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-benchmark-cloud-refresh-'))
  const benchmarkId = 'benchmarks/live-suite'
  const benchmark = path.join(root, benchmarkId)
  const run = path.join(benchmark, 'rollouts', 'task-004')
  fs.mkdirSync(run, { recursive: true })
  fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({
    state: 'running',
    updated_at: '2026-07-18T00:00:00Z',
    rollouts: {
      'task-004': {
        task_id: 'task-004',
        model_id: 'claude-fable-5',
        provider: 'anthropic',
        state: 'running',
      },
    },
  }))
  fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({
    task_id: 'task-004',
    model_id: 'claude-fable-5',
    provider: 'anthropic',
    state: 'running',
    accepted_actions: 1,
  }))
  let refreshCount = 0
  const api = createTraceApi({
    runsRoot: root,
    cloudBenchmarkLoader: async ({ benchmarkDirectory }) => {
      refreshCount += 1
      fs.writeFileSync(path.join(
        benchmarkDirectory, 'rollouts', 'task-004', 'run-state.json',
      ), JSON.stringify({
        task_id: 'task-004',
        model_id: 'claude-fable-5',
        provider: 'anthropic',
        state: 'running',
        accepted_actions: 87,
        turn_count: 21,
      }))
    },
  })
  const response = new TestResponse()
  const finished = once(response, 'finish')

  await api({
    method: 'GET',
    url: `/api/benchmarks/${encodeURIComponent(benchmarkId)}`,
  }, response)
  await finished

  assert.equal(response.statusCode, 200)
  assert.equal(refreshCount, 1)
  const detail = JSON.parse(Buffer.concat(response.body).toString())
  assert.equal(detail.tasks[0].acceptedActions, 87)
  assert.equal(detail.aggregate.acceptedActions, 87)
  assert.equal(detail.aggregate.usage.modelTurns, 21)
})

test('content-addresses model frames and replaces a stale local frame', async () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-stale-frame-'))
  const runId = 'batch/rollouts/task-001'
  const run = path.join(root, runId)
  const frames = path.join(run, 'frames')
  fs.mkdirSync(frames, { recursive: true })
  const freshFrame = Buffer.from('fresh-cloud-frame')
  const expectedSha256 = createHash('sha256').update(freshFrame).digest('hex')
  fs.writeFileSync(path.join(frames, 'frame_0042.png'), 'stale-local-frame')
  fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'gpt-test',
    provider: 'openai',
    state: 'completed',
  }))
  fs.writeFileSync(path.join(run, 'model_turn_summaries.jsonl'), `${JSON.stringify({
    turn: 1,
    model_input: {
      artifacts: [{
        artifact_path: 'frames/frame_0042.png',
        sha256: expectedSha256,
        sequence: 42,
      }],
    },
  })}\n`)

  const detail = _test.traceDetail(root, runId)
  assert.equal(detail.frames[0].sha256, expectedSha256)
  assert.equal(
    detail.frames[0].url,
    `/api/runs/batch%2Frollouts%2Ftask-001/file/frames/frame_0042.png?sha256=${expectedSha256}`,
  )

  let hydration = null
  const api = createTraceApi({
    runsRoot: root,
    cloudArtifactLoader: async (request) => {
      hydration = request
      fs.writeFileSync(request.artifact, freshFrame)
    },
  })
  const response = new TestResponse()
  const finished = once(response, 'finish')
  await api({
    method: 'GET',
    url: `${detail.frames[0].url}`,
  }, response)
  await finished

  assert.equal(response.statusCode, 200)
  assert.equal(Buffer.concat(response.body).toString(), freshFrame.toString())
  assert.equal(response.headers['Cache-Control'], 'private, max-age=31536000, immutable')
  assert.equal(hydration.force, true)
  assert.equal(hydration.expectedSha256, expectedSha256)
})

test('never serves a stale frame when cloud refresh does not match its checksum', async () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-reject-stale-frame-'))
  const runId = 'batch/rollouts/task-001'
  const run = path.join(root, runId)
  const frames = path.join(run, 'frames')
  fs.mkdirSync(frames, { recursive: true })
  const expectedSha256 = createHash('sha256').update('expected-frame').digest('hex')
  fs.writeFileSync(path.join(frames, 'frame_0001.png'), 'stale-frame')

  const api = createTraceApi({
    runsRoot: root,
    cloudArtifactLoader: async () => {},
  })
  const response = new TestResponse()
  const finished = once(response, 'finish')
  await api({
    method: 'GET',
    url: `/api/runs/${encodeURIComponent(runId)}/file/frames/frame_0001.png?sha256=${expectedSha256}`,
  }, response)
  await finished

  assert.equal(response.statusCode, 404)
  assert.match(Buffer.concat(response.body).toString(), /artifact_not_found/)
})

test('uses compact turn summaries without returning repeated request history', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-compact-'))
  const run = path.join(root, 'compact-run')
  fs.mkdirSync(run)
  fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'claude-test',
    provider: 'anthropic',
    state: 'completed',
    turn_count: 1,
  }))
  fs.writeFileSync(path.join(run, 'model_turns.jsonl'), 'x'.repeat(9 * 1024 * 1024))
  fs.writeFileSync(path.join(run, 'model_turn_summaries.jsonl'), `${JSON.stringify({
    turn: 1,
    tool_use_id: 'call-compact',
    action: { action: 'done' },
    model_input: {
      model: 'claude-test',
      artifacts: [{ artifact_path: 'frames/frame_0000.png' }],
    },
  })}\n`)

  const detail = _test.traceDetail(root, 'compact-run')

  assert.equal(detail.traceIndexed, true)
  assert.equal(detail.turns.length, 1)
  assert.equal(detail.turns[0].tool_use_id, 'call-compact')
  assert.equal(JSON.stringify(detail).includes('repeated request history'), false)
  assert.equal('events' in detail, false)
  assert.equal('frameEvents' in detail, false)
})

test('does not synchronously parse a large unindexed trace for benchmark usage', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-large-'))
  const run = path.join(root, 'large-run')
  fs.mkdirSync(run)
  const trace = path.join(run, 'model_turns.jsonl')
  fs.writeFileSync(trace, '')
  fs.truncateSync(trace, 9 * 1024 * 1024)

  const usage = _test.runUsage(run, { input_per_million: 5 }, {
    turn_count: 42,
    usage: { input_tokens: 1200, output_tokens: 30, total_tokens: 1230 },
  })

  assert.equal(usage.modelTurns, 42)
  assert.equal(usage.inputTokens, 1200)
  assert.equal(usage.totalTokens, 1230)
  assert.equal(usage.apiCost, null)
})

test('uses the live aggregate instead of parsing a large compact usage index', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-summary-large-'))
  const run = path.join(root, 'large-summary-run')
  fs.mkdirSync(run)
  const summary = path.join(run, 'model_turn_summaries.jsonl')
  fs.writeFileSync(summary, '')
  fs.truncateSync(summary, 600 * 1024)

  const usage = _test.runUsage(run, { input_per_million: 5 }, {
    turn_count: 77,
    usage: { input_tokens: 9000, output_tokens: 80, total_tokens: 9080 },
  })

  assert.equal(usage.modelTurns, 77)
  assert.equal(usage.inputTokens, 9000)
  assert.equal(usage.totalTokens, 9080)
  assert.equal(usage.apiCost, null)
})

test('reconciles a stale running batch row with a completed rollout trace', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-stale-batch-'))
  const benchmark = path.join(root, 'stale-batch')
  const run = path.join(benchmark, 'rollouts', 'task-001')
  fs.mkdirSync(run, { recursive: true })
  fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({
    state: 'running',
    updated_at: '2020-01-01T00:00:00Z',
    rollouts: {
      'task-001': {
        task_id: 'task-001',
        model_id: 'claude-test',
        provider: 'anthropic',
        state: 'running',
      },
      'task-002': {
        task_id: 'task-002',
        model_id: 'claude-test',
        provider: 'anthropic',
        state: 'queued',
      },
    },
  }))
  fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'claude-test',
    provider: 'anthropic',
    state: 'completed',
    turn_count: 1,
  }))

  const detail = _test.benchmarkDetail(root, 'stale-batch')

  assert.equal(detail.tasks[0].status, 'completed')
  assert.equal(detail.aggregate.completedTasks, 1)
  assert.equal(detail.aggregate.activeTasks, 0)
  assert.equal(detail.status, 'interrupted')
})

test('calculates cached and long-context token costs per model turn', () => {
  const pricing = {
    input_per_million: 5,
    cached_input_per_million: 0.5,
    output_per_million: 30,
    long_context_threshold_tokens: 272000,
    long_context_input_multiplier: 2,
    long_context_output_multiplier: 1.5,
  }
  const usage = _test.usageSummary([
    { usage: {
      input_tokens: 1000,
      cached_input_tokens: 400,
      output_tokens: 100,
      reasoning_output_tokens: 60,
      total_tokens: 1100,
    } },
    { usage: { input_tokens: 300000, output_tokens: 1000, total_tokens: 301000 } },
  ], pricing)

  assert.equal(usage.inputTokens, 301000)
  assert.equal(usage.cachedInputTokens, 400)
  assert.equal(usage.outputTokens, 1100)
  assert.equal(usage.reasoningOutputTokens, 60)
  assert.ok(Math.abs(usage.apiCost - 3.0512) < 1e-12)
})

test('aggregates a live multi-task benchmark from batch state and rollout traces', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-benchmark-'))
  const benchmark = path.join(root, 'benchmarks', 'full-suite')
  const completed = path.join(benchmark, 'rollouts', 'task-001')
  fs.mkdirSync(completed, { recursive: true })
  fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({
    schema_version: 2,
    batch_id: 'batch-full',
    benchmark_name: 'Full suite',
    state: 'running',
    started_at: '2026-07-16T00:00:00Z',
    updated_at: '2026-07-16T00:10:00Z',
    max_concurrency: 10,
    pricing: {
      input_per_million: 5,
      cached_input_per_million: 0.5,
      output_per_million: 30,
    },
    rollouts: {
      'task-001': {
        task_id: 'task-001',
        model_id: 'gpt-5.6-sol',
        provider: 'openai',
        state: 'completed',
      },
      'task-022': {
        task_id: 'task-022',
        model_id: 'gpt-5.6-sol',
        provider: 'openai',
        state: 'queued',
      },
    },
  }))
  fs.writeFileSync(path.join(completed, 'result.json'), JSON.stringify({
    task_id: 'task-001',
    model_id: 'gpt-5.6-sol',
    provider: 'openai',
    completed: true,
    accepted_actions: 20,
    artifact_bytes: 1234,
  }))
  fs.writeFileSync(path.join(completed, 'model_turns.jsonl'), `${JSON.stringify({
    usage: { input_tokens: 1000, cached_input_tokens: 400, output_tokens: 100 },
  })}\n`)

  const detail = _test.benchmarkDetail(root, 'benchmarks/full-suite')

  assert.equal(detail.name, 'Full suite')
  assert.equal(detail.aggregate.totalTasks, 2)
  assert.equal(detail.aggregate.completedTasks, 1)
  assert.equal(detail.aggregate.queuedTasks, 1)
  assert.equal(detail.aggregate.artifactTasks, 1)
  assert.equal(detail.aggregate.usage.inputTokens, 1000)
  assert.ok(Math.abs(detail.aggregate.usage.apiCost - 0.0062) < 1e-12)
  assert.deepEqual(detail.tasks.map((task) => task.level), ['basic', 'basic'])
  assert.deepEqual(detail.tasks.map((task) => task.drawingType), ['2d', '3d'])
  assert.deepEqual(detail.aggregate.drawingTypes, {
    '2d': { total: 1, scored: 0, meanScore: null },
    '3d': { total: 1, scored: 0, meanScore: null },
  })
  assert.deepEqual(detail.aggregate.evaluationStates, {
    awaiting_score: 1,
    queued: 1,
  })
  assert.equal(_test.listBenchmarks(root)[0].id, 'benchmarks/full-suite')
})

test('keeps execution, recovery, budget, infrastructure, and evaluation states separate', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-outcomes-'))
  const benchmark = path.join(root, 'benchmarks', 'outcomes')
  const cases = {
    completed: { completed: true, stop_reason: 'done', artifact_bytes: 10 },
    recovered: { completed: false, stop_reason: 'recovered_finalized_session', artifact_bytes: 10 },
    budget: { completed: false, stop_reason: 'max_actions', artifact_bytes: 10 },
    infra: { completed: false, stop_reason: 'rollout_error', failure: 'ConnectError', artifact_bytes: 10 },
    model: { completed: false, stop_reason: 'invalid_action_limit', failure: 'invalid', artifact_bytes: 10 },
  }
  fs.mkdirSync(benchmark, { recursive: true })
  fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({
    state: 'running',
    updated_at: '2020-01-01T00:00:00Z',
    rollouts: Object.fromEntries(Object.keys(cases).map((task) => [task, {
      task_id: 'task-001', model_id: 'model', provider: 'openai', state: 'failed',
      failure: 'stale wrapper failure',
    }])),
  }))
  for (const [name, result] of Object.entries(cases)) {
    const run = path.join(benchmark, 'rollouts', name)
    fs.mkdirSync(path.join(run, 'evaluation'), { recursive: true })
    fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({
      task_id: 'task-001', model_id: 'model', provider: 'openai', ...result,
    }))
  }
  fs.writeFileSync(
    path.join(benchmark, 'rollouts', 'completed', 'evaluation', 'status.json'),
    JSON.stringify({ stage: 'failed', error: 'VisionJudgeError: ReadTimeout' }),
  )

  const detail = _test.benchmarkDetail(root, 'benchmarks/outcomes')

  assert.equal(detail.status, 'completed_with_issues')
  assert.equal(detail.aggregate.settledTasks, 5)
  assert.equal(detail.aggregate.completedTasks, 1)
  assert.equal(detail.aggregate.recoveredTasks, 1)
  assert.equal(detail.aggregate.budgetExhaustedTasks, 1)
  assert.equal(detail.aggregate.infrastructureFailedTasks, 1)
  assert.equal(detail.aggregate.failedTasks, 1)
  assert.equal(detail.tasks.find((task) => task.rolloutId === 'completed').status, 'completed')
  assert.equal(detail.tasks.find((task) => task.rolloutId === 'completed').failure, null)
  assert.equal(detail.tasks.find((task) => task.rolloutId === 'budget').failure, null)
  assert.equal(detail.tasks.find((task) => task.rolloutId === 'infra').failure, 'ConnectError')
  assert.equal(detail.tasks.find((task) => task.rolloutId === 'completed').evaluationState, 'evaluation_failed')
})