| import assert from 'node:assert/strict' |
| import { createHash } from 'node:crypto' |
| import fs from 'node:fs' |
| import os from 'node:os' |
| import path from 'node:path' |
| import { once } from 'node:events' |
| import { Writable } from 'node:stream' |
| import test from 'node:test' |
|
|
| import { _test, createTraceApi } from '../trace-api.mjs' |
|
|
| class TestResponse extends Writable { |
| constructor() { |
| super() |
| this.headers = {} |
| this.body = [] |
| this.statusCode = 0 |
| } |
|
|
| setHeader(name, value) { |
| this.headers[name] = value |
| } |
|
|
| _write(chunk, _encoding, callback) { |
| this.body.push(Buffer.from(chunk)) |
| callback() |
| } |
| } |
|
|
| test('reads complete JSONL rows while a writer is appending a partial row', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-jsonl-')) |
| const file = path.join(root, 'events.json') |
| fs.writeFileSync(file, '{"actionIndex":1}\n{"actionIndex":2') |
| assert.deepEqual(_test.readJsonl(file), [{ actionIndex: 1 }]) |
| }) |
|
|
| test('rejects run and artifact path traversal', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-safe-')) |
| const run = path.join(root, 'run-001') |
| fs.mkdirSync(run) |
| assert.equal(_test.safeRunDirectory(root, '../outside'), null) |
| assert.equal(_test.safeRunDirectory(root, 'run-001'), run) |
| assert.equal(_test.safeRunDirectory(root, 'batch/rollouts/run-001'), path.join(root, 'batch', 'rollouts', 'run-001')) |
| assert.equal(_test.safeRunDirectory(root, 'batch//run-001'), null) |
| assert.equal(_test.safeArtifact(run, '../../secret'), null) |
| assert.equal(_test.safeArtifact(run, 'frames/frame_0000.png'), path.join(run, 'frames', 'frame_0000.png')) |
| }) |
|
|
| test('recursively indexes batch rollout directories', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-batch-')) |
| const rollout = path.join(root, 'batch-001', 'rollouts', 'model-a') |
| fs.mkdirSync(rollout, { recursive: true }) |
| fs.writeFileSync(path.join(rollout, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'model-a', |
| provider: 'openai', |
| state: 'running', |
| updated_at: '2026-07-16T00:00:00Z', |
| })) |
|
|
| const runs = _test.listRuns(root) |
|
|
| assert.equal(runs.length, 1) |
| assert.equal(runs[0].id, 'batch-001/rollouts/model-a') |
| assert.equal(runs[0].status, 'running') |
| }) |
|
|
| test('indexes an unsuccessful wall-time rollout as budget exhausted', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-run-')) |
| const run = path.join(root, 'gpt-run') |
| fs.mkdirSync(run) |
| fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'gpt-5.6-sol', |
| provider: 'openai', |
| reasoning_effort: 'xhigh', |
| completed: false, |
| stop_reason: 'wall_timeout', |
| accepted_actions: 40, |
| invalid_actions: 0, |
| screenshots: 41, |
| artifact_bytes: 0, |
| })) |
| const summary = _test.runSummary(root, 'gpt-run') |
| assert.equal(summary.status, 'budget_exhausted') |
| assert.equal(summary.stopReason, 'wall_timeout') |
| assert.equal(summary.acceptedActions, 40) |
| assert.equal(summary.evaluationState, 'no_artifact') |
| }) |
|
|
| test('prefers the versioned vision judgment over the legacy deterministic score', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-vision-score-')) |
| const run = path.join(root, 'vision-run') |
| fs.mkdirSync(path.join(run, 'evaluation', 'vision-judge'), { recursive: true }) |
| fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'gpt-5.6-sol', |
| provider: 'openai', |
| state: 'completed', |
| })) |
| fs.writeFileSync(path.join(run, 'evaluation', 'score.json'), JSON.stringify({ |
| score: 0.52, |
| valid: true, |
| scoring_method: 'vector-semantic-v2', |
| })) |
| fs.writeFileSync( |
| path.join(run, 'evaluation', 'vision-judge', 'score.json'), |
| JSON.stringify({ |
| prompt_version: 'autocad-vision-judge-v1', |
| overall_score: 0.93, |
| target_acquired: true, |
| confidence: 0.91, |
| needs_human_review: false, |
| component_scores: { |
| geometry: 0.98, |
| dimensions: 0.96, |
| annotations: 0.75, |
| presentation: 0.70, |
| }, |
| }), |
| ) |
|
|
| const summary = _test.runSummary(root, 'vision-run') |
| const detail = _test.traceDetail(root, 'vision-run') |
|
|
| assert.equal(summary.score, 0.93) |
| assert.equal(summary.scoreMethod, 'vision:autocad-vision-judge-v1') |
| assert.equal(summary.scoreSource, 'vision_judge') |
| assert.equal(summary.visionScore.score, 0.93) |
| assert.equal(summary.visionScore.geometry, 0.98) |
| assert.equal(summary.deterministicScore.score, 0.52) |
| assert.equal(summary.deterministicScore.method, 'vector-semantic-v2') |
| assert.equal(summary.targetAcquired, true) |
| assert.equal(summary.evaluationState, 'scored') |
| assert.equal(detail.score.geometry_similarity, 0.98) |
| assert.equal(detail.score.score_source, 'vision_judge') |
| assert.equal(detail.scores.vision.overall_score, 0.93) |
| assert.equal(detail.scores.deterministic.score, 0.52) |
| }) |
|
|
| test('shows automatic evaluation progress immediately after model completion', () => { |
| const heartbeat = new Date().toISOString() |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-evaluating-')) |
| const run = path.join(root, 'evaluating-run') |
| fs.mkdirSync(path.join(run, 'evaluation'), { recursive: true }) |
| fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'gpt-test', |
| provider: 'openai', |
| completed: true, |
| artifact_bytes: 1234, |
| })) |
| fs.writeFileSync(path.join(run, 'evaluation', 'status.json'), JSON.stringify({ |
| stage: 'vision_judging', |
| updated_at: heartbeat, |
| })) |
|
|
| const summary = _test.runSummary(root, 'evaluating-run') |
|
|
| assert.equal(summary.evaluationState, 'evaluating') |
| assert.equal(summary.evaluationStage, 'vision_judging') |
| assert.equal(summary.updatedAt, heartbeat) |
| }) |
|
|
| test('reports independent vision and deterministic benchmark means', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-dual-score-')) |
| const benchmark = path.join(root, 'benchmarks', 'dual-score') |
| const run = path.join(benchmark, 'rollouts', 'task-001') |
| fs.mkdirSync(path.join(run, 'evaluation', 'vision-judge'), { recursive: true }) |
| fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({ |
| state: 'completed', |
| rollouts: { |
| 'task-001': { |
| task_id: 'task-001', model_id: 'model', provider: 'openai', state: 'completed', |
| }, |
| }, |
| })) |
| fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({ |
| task_id: 'task-001', model_id: 'model', provider: 'openai', |
| completed: true, artifact_bytes: 100, |
| })) |
| fs.writeFileSync(path.join(run, 'evaluation', 'score.json'), JSON.stringify({ |
| score: 0.4, |
| valid: true, |
| scoring_method: 'vector-semantic-v2', |
| geometry_similarity: 0.5, |
| })) |
| fs.writeFileSync( |
| path.join(run, 'evaluation', 'vision-judge', 'score.json'), |
| JSON.stringify({ |
| overall_score: 0.8, |
| verdict: 'mostly_acquired', |
| component_scores: { geometry: 0.9, dimensions: 0.7, annotations: 0.6, presentation: 0.5 }, |
| }), |
| ) |
|
|
| const detail = _test.benchmarkDetail(root, 'benchmarks/dual-score') |
|
|
| assert.equal(detail.aggregate.meanScore, 0.8) |
| assert.equal(detail.aggregate.visionMeanScore, 0.8) |
| assert.equal(detail.aggregate.visionScoredTasks, 1) |
| assert.equal(detail.aggregate.deterministicMeanScore, 0.4) |
| assert.equal(detail.aggregate.deterministicScoredTasks, 1) |
| assert.equal(detail.tasks[0].visionScore.score, 0.8) |
| assert.equal(detail.tasks[0].deterministicScore.score, 0.4) |
| }) |
|
|
| test('does not show a stale evaluator heartbeat as live forever', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-stale-evaluation-')) |
| const run = path.join(root, 'stale-evaluation') |
| fs.mkdirSync(path.join(run, 'evaluation'), { recursive: true }) |
| fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({ |
| task_id: 'task-001', |
| completed: true, |
| stop_reason: 'done', |
| artifact_bytes: 100, |
| })) |
| fs.writeFileSync(path.join(run, 'evaluation', 'status.json'), JSON.stringify({ |
| stage: 'vision_judging', |
| updated_at: '2020-01-01T00:00:00Z', |
| })) |
|
|
| assert.equal(_test.runSummary(root, 'stale-evaluation').evaluationState, 'evaluation_interrupted') |
| }) |
|
|
| test('labels a batch process without a pilot trace as startup, not handoff', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-startup-')) |
| const benchmark = path.join(root, 'benchmark-starting') |
| fs.mkdirSync(benchmark, { recursive: true }) |
| fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({ |
| batch_id: 'batch-starting', |
| state: 'running', |
| rollouts: { |
| 'task-002': { |
| task_id: 'task-002', |
| model_id: 'model-test', |
| provider: 'openrouter', |
| state: 'running', |
| }, |
| }, |
| })) |
|
|
| const detail = _test.benchmarkDetail(root, 'benchmark-starting') |
|
|
| assert.equal(detail.tasks[0].status, 'starting') |
| assert.equal(detail.tasks[0].evaluationState, 'running') |
| }) |
|
|
| test('retains batch grouping receipts and intermediate audit frames', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-batch-turn-')) |
| const run = path.join(root, 'batched-run') |
| const frames = path.join(run, 'frames') |
| fs.mkdirSync(frames, { recursive: true }) |
| fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'gpt-test', |
| provider: 'openai', |
| state: 'running', |
| })) |
| fs.writeFileSync(path.join(run, 'model_turns.jsonl'), `${JSON.stringify({ |
| turn: 1, |
| tool_use_id: 'call-batch', |
| action: { |
| actions: [ |
| { action: 'type', text: 'LINE', intent: 'Type LINE.' }, |
| { action: 'key', keys: ['ENTER'], intent: 'Submit LINE.' }, |
| ], |
| }, |
| })}\n`) |
| fs.writeFileSync(path.join(run, 'input_receipts.jsonl'), [ |
| { action_index: 1, tool_use_id: 'call-batch', batch_index: 1, batch_size: 2 }, |
| { action_index: 2, tool_use_id: 'call-batch', batch_index: 2, batch_size: 2 }, |
| ].map(JSON.stringify).join('\n') + '\n') |
| for (const name of ['frame_0000.png', 'frame_0001.png', 'frame_0002.png']) { |
| fs.writeFileSync(path.join(frames, name), 'png') |
| } |
|
|
| const detail = _test.traceDetail(root, 'batched-run') |
|
|
| assert.equal(detail.turns[0].action.actions.length, 2) |
| assert.deepEqual(detail.receipts.map((receipt) => receipt.batch_index), [1, 2]) |
| assert.deepEqual(detail.frames.map((frame) => frame.name), [ |
| 'frame_0000.png', |
| 'frame_0001.png', |
| 'frame_0002.png', |
| ]) |
| }) |
|
|
| test('indexes cloud-backed frames referenced by compact resumed traces', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-frames-')) |
| const run = path.join(root, 'cloud-frame-run') |
| const frames = path.join(run, 'frames') |
| fs.mkdirSync(frames, { recursive: true }) |
| fs.writeFileSync(path.join(frames, 'frame_0000.png'), 'png') |
| fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'gpt-test', |
| provider: 'openai', |
| state: 'completed', |
| accepted_actions: 12, |
| })) |
| fs.writeFileSync(path.join(run, 'model_turn_summaries.jsonl'), `${JSON.stringify({ |
| turn: 2, |
| model_input: { |
| artifacts: [{ artifact_path: 'frames/frame_0010.png', sequence: 10 }], |
| }, |
| })}\n`) |
|
|
| const detail = _test.traceDetail(root, 'cloud-frame-run') |
|
|
| assert.deepEqual(detail.frames.map((frame) => frame.name), [ |
| 'frame_0000.png', |
| 'frame_0010.png', |
| 'frame_0012.png', |
| ]) |
| }) |
|
|
| test('advertises and lazily hydrates cloud reference and trusted render images', async () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-images-')) |
| const runId = 'benchmarks/sol/rollouts/task-022' |
| const run = path.join(root, runId) |
| fs.mkdirSync(path.join(run, 'evaluation', 'vision-judge'), { recursive: true }) |
| fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({ |
| task_id: 'cloud-only-task', |
| model_id: 'gpt-test', |
| provider: 'openai', |
| state: 'completed', |
| })) |
| fs.writeFileSync( |
| path.join(run, 'evaluation', 'vision-judge', 'score.json'), |
| JSON.stringify({ |
| prompt_version: 'autocad-vision-judge-v3', |
| overall_score: 0.9, |
| component_scores: { geometry: 0.9 }, |
| }), |
| ) |
|
|
| const detail = _test.traceDetail(root, runId) |
| assert.equal( |
| detail.referenceUrl, |
| `/api/runs/${encodeURIComponent(runId)}/file/reference.png`, |
| ) |
| assert.equal( |
| detail.artifacts.candidateRender, |
| `/api/runs/${encodeURIComponent(runId)}/file/evaluation/render.png`, |
| ) |
|
|
| const hydrated = [] |
| const api = createTraceApi({ |
| runsRoot: root, |
| cloudArtifactLoader: async (request) => { |
| hydrated.push(request.relativePath) |
| fs.mkdirSync(path.dirname(request.artifact), { recursive: true }) |
| fs.writeFileSync(request.artifact, `cloud:${request.relativePath}`) |
| }, |
| }) |
| for (const relativePath of ['reference.png', 'evaluation/render.png']) { |
| const response = new TestResponse() |
| const finished = once(response, 'finish') |
| await api({ |
| method: 'GET', |
| url: `/api/runs/${encodeURIComponent(runId)}/file/${relativePath}`, |
| }, response) |
| await finished |
| assert.equal(response.statusCode, 200) |
| assert.equal(response.headers['Content-Type'], 'image/png') |
| assert.equal(response.headers['Cache-Control'], 'private, max-age=31536000, immutable') |
| assert.equal(Buffer.concat(response.body).toString(), `cloud:${relativePath}`) |
| } |
| assert.deepEqual(hydrated, ['reference.png', 'evaluation/render.png']) |
| }) |
|
|
| test('hydrates a missing frame from the cloud artifact loader on demand', async () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-fetch-')) |
| const run = path.join(root, 'batch', 'rollouts', 'task-001') |
| fs.mkdirSync(run, { recursive: true }) |
| let hydrated = null |
| const api = createTraceApi({ |
| runsRoot: root, |
| cloudArtifactLoader: async (request) => { |
| hydrated = request |
| fs.mkdirSync(path.dirname(request.artifact), { recursive: true }) |
| fs.writeFileSync(request.artifact, 'cloud-png') |
| }, |
| }) |
| const response = new TestResponse() |
| const finished = once(response, 'finish') |
|
|
| const handled = await api({ |
| method: 'GET', |
| url: '/api/runs/batch%2Frollouts%2Ftask-001/file/frames/frame_0042.png', |
| }, response) |
| await finished |
|
|
| assert.equal(handled, true) |
| assert.equal(response.statusCode, 200) |
| assert.equal(response.headers['Content-Type'], 'image/png') |
| assert.equal(Buffer.concat(response.body).toString(), 'cloud-png') |
| assert.equal(hydrated.runId, 'batch/rollouts/task-001') |
| assert.equal(hydrated.relativePath, 'frames/frame_0042.png') |
| }) |
|
|
| test('redirects a cold cloud image directly to a short-lived private URL', async () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-redirect-')) |
| const runId = 'batch/rollouts/task-001' |
| fs.mkdirSync(path.join(root, runId), { recursive: true }) |
| let signed = null |
| const api = createTraceApi({ |
| runsRoot: root, |
| cloudArtifactUrl: async (request) => { |
| signed = request |
| return 'https://private-origin.example/signed-frame' |
| }, |
| cloudArtifactLoader: async () => { |
| assert.fail('the proxy loader must not run after a successful signed redirect') |
| }, |
| }) |
| const response = new TestResponse() |
| const finished = once(response, 'finish') |
|
|
| await api({ |
| method: 'GET', |
| url: `/api/runs/${encodeURIComponent(runId)}/file/frames/frame_0042.png`, |
| }, response) |
| await finished |
|
|
| assert.equal(response.statusCode, 307) |
| assert.equal(response.headers.Location, 'https://private-origin.example/signed-frame') |
| assert.equal(response.headers['Cache-Control'], 'private, max-age=240') |
| assert.deepEqual(signed, { |
| runId, |
| relativePath: 'frames/frame_0042.png', |
| }) |
| }) |
|
|
| test('bootstraps a missing live rollout trace from cloud metadata', async () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-cloud-index-')) |
| const runId = 'benchmarks/live-suite/rollouts/task-004' |
| const run = path.join(root, runId) |
| let hydration = null |
| const api = createTraceApi({ |
| runsRoot: root, |
| cloudTraceLoader: async (request) => { |
| hydration = request |
| fs.mkdirSync(request.runDirectory, { recursive: true }) |
| fs.writeFileSync(path.join(request.runDirectory, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-004', |
| model_id: 'claude-fable-5', |
| provider: 'anthropic', |
| state: 'running', |
| accepted_actions: 42, |
| })) |
| fs.writeFileSync(path.join(request.runDirectory, 'model_turn_summaries.jsonl'), `${JSON.stringify({ |
| turn: 1, |
| tool_use_id: 'tool-1', |
| action: { actions: [{ action: 'click', x: 1, y: 2 }] }, |
| model_input: { |
| artifacts: [{ artifact_path: 'frames/frame_0041.png' }], |
| }, |
| })}\n`) |
| }, |
| }) |
| const response = new TestResponse() |
| const finished = once(response, 'finish') |
|
|
| await api({ |
| method: 'GET', |
| url: `/api/runs/${encodeURIComponent(runId)}`, |
| }, response) |
| await finished |
|
|
| assert.equal(response.statusCode, 200) |
| assert.equal(hydration.runId, runId) |
| assert.equal(hydration.runDirectory, run) |
| const detail = JSON.parse(Buffer.concat(response.body).toString()) |
| assert.equal(detail.summary.acceptedActions, 42) |
| assert.equal(detail.turns.length, 1) |
| assert.deepEqual(detail.frames.map((frame) => frame.name), [ |
| 'frame_0041.png', |
| 'frame_0042.png', |
| ]) |
| }) |
|
|
| test('refreshes stale live benchmark metadata before returning its detail', async () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-benchmark-cloud-refresh-')) |
| const benchmarkId = 'benchmarks/live-suite' |
| const benchmark = path.join(root, benchmarkId) |
| const run = path.join(benchmark, 'rollouts', 'task-004') |
| fs.mkdirSync(run, { recursive: true }) |
| fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({ |
| state: 'running', |
| updated_at: '2026-07-18T00:00:00Z', |
| rollouts: { |
| 'task-004': { |
| task_id: 'task-004', |
| model_id: 'claude-fable-5', |
| provider: 'anthropic', |
| state: 'running', |
| }, |
| }, |
| })) |
| fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-004', |
| model_id: 'claude-fable-5', |
| provider: 'anthropic', |
| state: 'running', |
| accepted_actions: 1, |
| })) |
| let refreshCount = 0 |
| const api = createTraceApi({ |
| runsRoot: root, |
| cloudBenchmarkLoader: async ({ benchmarkDirectory }) => { |
| refreshCount += 1 |
| fs.writeFileSync(path.join( |
| benchmarkDirectory, 'rollouts', 'task-004', 'run-state.json', |
| ), JSON.stringify({ |
| task_id: 'task-004', |
| model_id: 'claude-fable-5', |
| provider: 'anthropic', |
| state: 'running', |
| accepted_actions: 87, |
| turn_count: 21, |
| })) |
| }, |
| }) |
| const response = new TestResponse() |
| const finished = once(response, 'finish') |
|
|
| await api({ |
| method: 'GET', |
| url: `/api/benchmarks/${encodeURIComponent(benchmarkId)}`, |
| }, response) |
| await finished |
|
|
| assert.equal(response.statusCode, 200) |
| assert.equal(refreshCount, 1) |
| const detail = JSON.parse(Buffer.concat(response.body).toString()) |
| assert.equal(detail.tasks[0].acceptedActions, 87) |
| assert.equal(detail.aggregate.acceptedActions, 87) |
| assert.equal(detail.aggregate.usage.modelTurns, 21) |
| }) |
|
|
| test('content-addresses model frames and replaces a stale local frame', async () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-stale-frame-')) |
| const runId = 'batch/rollouts/task-001' |
| const run = path.join(root, runId) |
| const frames = path.join(run, 'frames') |
| fs.mkdirSync(frames, { recursive: true }) |
| const freshFrame = Buffer.from('fresh-cloud-frame') |
| const expectedSha256 = createHash('sha256').update(freshFrame).digest('hex') |
| fs.writeFileSync(path.join(frames, 'frame_0042.png'), 'stale-local-frame') |
| fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'gpt-test', |
| provider: 'openai', |
| state: 'completed', |
| })) |
| fs.writeFileSync(path.join(run, 'model_turn_summaries.jsonl'), `${JSON.stringify({ |
| turn: 1, |
| model_input: { |
| artifacts: [{ |
| artifact_path: 'frames/frame_0042.png', |
| sha256: expectedSha256, |
| sequence: 42, |
| }], |
| }, |
| })}\n`) |
|
|
| const detail = _test.traceDetail(root, runId) |
| assert.equal(detail.frames[0].sha256, expectedSha256) |
| assert.equal( |
| detail.frames[0].url, |
| `/api/runs/batch%2Frollouts%2Ftask-001/file/frames/frame_0042.png?sha256=${expectedSha256}`, |
| ) |
|
|
| let hydration = null |
| const api = createTraceApi({ |
| runsRoot: root, |
| cloudArtifactLoader: async (request) => { |
| hydration = request |
| fs.writeFileSync(request.artifact, freshFrame) |
| }, |
| }) |
| const response = new TestResponse() |
| const finished = once(response, 'finish') |
| await api({ |
| method: 'GET', |
| url: `${detail.frames[0].url}`, |
| }, response) |
| await finished |
|
|
| assert.equal(response.statusCode, 200) |
| assert.equal(Buffer.concat(response.body).toString(), freshFrame.toString()) |
| assert.equal(response.headers['Cache-Control'], 'private, max-age=31536000, immutable') |
| assert.equal(hydration.force, true) |
| assert.equal(hydration.expectedSha256, expectedSha256) |
| }) |
|
|
| test('never serves a stale frame when cloud refresh does not match its checksum', async () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-reject-stale-frame-')) |
| const runId = 'batch/rollouts/task-001' |
| const run = path.join(root, runId) |
| const frames = path.join(run, 'frames') |
| fs.mkdirSync(frames, { recursive: true }) |
| const expectedSha256 = createHash('sha256').update('expected-frame').digest('hex') |
| fs.writeFileSync(path.join(frames, 'frame_0001.png'), 'stale-frame') |
|
|
| const api = createTraceApi({ |
| runsRoot: root, |
| cloudArtifactLoader: async () => {}, |
| }) |
| const response = new TestResponse() |
| const finished = once(response, 'finish') |
| await api({ |
| method: 'GET', |
| url: `/api/runs/${encodeURIComponent(runId)}/file/frames/frame_0001.png?sha256=${expectedSha256}`, |
| }, response) |
| await finished |
|
|
| assert.equal(response.statusCode, 404) |
| assert.match(Buffer.concat(response.body).toString(), /artifact_not_found/) |
| }) |
|
|
| test('uses compact turn summaries without returning repeated request history', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-compact-')) |
| const run = path.join(root, 'compact-run') |
| fs.mkdirSync(run) |
| fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'claude-test', |
| provider: 'anthropic', |
| state: 'completed', |
| turn_count: 1, |
| })) |
| fs.writeFileSync(path.join(run, 'model_turns.jsonl'), 'x'.repeat(9 * 1024 * 1024)) |
| fs.writeFileSync(path.join(run, 'model_turn_summaries.jsonl'), `${JSON.stringify({ |
| turn: 1, |
| tool_use_id: 'call-compact', |
| action: { action: 'done' }, |
| model_input: { |
| model: 'claude-test', |
| artifacts: [{ artifact_path: 'frames/frame_0000.png' }], |
| }, |
| })}\n`) |
|
|
| const detail = _test.traceDetail(root, 'compact-run') |
|
|
| assert.equal(detail.traceIndexed, true) |
| assert.equal(detail.turns.length, 1) |
| assert.equal(detail.turns[0].tool_use_id, 'call-compact') |
| assert.equal(JSON.stringify(detail).includes('repeated request history'), false) |
| assert.equal('events' in detail, false) |
| assert.equal('frameEvents' in detail, false) |
| }) |
|
|
| test('does not synchronously parse a large unindexed trace for benchmark usage', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-large-')) |
| const run = path.join(root, 'large-run') |
| fs.mkdirSync(run) |
| const trace = path.join(run, 'model_turns.jsonl') |
| fs.writeFileSync(trace, '') |
| fs.truncateSync(trace, 9 * 1024 * 1024) |
|
|
| const usage = _test.runUsage(run, { input_per_million: 5 }, { |
| turn_count: 42, |
| usage: { input_tokens: 1200, output_tokens: 30, total_tokens: 1230 }, |
| }) |
|
|
| assert.equal(usage.modelTurns, 42) |
| assert.equal(usage.inputTokens, 1200) |
| assert.equal(usage.totalTokens, 1230) |
| assert.equal(usage.apiCost, null) |
| }) |
|
|
| test('uses the live aggregate instead of parsing a large compact usage index', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-summary-large-')) |
| const run = path.join(root, 'large-summary-run') |
| fs.mkdirSync(run) |
| const summary = path.join(run, 'model_turn_summaries.jsonl') |
| fs.writeFileSync(summary, '') |
| fs.truncateSync(summary, 600 * 1024) |
|
|
| const usage = _test.runUsage(run, { input_per_million: 5 }, { |
| turn_count: 77, |
| usage: { input_tokens: 9000, output_tokens: 80, total_tokens: 9080 }, |
| }) |
|
|
| assert.equal(usage.modelTurns, 77) |
| assert.equal(usage.inputTokens, 9000) |
| assert.equal(usage.totalTokens, 9080) |
| assert.equal(usage.apiCost, null) |
| }) |
|
|
| test('reconciles a stale running batch row with a completed rollout trace', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-trace-stale-batch-')) |
| const benchmark = path.join(root, 'stale-batch') |
| const run = path.join(benchmark, 'rollouts', 'task-001') |
| fs.mkdirSync(run, { recursive: true }) |
| fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({ |
| state: 'running', |
| updated_at: '2020-01-01T00:00:00Z', |
| rollouts: { |
| 'task-001': { |
| task_id: 'task-001', |
| model_id: 'claude-test', |
| provider: 'anthropic', |
| state: 'running', |
| }, |
| 'task-002': { |
| task_id: 'task-002', |
| model_id: 'claude-test', |
| provider: 'anthropic', |
| state: 'queued', |
| }, |
| }, |
| })) |
| fs.writeFileSync(path.join(run, 'run-state.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'claude-test', |
| provider: 'anthropic', |
| state: 'completed', |
| turn_count: 1, |
| })) |
|
|
| const detail = _test.benchmarkDetail(root, 'stale-batch') |
|
|
| assert.equal(detail.tasks[0].status, 'completed') |
| assert.equal(detail.aggregate.completedTasks, 1) |
| assert.equal(detail.aggregate.activeTasks, 0) |
| assert.equal(detail.status, 'interrupted') |
| }) |
|
|
| test('calculates cached and long-context token costs per model turn', () => { |
| const pricing = { |
| input_per_million: 5, |
| cached_input_per_million: 0.5, |
| output_per_million: 30, |
| long_context_threshold_tokens: 272000, |
| long_context_input_multiplier: 2, |
| long_context_output_multiplier: 1.5, |
| } |
| const usage = _test.usageSummary([ |
| { usage: { |
| input_tokens: 1000, |
| cached_input_tokens: 400, |
| output_tokens: 100, |
| reasoning_output_tokens: 60, |
| total_tokens: 1100, |
| } }, |
| { usage: { input_tokens: 300000, output_tokens: 1000, total_tokens: 301000 } }, |
| ], pricing) |
|
|
| assert.equal(usage.inputTokens, 301000) |
| assert.equal(usage.cachedInputTokens, 400) |
| assert.equal(usage.outputTokens, 1100) |
| assert.equal(usage.reasoningOutputTokens, 60) |
| assert.ok(Math.abs(usage.apiCost - 3.0512) < 1e-12) |
| }) |
|
|
| test('aggregates a live multi-task benchmark from batch state and rollout traces', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-benchmark-')) |
| const benchmark = path.join(root, 'benchmarks', 'full-suite') |
| const completed = path.join(benchmark, 'rollouts', 'task-001') |
| fs.mkdirSync(completed, { recursive: true }) |
| fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({ |
| schema_version: 2, |
| batch_id: 'batch-full', |
| benchmark_name: 'Full suite', |
| state: 'running', |
| started_at: '2026-07-16T00:00:00Z', |
| updated_at: '2026-07-16T00:10:00Z', |
| max_concurrency: 10, |
| pricing: { |
| input_per_million: 5, |
| cached_input_per_million: 0.5, |
| output_per_million: 30, |
| }, |
| rollouts: { |
| 'task-001': { |
| task_id: 'task-001', |
| model_id: 'gpt-5.6-sol', |
| provider: 'openai', |
| state: 'completed', |
| }, |
| 'task-022': { |
| task_id: 'task-022', |
| model_id: 'gpt-5.6-sol', |
| provider: 'openai', |
| state: 'queued', |
| }, |
| }, |
| })) |
| fs.writeFileSync(path.join(completed, 'result.json'), JSON.stringify({ |
| task_id: 'task-001', |
| model_id: 'gpt-5.6-sol', |
| provider: 'openai', |
| completed: true, |
| accepted_actions: 20, |
| artifact_bytes: 1234, |
| })) |
| fs.writeFileSync(path.join(completed, 'model_turns.jsonl'), `${JSON.stringify({ |
| usage: { input_tokens: 1000, cached_input_tokens: 400, output_tokens: 100 }, |
| })}\n`) |
|
|
| const detail = _test.benchmarkDetail(root, 'benchmarks/full-suite') |
|
|
| assert.equal(detail.name, 'Full suite') |
| assert.equal(detail.aggregate.totalTasks, 2) |
| assert.equal(detail.aggregate.completedTasks, 1) |
| assert.equal(detail.aggregate.queuedTasks, 1) |
| assert.equal(detail.aggregate.artifactTasks, 1) |
| assert.equal(detail.aggregate.usage.inputTokens, 1000) |
| assert.ok(Math.abs(detail.aggregate.usage.apiCost - 0.0062) < 1e-12) |
| assert.deepEqual(detail.tasks.map((task) => task.level), ['basic', 'basic']) |
| assert.deepEqual(detail.tasks.map((task) => task.drawingType), ['2d', '3d']) |
| assert.deepEqual(detail.aggregate.drawingTypes, { |
| '2d': { total: 1, scored: 0, meanScore: null }, |
| '3d': { total: 1, scored: 0, meanScore: null }, |
| }) |
| assert.deepEqual(detail.aggregate.evaluationStates, { |
| awaiting_score: 1, |
| queued: 1, |
| }) |
| assert.equal(_test.listBenchmarks(root)[0].id, 'benchmarks/full-suite') |
| }) |
|
|
| test('keeps execution, recovery, budget, infrastructure, and evaluation states separate', () => { |
| const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autocad-outcomes-')) |
| const benchmark = path.join(root, 'benchmarks', 'outcomes') |
| const cases = { |
| completed: { completed: true, stop_reason: 'done', artifact_bytes: 10 }, |
| recovered: { completed: false, stop_reason: 'recovered_finalized_session', artifact_bytes: 10 }, |
| budget: { completed: false, stop_reason: 'max_actions', artifact_bytes: 10 }, |
| infra: { completed: false, stop_reason: 'rollout_error', failure: 'ConnectError', artifact_bytes: 10 }, |
| model: { completed: false, stop_reason: 'invalid_action_limit', failure: 'invalid', artifact_bytes: 10 }, |
| } |
| fs.mkdirSync(benchmark, { recursive: true }) |
| fs.writeFileSync(path.join(benchmark, 'batch-state.json'), JSON.stringify({ |
| state: 'running', |
| updated_at: '2020-01-01T00:00:00Z', |
| rollouts: Object.fromEntries(Object.keys(cases).map((task) => [task, { |
| task_id: 'task-001', model_id: 'model', provider: 'openai', state: 'failed', |
| failure: 'stale wrapper failure', |
| }])), |
| })) |
| for (const [name, result] of Object.entries(cases)) { |
| const run = path.join(benchmark, 'rollouts', name) |
| fs.mkdirSync(path.join(run, 'evaluation'), { recursive: true }) |
| fs.writeFileSync(path.join(run, 'result.json'), JSON.stringify({ |
| task_id: 'task-001', model_id: 'model', provider: 'openai', ...result, |
| })) |
| } |
| fs.writeFileSync( |
| path.join(benchmark, 'rollouts', 'completed', 'evaluation', 'status.json'), |
| JSON.stringify({ stage: 'failed', error: 'VisionJudgeError: ReadTimeout' }), |
| ) |
|
|
| const detail = _test.benchmarkDetail(root, 'benchmarks/outcomes') |
|
|
| assert.equal(detail.status, 'completed_with_issues') |
| assert.equal(detail.aggregate.settledTasks, 5) |
| assert.equal(detail.aggregate.completedTasks, 1) |
| assert.equal(detail.aggregate.recoveredTasks, 1) |
| assert.equal(detail.aggregate.budgetExhaustedTasks, 1) |
| assert.equal(detail.aggregate.infrastructureFailedTasks, 1) |
| assert.equal(detail.aggregate.failedTasks, 1) |
| assert.equal(detail.tasks.find((task) => task.rolloutId === 'completed').status, 'completed') |
| assert.equal(detail.tasks.find((task) => task.rolloutId === 'completed').failure, null) |
| assert.equal(detail.tasks.find((task) => task.rolloutId === 'budget').failure, null) |
| assert.equal(detail.tasks.find((task) => task.rolloutId === 'infra').failure, 'ConnectError') |
| assert.equal(detail.tasks.find((task) => task.rolloutId === 'completed').evaluationState, 'evaluation_failed') |
| }) |
|
|
