| import { writeFile } from 'node:fs/promises'; |
| import { BenchmarkArmSchema } from '@her2/shared'; |
| import { runBenchmark } from '../apps/api/src/bench.js'; |
|
|
| |
| const argv = process.argv.slice(2); |
| const flag = (name: string) => { const index = argv.indexOf(name); return index >= 0 ? argv[index + 1] : undefined; }; |
| const live = argv.includes('--live'); |
| if (live === argv.includes('--offline')) throw new Error('Pass exactly one of --offline or --live.'); |
| const maxCalls = Number(flag('--max-calls') ?? 220); |
| const arms = flag('--arms')?.split(',').map(arm => BenchmarkArmSchema.parse(arm.trim())); |
| const out = new URL(flag('--out') ?? '../evals/benchmark.json', flag('--out') ? `file://${process.cwd()}/` : import.meta.url); |
|
|
| let done = 0; |
| const apiKey = process.env.ANTHROPIC_API_KEY; |
| const model = process.env.CLAUDE_MODEL; |
| const artifact = await runBenchmark({ |
| mode: live ? 'live' : 'offline', maxCalls, ...(arms ? { arms } : {}), ...(apiKey ? { apiKey } : {}), ...(model ? { model } : {}), |
| onRow: row => { done++; if (live) console.error(`${done} ${row.arm} ${row.item_id} ${row.outcome}`); } |
| }); |
| await writeFile(out, JSON.stringify(artifact, null, 2) + '\n'); |
| const totals = Object.fromEntries(artifact.arms.map(arm => [arm, Object.values(artifact.summary[arm] ?? {}).reduce((sum, cell) => ({ n: sum.n + cell.n, correct: sum.correct + cell.correct, errors: sum.errors + cell.errors }), { n: 0, correct: 0, errors: 0 })])); |
| console.log(JSON.stringify({ mode: artifact.mode, items: artifact.items.length, calls_used: artifact.budget.calls_used, max_calls: artifact.budget.max_calls, retries: 0, totals })); |
|
|
