/** * eval.ts — headless agent benchmark. * * Runs the same agent loop as server.ts but without any HTTP server, on a * deterministic seeded world, using a configurable variant (viewport size, * thinking on/off, temperature, cache_prompt). Emits one JSON metrics object * to stdout at the end. * * Usage: * ENDPOINT=https://… bun eval.ts \ * --variant baseline --seed 1 --turns 30 [--out path.json] * * Variants are defined in VARIANTS below. */ import { SinglePlayerGame, DEFAULTS } from './engine.js'; import { PICK_FIRST_TOOLS, PICK_FIRST_SYSTEM_PROMPT, buildPickFirstPrompt, AgentMemory, } from './agent-brain.js'; // ─── Config ────────────────────────────────────────────────────────────────── interface Variant { viewportRadius: number; thinking: boolean; temperature: number; cache: boolean; systemPrompt: string; maxTokens: number; } const VARIANTS: Record = { baseline: { viewportRadius: 4, thinking: false, temperature: 0.6, cache: false, systemPrompt: PICK_FIRST_SYSTEM_PROMPT, maxTokens: 100, }, thinking_on: { viewportRadius: 4, thinking: true, temperature: 0.6, cache: false, systemPrompt: PICK_FIRST_SYSTEM_PROMPT, maxTokens: 1024, }, viewport_5: { viewportRadius: 2, thinking: false, temperature: 0.6, cache: false, systemPrompt: PICK_FIRST_SYSTEM_PROMPT, maxTokens: 100, }, viewport_13: { viewportRadius: 6, thinking: false, temperature: 0.6, cache: false, systemPrompt: PICK_FIRST_SYSTEM_PROMPT, maxTokens: 100, }, temp_0: { viewportRadius: 4, thinking: false, temperature: 0.0, cache: false, systemPrompt: PICK_FIRST_SYSTEM_PROMPT, maxTokens: 100, }, temp_high: { viewportRadius: 4, thinking: false, temperature: 0.9, cache: false, systemPrompt: PICK_FIRST_SYSTEM_PROMPT, maxTokens: 100, }, cache_on: { viewportRadius: 4, thinking: false, temperature: 0.6, cache: true, systemPrompt: PICK_FIRST_SYSTEM_PROMPT, maxTokens: 100, }, }; // ─── CLI args ──────────────────────────────────────────────────────────────── function getArg(name: string, fallback?: string): string { const idx = process.argv.indexOf(`--${name}`); if (idx >= 0 && process.argv[idx + 1]) return process.argv[idx + 1]; if (fallback != null) return fallback; throw new Error(`Missing --${name}`); } const variantName = getArg('variant', 'baseline'); const seed = parseInt(getArg('seed', '1'), 10); const turns = parseInt(getArg('turns', '30'), 10); const outPath = (() => { const i = process.argv.indexOf('--out'); return i >= 0 ? process.argv[i + 1] : null; })(); const variant = VARIANTS[variantName]; if (!variant) { console.error(`Unknown variant: ${variantName}. Available: ${Object.keys(VARIANTS).join(', ')}`); process.exit(1); } const ENDPOINT = (process.env.ENDPOINT ?? 'https://zopt7f90d1ydgy-8080.proxy.runpod.net').replace(/\/$/, ''); const REMOTE_MODEL = process.env.MODEL ?? 'default'; // ─── Inference (single attempt + 3-retry on Context size) ──────────────────── async function callModel(messages: any[]): Promise<{ message: any; usage: any; durationMs: number; cacheUsed: boolean }> { const t0 = performance.now(); const baseBody: any = { model: REMOTE_MODEL, messages, tools: PICK_FIRST_TOOLS, tool_choice: 'auto', temperature: variant.temperature, max_tokens: variant.maxTokens, chat_template_kwargs: { enable_thinking: variant.thinking }, }; let useCache = variant.cache; baseBody.cache_prompt = useCache; let res: Response; let attempt = 0; let body = { ...baseBody }; while (true) { res = await fetch(`${ENDPOINT}/v1/chat/completions`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), }); if (res.ok) break; const text = await res.text(); const isContextErr = res.status === 500 && /Context size/i.test(text); if (!isContextErr || attempt >= 3) { throw new Error(`HTTP ${res.status}: ${text}`); } attempt++; body = { ...baseBody, cache_prompt: false }; useCache = false; await new Promise((r) => setTimeout(r, 200)); } const data = await res.json(); const totalMs = Math.round(performance.now() - t0); return { message: data.choices?.[0]?.message ?? {}, usage: data.usage ?? {}, durationMs: totalMs, cacheUsed: useCache, }; } // ─── Run one eval ──────────────────────────────────────────────────────────── interface Metrics { variant: string; seed: number; turns: number; realActions: number; fallbacks: number; illegalMoves: number; diamonds: number; picks: number; pickPrecisionOffered: number; pickPrecisionTaken: number; uniqueTilesVisited: number; directions: Record; toolCalls: Record; latencies: number[]; inputTokens: number[]; cachedTokens: number[]; outputTokens: number[]; errorMessages: string[]; } function checkAdjacentDiamond(viewportStr: string): string | null { const rows = viewportStr.trim().split('\n').map((r: string) => [...r]); let cr = Math.floor(rows.length / 2); let cc = Math.floor((rows[0]?.length ?? 0) / 2); for (let r = 0; r < rows.length; r++) { for (let c = 0; c < rows[r].length; c++) { if (rows[r][c] === '🤖' || rows[r][c] === '🚶') { cr = r; cc = c; } } } const t = (dr: number, dc: number) => rows[cr + dr]?.[cc + dc]; if (t(-1, 0) === '💎') return 'up'; if (t(1, 0) === '💎') return 'down'; if (t(0, -1) === '💎') return 'left'; if (t(0, 1) === '💎') return 'right'; return null; } function median(arr: number[]): number { if (!arr.length) return 0; const sorted = [...arr].sort((a, b) => a - b); const mid = Math.floor(sorted.length / 2); return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; } function entropy(counts: Record): number { const total = Object.values(counts).reduce((a, b) => a + b, 0); if (!total) return 0; let h = 0; for (const v of Object.values(counts)) { if (v > 0) { const p = v / total; h -= p * Math.log2(p); } } return h; } async function run(): Promise { const game = new SinglePlayerGame({ seed }); const memory = new AgentMemory(variant.systemPrompt); const m: Metrics = { variant: variantName, seed, turns: 0, realActions: 0, fallbacks: 0, illegalMoves: 0, diamonds: 0, picks: 0, pickPrecisionOffered: 0, pickPrecisionTaken: 0, uniqueTilesVisited: 0, directions: { up: 0, down: 0, left: 0, right: 0 }, toolCalls: { move: 0, pick: 0, other: 0 }, latencies: [], inputTokens: [], cachedTokens: [], outputTokens: [], errorMessages: [], }; const visited = new Set(); for (let i = 1; i <= turns; i++) { const viewport = game.getViewport(); visited.add(`${game.agent.x},${game.agent.y}`); const adjDiamondDir = checkAdjacentDiamond(viewport); if (adjDiamondDir) m.pickPrecisionOffered++; const memoryDepthBefore = memory.messages.length; memory.addUserText(buildPickFirstPrompt(viewport, game.inventory, variant.viewportRadius)); let action: { action: string; detail: string; tool_call_id: string } | null = null; try { const { message, usage, durationMs } = await callModel(memory.getMessages()); memory.addAssistant(message); m.latencies.push(durationMs); m.inputTokens.push(usage.prompt_tokens ?? 0); m.cachedTokens.push(usage.prompt_tokens_details?.cached_tokens ?? 0); m.outputTokens.push(usage.completion_tokens ?? 0); const tc = message.tool_calls?.[0]; if (tc) { const name = tc.function?.name; m.toolCalls[name === 'move' || name === 'pick' ? name : 'other']++; let args: any = {}; try { args = JSON.parse(tc.function.arguments); } catch {} if ( (name === 'move' || name === 'pick') && ['up', 'down', 'left', 'right'].includes(args.direction) ) { action = { action: name, detail: args.direction, tool_call_id: tc.id }; } } } catch (err: any) { m.errorMessages.push(String(err?.message ?? err).slice(0, 200)); } if (!action) { while (memory.messages.length > memoryDepthBefore) memory.popLast(); m.fallbacks++; continue; } m.realActions++; m.directions[action.detail]++; if ( adjDiamondDir && action.action === 'pick' && action.detail === adjDiamondDir ) { m.pickPrecisionTaken++; } let toolResult = ''; let success = true; try { const r = game.executeAction(action) as { pickedUp?: string | null } | undefined; const picked = r?.pickedUp ? ` — picked up ${r.pickedUp}` : ''; toolResult = `${action.action} ${action.detail} succeeded${picked}`; } catch (err: any) { toolResult = `${action.action} ${action.detail} failed: ${err.message}`; success = false; m.illegalMoves++; } memory.addToolResult(action.tool_call_id, toolResult); } m.turns = turns; m.diamonds = game.stats.diamonds; m.picks = game.stats.picks; m.uniqueTilesVisited = visited.size; return m; } // ─── Compute summary stats and emit ────────────────────────────────────────── function summarize(m: Metrics) { const realActions = Math.max(1, m.realActions); const offered = Math.max(1, m.pickPrecisionOffered); return { ...m, summary: { diamondsPerN: m.diamonds / m.turns, realActionRate: m.realActions / m.turns, illegalMoveRate: m.illegalMoves / realActions, pickPrecision: m.pickPrecisionOffered > 0 ? m.pickPrecisionTaken / m.pickPrecisionOffered : null, directionEntropy: entropy(m.directions), medianLatencyMs: median(m.latencies), avgInputTokens: m.inputTokens.length ? m.inputTokens.reduce((a, b) => a + b, 0) / m.inputTokens.length : 0, cacheHitRate: m.inputTokens.reduce((a, b) => a + b, 0) > 0 ? m.cachedTokens.reduce((a, b) => a + b, 0) / m.inputTokens.reduce((a, b) => a + b, 0) : 0, }, }; } // ─── Main ──────────────────────────────────────────────────────────────────── const startedAt = new Date().toISOString(); console.error(`[eval] variant=${variantName} seed=${seed} turns=${turns} endpoint=${ENDPOINT}`); const metrics = await run(); const summary = summarize(metrics); const finalOutput = { startedAt, endedAt: new Date().toISOString(), ...summary }; const json = JSON.stringify(finalOutput, null, 2); if (outPath) { await Bun.write(outPath, json); console.error(`[eval] wrote ${outPath}`); } console.log(json);