Spaces:
Sleeping
Sleeping
Download eval.ts from victor/mini-world: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/spaces/victor/mini-world/resolve/main/eval.ts
- Command line
-
hf download hf://spaces/victor/mini-world/eval.ts
-
curl -L -o eval.ts https://huggingface.co/spaces/victor/mini-world/resolve/main/eval.ts
11.5 kB
| /** | |
| * eval.ts β headless agent benchmark. | |
| * | |
| * Runs the same agent loop as server.ts but without any HTTP server, on a | |
| * deterministic seeded world, using a configurable variant (viewport size, | |
| * thinking on/off, temperature, cache_prompt). Emits one JSON metrics object | |
| * to stdout at the end. | |
| * | |
| * Usage: | |
| * ENDPOINT=https://β¦ bun eval.ts \ | |
| * --variant baseline --seed 1 --turns 30 [--out path.json] | |
| * | |
| * Variants are defined in VARIANTS below. | |
| */ | |
| import { SinglePlayerGame, DEFAULTS } from './engine.js'; | |
| import { | |
| PICK_FIRST_TOOLS, | |
| PICK_FIRST_SYSTEM_PROMPT, | |
| buildPickFirstPrompt, | |
| AgentMemory, | |
| } from './agent-brain.js'; | |
| // βββ Config ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| interface Variant { | |
| viewportRadius: number; | |
| thinking: boolean; | |
| temperature: number; | |
| cache: boolean; | |
| systemPrompt: string; | |
| maxTokens: number; | |
| } | |
| const VARIANTS: Record<string, Variant> = { | |
| baseline: { | |
| viewportRadius: 4, | |
| thinking: false, | |
| temperature: 0.6, | |
| cache: false, | |
| systemPrompt: PICK_FIRST_SYSTEM_PROMPT, | |
| maxTokens: 100, | |
| }, | |
| thinking_on: { | |
| viewportRadius: 4, | |
| thinking: true, | |
| temperature: 0.6, | |
| cache: false, | |
| systemPrompt: PICK_FIRST_SYSTEM_PROMPT, | |
| maxTokens: 1024, | |
| }, | |
| viewport_5: { | |
| viewportRadius: 2, | |
| thinking: false, | |
| temperature: 0.6, | |
| cache: false, | |
| systemPrompt: PICK_FIRST_SYSTEM_PROMPT, | |
| maxTokens: 100, | |
| }, | |
| viewport_13: { | |
| viewportRadius: 6, | |
| thinking: false, | |
| temperature: 0.6, | |
| cache: false, | |
| systemPrompt: PICK_FIRST_SYSTEM_PROMPT, | |
| maxTokens: 100, | |
| }, | |
| temp_0: { | |
| viewportRadius: 4, | |
| thinking: false, | |
| temperature: 0.0, | |
| cache: false, | |
| systemPrompt: PICK_FIRST_SYSTEM_PROMPT, | |
| maxTokens: 100, | |
| }, | |
| temp_high: { | |
| viewportRadius: 4, | |
| thinking: false, | |
| temperature: 0.9, | |
| cache: false, | |
| systemPrompt: PICK_FIRST_SYSTEM_PROMPT, | |
| maxTokens: 100, | |
| }, | |
| cache_on: { | |
| viewportRadius: 4, | |
| thinking: false, | |
| temperature: 0.6, | |
| cache: true, | |
| systemPrompt: PICK_FIRST_SYSTEM_PROMPT, | |
| maxTokens: 100, | |
| }, | |
| }; | |
| // βββ CLI args ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function getArg(name: string, fallback?: string): string { | |
| const idx = process.argv.indexOf(`--${name}`); | |
| if (idx >= 0 && process.argv[idx + 1]) return process.argv[idx + 1]; | |
| if (fallback != null) return fallback; | |
| throw new Error(`Missing --${name}`); | |
| } | |
| const variantName = getArg('variant', 'baseline'); | |
| const seed = parseInt(getArg('seed', '1'), 10); | |
| const turns = parseInt(getArg('turns', '30'), 10); | |
| const outPath = (() => { | |
| const i = process.argv.indexOf('--out'); | |
| return i >= 0 ? process.argv[i + 1] : null; | |
| })(); | |
| const variant = VARIANTS[variantName]; | |
| if (!variant) { | |
| console.error(`Unknown variant: ${variantName}. Available: ${Object.keys(VARIANTS).join(', ')}`); | |
| process.exit(1); | |
| } | |
| const ENDPOINT = (process.env.ENDPOINT ?? 'https://zopt7f90d1ydgy-8080.proxy.runpod.net').replace(/\/$/, ''); | |
| const REMOTE_MODEL = process.env.MODEL ?? 'default'; | |
| // βββ Inference (single attempt + 3-retry on Context size) ββββββββββββββββββββ | |
| async function callModel(messages: any[]): Promise<{ message: any; usage: any; durationMs: number; cacheUsed: boolean }> { | |
| const t0 = performance.now(); | |
| const baseBody: any = { | |
| model: REMOTE_MODEL, | |
| messages, | |
| tools: PICK_FIRST_TOOLS, | |
| tool_choice: 'auto', | |
| temperature: variant.temperature, | |
| max_tokens: variant.maxTokens, | |
| chat_template_kwargs: { enable_thinking: variant.thinking }, | |
| }; | |
| let useCache = variant.cache; | |
| baseBody.cache_prompt = useCache; | |
| let res: Response; | |
| let attempt = 0; | |
| let body = { ...baseBody }; | |
| while (true) { | |
| res = await fetch(`${ENDPOINT}/v1/chat/completions`, { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify(body), | |
| }); | |
| if (res.ok) break; | |
| const text = await res.text(); | |
| const isContextErr = res.status === 500 && /Context size/i.test(text); | |
| if (!isContextErr || attempt >= 3) { | |
| throw new Error(`HTTP ${res.status}: ${text}`); | |
| } | |
| attempt++; | |
| body = { ...baseBody, cache_prompt: false }; | |
| useCache = false; | |
| await new Promise((r) => setTimeout(r, 200)); | |
| } | |
| const data = await res.json(); | |
| const totalMs = Math.round(performance.now() - t0); | |
| return { | |
| message: data.choices?.[0]?.message ?? {}, | |
| usage: data.usage ?? {}, | |
| durationMs: totalMs, | |
| cacheUsed: useCache, | |
| }; | |
| } | |
| // βββ Run one eval ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| interface Metrics { | |
| variant: string; | |
| seed: number; | |
| turns: number; | |
| realActions: number; | |
| fallbacks: number; | |
| illegalMoves: number; | |
| diamonds: number; | |
| picks: number; | |
| pickPrecisionOffered: number; | |
| pickPrecisionTaken: number; | |
| uniqueTilesVisited: number; | |
| directions: Record<string, number>; | |
| toolCalls: Record<string, number>; | |
| latencies: number[]; | |
| inputTokens: number[]; | |
| cachedTokens: number[]; | |
| outputTokens: number[]; | |
| errorMessages: string[]; | |
| } | |
| function checkAdjacentDiamond(viewportStr: string): string | null { | |
| const rows = viewportStr.trim().split('\n').map((r: string) => [...r]); | |
| let cr = Math.floor(rows.length / 2); | |
| let cc = Math.floor((rows[0]?.length ?? 0) / 2); | |
| for (let r = 0; r < rows.length; r++) { | |
| for (let c = 0; c < rows[r].length; c++) { | |
| if (rows[r][c] === 'π€' || rows[r][c] === 'πΆ') { cr = r; cc = c; } | |
| } | |
| } | |
| const t = (dr: number, dc: number) => rows[cr + dr]?.[cc + dc]; | |
| if (t(-1, 0) === 'π') return 'up'; | |
| if (t(1, 0) === 'π') return 'down'; | |
| if (t(0, -1) === 'π') return 'left'; | |
| if (t(0, 1) === 'π') return 'right'; | |
| return null; | |
| } | |
| function median(arr: number[]): number { | |
| if (!arr.length) return 0; | |
| const sorted = [...arr].sort((a, b) => a - b); | |
| const mid = Math.floor(sorted.length / 2); | |
| return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; | |
| } | |
| function entropy(counts: Record<string, number>): number { | |
| const total = Object.values(counts).reduce((a, b) => a + b, 0); | |
| if (!total) return 0; | |
| let h = 0; | |
| for (const v of Object.values(counts)) { | |
| if (v > 0) { | |
| const p = v / total; | |
| h -= p * Math.log2(p); | |
| } | |
| } | |
| return h; | |
| } | |
| async function run(): Promise<Metrics> { | |
| const game = new SinglePlayerGame({ seed }); | |
| const memory = new AgentMemory(variant.systemPrompt); | |
| const m: Metrics = { | |
| variant: variantName, | |
| seed, | |
| turns: 0, | |
| realActions: 0, | |
| fallbacks: 0, | |
| illegalMoves: 0, | |
| diamonds: 0, | |
| picks: 0, | |
| pickPrecisionOffered: 0, | |
| pickPrecisionTaken: 0, | |
| uniqueTilesVisited: 0, | |
| directions: { up: 0, down: 0, left: 0, right: 0 }, | |
| toolCalls: { move: 0, pick: 0, other: 0 }, | |
| latencies: [], | |
| inputTokens: [], | |
| cachedTokens: [], | |
| outputTokens: [], | |
| errorMessages: [], | |
| }; | |
| const visited = new Set<string>(); | |
| for (let i = 1; i <= turns; i++) { | |
| const viewport = game.getViewport(); | |
| visited.add(`${game.agent.x},${game.agent.y}`); | |
| const adjDiamondDir = checkAdjacentDiamond(viewport); | |
| if (adjDiamondDir) m.pickPrecisionOffered++; | |
| const memoryDepthBefore = memory.messages.length; | |
| memory.addUserText(buildPickFirstPrompt(viewport, game.inventory, variant.viewportRadius)); | |
| let action: { action: string; detail: string; tool_call_id: string } | null = null; | |
| try { | |
| const { message, usage, durationMs } = await callModel(memory.getMessages()); | |
| memory.addAssistant(message); | |
| m.latencies.push(durationMs); | |
| m.inputTokens.push(usage.prompt_tokens ?? 0); | |
| m.cachedTokens.push(usage.prompt_tokens_details?.cached_tokens ?? 0); | |
| m.outputTokens.push(usage.completion_tokens ?? 0); | |
| const tc = message.tool_calls?.[0]; | |
| if (tc) { | |
| const name = tc.function?.name; | |
| m.toolCalls[name === 'move' || name === 'pick' ? name : 'other']++; | |
| let args: any = {}; | |
| try { | |
| args = JSON.parse(tc.function.arguments); | |
| } catch {} | |
| if ( | |
| (name === 'move' || name === 'pick') && | |
| ['up', 'down', 'left', 'right'].includes(args.direction) | |
| ) { | |
| action = { action: name, detail: args.direction, tool_call_id: tc.id }; | |
| } | |
| } | |
| } catch (err: any) { | |
| m.errorMessages.push(String(err?.message ?? err).slice(0, 200)); | |
| } | |
| if (!action) { | |
| while (memory.messages.length > memoryDepthBefore) memory.popLast(); | |
| m.fallbacks++; | |
| continue; | |
| } | |
| m.realActions++; | |
| m.directions[action.detail]++; | |
| if ( | |
| adjDiamondDir && | |
| action.action === 'pick' && | |
| action.detail === adjDiamondDir | |
| ) { | |
| m.pickPrecisionTaken++; | |
| } | |
| let toolResult = ''; | |
| let success = true; | |
| try { | |
| const r = game.executeAction(action) as { pickedUp?: string | null } | undefined; | |
| const picked = r?.pickedUp ? ` β picked up ${r.pickedUp}` : ''; | |
| toolResult = `${action.action} ${action.detail} succeeded${picked}`; | |
| } catch (err: any) { | |
| toolResult = `${action.action} ${action.detail} failed: ${err.message}`; | |
| success = false; | |
| m.illegalMoves++; | |
| } | |
| memory.addToolResult(action.tool_call_id, toolResult); | |
| } | |
| m.turns = turns; | |
| m.diamonds = game.stats.diamonds; | |
| m.picks = game.stats.picks; | |
| m.uniqueTilesVisited = visited.size; | |
| return m; | |
| } | |
| // βββ Compute summary stats and emit ββββββββββββββββββββββββββββββββββββββββββ | |
| function summarize(m: Metrics) { | |
| const realActions = Math.max(1, m.realActions); | |
| const offered = Math.max(1, m.pickPrecisionOffered); | |
| return { | |
| ...m, | |
| summary: { | |
| diamondsPerN: m.diamonds / m.turns, | |
| realActionRate: m.realActions / m.turns, | |
| illegalMoveRate: m.illegalMoves / realActions, | |
| pickPrecision: m.pickPrecisionOffered > 0 ? m.pickPrecisionTaken / m.pickPrecisionOffered : null, | |
| directionEntropy: entropy(m.directions), | |
| medianLatencyMs: median(m.latencies), | |
| avgInputTokens: m.inputTokens.length ? m.inputTokens.reduce((a, b) => a + b, 0) / m.inputTokens.length : 0, | |
| cacheHitRate: m.inputTokens.reduce((a, b) => a + b, 0) > 0 | |
| ? m.cachedTokens.reduce((a, b) => a + b, 0) / m.inputTokens.reduce((a, b) => a + b, 0) | |
| : 0, | |
| }, | |
| }; | |
| } | |
| // βββ Main ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const startedAt = new Date().toISOString(); | |
| console.error(`[eval] variant=${variantName} seed=${seed} turns=${turns} endpoint=${ENDPOINT}`); | |
| const metrics = await run(); | |
| const summary = summarize(metrics); | |
| const finalOutput = { startedAt, endedAt: new Date().toISOString(), ...summary }; | |
| const json = JSON.stringify(finalOutput, null, 2); | |
| if (outPath) { | |
| await Bun.write(outPath, json); | |
| console.error(`[eval] wrote ${outPath}`); | |
| } | |
| console.log(json); | |