mini-world / eval.ts
victor's picture
victor HF Staff
chore: switch ENDPOINT to dedicated pod (zopt7f90d1ydgy)
a0a680b
Raw History Blame Contribute Delete
11.5 kB
/**
* eval.ts β€” headless agent benchmark.
*
* Runs the same agent loop as server.ts but without any HTTP server, on a
* deterministic seeded world, using a configurable variant (viewport size,
* thinking on/off, temperature, cache_prompt). Emits one JSON metrics object
* to stdout at the end.
*
* Usage:
* ENDPOINT=https://… bun eval.ts \
* --variant baseline --seed 1 --turns 30 [--out path.json]
*
* Variants are defined in VARIANTS below.
*/
import { SinglePlayerGame, DEFAULTS } from './engine.js';
import {
PICK_FIRST_TOOLS,
PICK_FIRST_SYSTEM_PROMPT,
buildPickFirstPrompt,
AgentMemory,
} from './agent-brain.js';
// ─── Config ──────────────────────────────────────────────────────────────────
interface Variant {
viewportRadius: number;
thinking: boolean;
temperature: number;
cache: boolean;
systemPrompt: string;
maxTokens: number;
}
const VARIANTS: Record<string, Variant> = {
baseline: {
viewportRadius: 4,
thinking: false,
temperature: 0.6,
cache: false,
systemPrompt: PICK_FIRST_SYSTEM_PROMPT,
maxTokens: 100,
},
thinking_on: {
viewportRadius: 4,
thinking: true,
temperature: 0.6,
cache: false,
systemPrompt: PICK_FIRST_SYSTEM_PROMPT,
maxTokens: 1024,
},
viewport_5: {
viewportRadius: 2,
thinking: false,
temperature: 0.6,
cache: false,
systemPrompt: PICK_FIRST_SYSTEM_PROMPT,
maxTokens: 100,
},
viewport_13: {
viewportRadius: 6,
thinking: false,
temperature: 0.6,
cache: false,
systemPrompt: PICK_FIRST_SYSTEM_PROMPT,
maxTokens: 100,
},
temp_0: {
viewportRadius: 4,
thinking: false,
temperature: 0.0,
cache: false,
systemPrompt: PICK_FIRST_SYSTEM_PROMPT,
maxTokens: 100,
},
temp_high: {
viewportRadius: 4,
thinking: false,
temperature: 0.9,
cache: false,
systemPrompt: PICK_FIRST_SYSTEM_PROMPT,
maxTokens: 100,
},
cache_on: {
viewportRadius: 4,
thinking: false,
temperature: 0.6,
cache: true,
systemPrompt: PICK_FIRST_SYSTEM_PROMPT,
maxTokens: 100,
},
};
// ─── CLI args ────────────────────────────────────────────────────────────────
function getArg(name: string, fallback?: string): string {
const idx = process.argv.indexOf(`--${name}`);
if (idx >= 0 && process.argv[idx + 1]) return process.argv[idx + 1];
if (fallback != null) return fallback;
throw new Error(`Missing --${name}`);
}
const variantName = getArg('variant', 'baseline');
const seed = parseInt(getArg('seed', '1'), 10);
const turns = parseInt(getArg('turns', '30'), 10);
const outPath = (() => {
const i = process.argv.indexOf('--out');
return i >= 0 ? process.argv[i + 1] : null;
})();
const variant = VARIANTS[variantName];
if (!variant) {
console.error(`Unknown variant: ${variantName}. Available: ${Object.keys(VARIANTS).join(', ')}`);
process.exit(1);
}
const ENDPOINT = (process.env.ENDPOINT ?? 'https://zopt7f90d1ydgy-8080.proxy.runpod.net').replace(/\/$/, '');
const REMOTE_MODEL = process.env.MODEL ?? 'default';
// ─── Inference (single attempt + 3-retry on Context size) ────────────────────
async function callModel(messages: any[]): Promise<{ message: any; usage: any; durationMs: number; cacheUsed: boolean }> {
const t0 = performance.now();
const baseBody: any = {
model: REMOTE_MODEL,
messages,
tools: PICK_FIRST_TOOLS,
tool_choice: 'auto',
temperature: variant.temperature,
max_tokens: variant.maxTokens,
chat_template_kwargs: { enable_thinking: variant.thinking },
};
let useCache = variant.cache;
baseBody.cache_prompt = useCache;
let res: Response;
let attempt = 0;
let body = { ...baseBody };
while (true) {
res = await fetch(`${ENDPOINT}/v1/chat/completions`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
if (res.ok) break;
const text = await res.text();
const isContextErr = res.status === 500 && /Context size/i.test(text);
if (!isContextErr || attempt >= 3) {
throw new Error(`HTTP ${res.status}: ${text}`);
}
attempt++;
body = { ...baseBody, cache_prompt: false };
useCache = false;
await new Promise((r) => setTimeout(r, 200));
}
const data = await res.json();
const totalMs = Math.round(performance.now() - t0);
return {
message: data.choices?.[0]?.message ?? {},
usage: data.usage ?? {},
durationMs: totalMs,
cacheUsed: useCache,
};
}
// ─── Run one eval ────────────────────────────────────────────────────────────
interface Metrics {
variant: string;
seed: number;
turns: number;
realActions: number;
fallbacks: number;
illegalMoves: number;
diamonds: number;
picks: number;
pickPrecisionOffered: number;
pickPrecisionTaken: number;
uniqueTilesVisited: number;
directions: Record<string, number>;
toolCalls: Record<string, number>;
latencies: number[];
inputTokens: number[];
cachedTokens: number[];
outputTokens: number[];
errorMessages: string[];
}
function checkAdjacentDiamond(viewportStr: string): string | null {
const rows = viewportStr.trim().split('\n').map((r: string) => [...r]);
let cr = Math.floor(rows.length / 2);
let cc = Math.floor((rows[0]?.length ?? 0) / 2);
for (let r = 0; r < rows.length; r++) {
for (let c = 0; c < rows[r].length; c++) {
if (rows[r][c] === 'πŸ€–' || rows[r][c] === '🚢') { cr = r; cc = c; }
}
}
const t = (dr: number, dc: number) => rows[cr + dr]?.[cc + dc];
if (t(-1, 0) === 'πŸ’Ž') return 'up';
if (t(1, 0) === 'πŸ’Ž') return 'down';
if (t(0, -1) === 'πŸ’Ž') return 'left';
if (t(0, 1) === 'πŸ’Ž') return 'right';
return null;
}
function median(arr: number[]): number {
if (!arr.length) return 0;
const sorted = [...arr].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
}
function entropy(counts: Record<string, number>): number {
const total = Object.values(counts).reduce((a, b) => a + b, 0);
if (!total) return 0;
let h = 0;
for (const v of Object.values(counts)) {
if (v > 0) {
const p = v / total;
h -= p * Math.log2(p);
}
}
return h;
}
async function run(): Promise<Metrics> {
const game = new SinglePlayerGame({ seed });
const memory = new AgentMemory(variant.systemPrompt);
const m: Metrics = {
variant: variantName,
seed,
turns: 0,
realActions: 0,
fallbacks: 0,
illegalMoves: 0,
diamonds: 0,
picks: 0,
pickPrecisionOffered: 0,
pickPrecisionTaken: 0,
uniqueTilesVisited: 0,
directions: { up: 0, down: 0, left: 0, right: 0 },
toolCalls: { move: 0, pick: 0, other: 0 },
latencies: [],
inputTokens: [],
cachedTokens: [],
outputTokens: [],
errorMessages: [],
};
const visited = new Set<string>();
for (let i = 1; i <= turns; i++) {
const viewport = game.getViewport();
visited.add(`${game.agent.x},${game.agent.y}`);
const adjDiamondDir = checkAdjacentDiamond(viewport);
if (adjDiamondDir) m.pickPrecisionOffered++;
const memoryDepthBefore = memory.messages.length;
memory.addUserText(buildPickFirstPrompt(viewport, game.inventory, variant.viewportRadius));
let action: { action: string; detail: string; tool_call_id: string } | null = null;
try {
const { message, usage, durationMs } = await callModel(memory.getMessages());
memory.addAssistant(message);
m.latencies.push(durationMs);
m.inputTokens.push(usage.prompt_tokens ?? 0);
m.cachedTokens.push(usage.prompt_tokens_details?.cached_tokens ?? 0);
m.outputTokens.push(usage.completion_tokens ?? 0);
const tc = message.tool_calls?.[0];
if (tc) {
const name = tc.function?.name;
m.toolCalls[name === 'move' || name === 'pick' ? name : 'other']++;
let args: any = {};
try {
args = JSON.parse(tc.function.arguments);
} catch {}
if (
(name === 'move' || name === 'pick') &&
['up', 'down', 'left', 'right'].includes(args.direction)
) {
action = { action: name, detail: args.direction, tool_call_id: tc.id };
}
}
} catch (err: any) {
m.errorMessages.push(String(err?.message ?? err).slice(0, 200));
}
if (!action) {
while (memory.messages.length > memoryDepthBefore) memory.popLast();
m.fallbacks++;
continue;
}
m.realActions++;
m.directions[action.detail]++;
if (
adjDiamondDir &&
action.action === 'pick' &&
action.detail === adjDiamondDir
) {
m.pickPrecisionTaken++;
}
let toolResult = '';
let success = true;
try {
const r = game.executeAction(action) as { pickedUp?: string | null } | undefined;
const picked = r?.pickedUp ? ` β€” picked up ${r.pickedUp}` : '';
toolResult = `${action.action} ${action.detail} succeeded${picked}`;
} catch (err: any) {
toolResult = `${action.action} ${action.detail} failed: ${err.message}`;
success = false;
m.illegalMoves++;
}
memory.addToolResult(action.tool_call_id, toolResult);
}
m.turns = turns;
m.diamonds = game.stats.diamonds;
m.picks = game.stats.picks;
m.uniqueTilesVisited = visited.size;
return m;
}
// ─── Compute summary stats and emit ──────────────────────────────────────────
function summarize(m: Metrics) {
const realActions = Math.max(1, m.realActions);
const offered = Math.max(1, m.pickPrecisionOffered);
return {
...m,
summary: {
diamondsPerN: m.diamonds / m.turns,
realActionRate: m.realActions / m.turns,
illegalMoveRate: m.illegalMoves / realActions,
pickPrecision: m.pickPrecisionOffered > 0 ? m.pickPrecisionTaken / m.pickPrecisionOffered : null,
directionEntropy: entropy(m.directions),
medianLatencyMs: median(m.latencies),
avgInputTokens: m.inputTokens.length ? m.inputTokens.reduce((a, b) => a + b, 0) / m.inputTokens.length : 0,
cacheHitRate: m.inputTokens.reduce((a, b) => a + b, 0) > 0
? m.cachedTokens.reduce((a, b) => a + b, 0) / m.inputTokens.reduce((a, b) => a + b, 0)
: 0,
},
};
}
// ─── Main ────────────────────────────────────────────────────────────────────
const startedAt = new Date().toISOString();
console.error(`[eval] variant=${variantName} seed=${seed} turns=${turns} endpoint=${ENDPOINT}`);
const metrics = await run();
const summary = summarize(metrics);
const finalOutput = { startedAt, endedAt: new Date().toISOString(), ...summary };
const json = JSON.stringify(finalOutput, null, 2);
if (outPath) {
await Bun.write(outPath, json);
console.error(`[eval] wrote ${outPath}`);
}
console.log(json);