Spaces:

Almaatla
/

SpaceCities

Paused

Claude

Claude Sonnet 5

T-014 (partial): commit tools/bench.js, wire a temporary /__bench endpoint

f8aa93f unverified 2 months ago

8.51 kB

/* ============================================================
   BENCH — the committed, re-runnable form of the two throwaway spikes in
   docs/analysis/00-feasibility-spikes.md (ADR-0003, PRD NFR-2/NFR-4). Those spikes settled that
   server-side simulation is affordable — this is what re-runs the same two measurements on
   whatever hardware it's actually run on, so "affordable on the session container" (where the
   spikes were first run) becomes "affordable on the real Hugging Face Space" (TASKS.md T-014)
   instead of staying an assumption.
   THREE MEASUREMENTS, matching the spike's own method:
     1. NATURAL MATCHES — tools/selfplay.js drives a real AI-vs-AI game; per-tick wall-clock cost
        is measured as it's actually incurred in play, not synthetically.
     2. STRESS — large armies seeded directly (engine/state.js makeUnit) on a Gigantic (4x) map
        and sent at each other (engine/commands.js issueAttackMove), so combat/pathing/targeting/
        separation are all hot at once — the honest worst case a natural 20-40 unit match would
        flatter.
     3. MEMORY PER MATCH — server/session.js createSession N times, measuring heap growth. Neither
        original spike measured this; ADR-0003's "N concurrent matches per box" question needs it
        as much as CPU.
   Deterministic wherever it can be: same seed -> same units, same ticks, same outcome (proven in
   test/bench.test.js) — only the WALL-CLOCK TIMING numbers themselves vary run to run, which is
   the whole point of measuring them fresh rather than trusting the spike's old numbers forever.
   Lives in tools/, like tools/ailab.js and tools/selfplay.js — a dev/ops bench, never imported by
   the shipped client or the (future) match server.
   ============================================================ */
"use strict";

import { cpus } from "node:os";
import { createGameState, makeUnit } from "../engine/state.js";
import { tick } from "../engine/sim.js";
import { issueAttackMove } from "../engine/commands.js";
import { mulberry32 } from "../engine/rng.js";
import { createSelfPlayState, tickSelfPlay, SELFPLAY_DT } from "./selfplay.js";
import { createSession } from "../server/session.js";

// p99 by nearest-rank on a copy sorted ascending — same simple method the original spike used
// (docs/analysis/00's own tables), good enough for a bench, not a statistics paper.
function percentile(sortedAsc, p) {
  const idx = Math.min(sortedAsc.length - 1, Math.floor(p * sortedAsc.length));
  return sortedAsc[idx];
}

/** @param {number[]} samplesMs @returns {{mean:number, p50:number, p99:number, max:number}} */
export function summarizeTimings(samplesMs) {
  const sorted = [...samplesMs].sort((a, b) => a - b);
  const mean = samplesMs.reduce((a, v) => a + v, 0) / samplesMs.length;
  return {
    mean: round(mean), p50: round(percentile(sorted, 0.5)),
    p99: round(percentile(sorted, 0.99)), max: round(sorted[sorted.length - 1]),
  };
}

function round(ms) { return Math.round(ms * 1000) / 1000; }   // 3 decimal places — the spike's own tables use 2-3

/**
 * One natural AI-vs-AI match, timed tick by tick. Runs to victory/timeout (state.over) or
 * `maxSeconds` of SIM time, whichever comes first — same "to victory" framing the original
 * spike's 40-minute row used.
 * @param {{planetId?:string, seed?:number, sizeMult?:number, maxSeconds?:number}} [opts]
 */
export function benchNaturalMatch({ planetId = "ferros", seed = 1, sizeMult, maxSeconds = 2400 } = {}) {
  const state = createSelfPlayState({ planetId, seed, sizeMult });
  const samples = [];
  while (!state.over && state.time < maxSeconds) {
    const t0 = performance.now();
    tickSelfPlay(state, SELFPLAY_DT);
    samples.push(performance.now() - t0);
  }
  return {
    scenario: `${planetId}${sizeMult ? ` ${sizeMult}x` : ""}, seed ${seed}, ${Math.round(maxSeconds / 60)} sim-min cap`,
    ...summarizeTimings(samples), ticks: samples.length, units: state.units.size,
    over: state.over, winner: state.winner ?? null, time: round(state.time),
  };
}

// Two facing lines, `armySize` skiffs each, ALREADY IN CONTACT — a fixed, modest gap between them
// regardless of map size (a Gigantic map's edges are thousands of pixels apart; anchoring armies
// there instead of near the centre makes almost every one of `ticks` a pure march with near-zero
// combat cost, and only the last few ticks after they finally close the distance actually stress
// combat/pathing/targeting/separation — measured, not assumed: an earlier edge-to-edge version of
// this file produced a p50 of 0ms for 400+ unit armies for exactly that reason). Spread across
// rows so units don't all stack on one point (the same reason a real army isn't a single-file
// line); 40-wide rows keep even an 800-unit army's own footprint modest next to the gap.
function seedFacingArmies(state, armySize) {
  const { width: w, height: h } = state.map;
  const rowW = 40, spacing = 22, gap = 260;   // gap comfortably inside a skiff's attack range + a short close
  const cx = w / 2, cy = h / 2;
  const army = (owner, dir) => {
    const units = [];
    for (let i = 0; i < armySize; i++) {
      const row = Math.floor(i / rowW), col = i % rowW;
      const u = makeUnit("skiff", owner, cx + dir * (gap / 2 + row * spacing), cy + (col - rowW / 2) * spacing);
      state.units.set(u.id, u);
      units.push(u);
    }
    return units;
  };
  return { a: army("player", -1), b: army("ai", 1) };
}

/**
 * Two `armySize`-unit armies in contact on a Gigantic map for `ticks` steps — the spike's
 * "honest worst case", combat/pathing/targeting/separation all hot simultaneously.
 * @param {{armySize:number, ticks?:number, seed?:number}} opts
 */
export function benchStress({ armySize, ticks = 1200, seed = 1 }) {
  const state = createGameState({ planetId: "ferros", seed, rng: mulberry32(seed), sizeMult: 4 });
  const { width: w, height: h } = state.map;
  const { a, b } = seedFacingArmies(state, armySize);
  // Attack-move THROUGH the opposing line (well past it, not just to its near edge) so units keep
  // engaging instead of arriving, disengaging once nothing is left in range, and going idle.
  issueAttackMove(a, w / 2 + w * 0.1, h / 2);
  issueAttackMove(b, w / 2 - w * 0.1, h / 2);
  const samples = [];
  for (let i = 0; i < ticks; i++) {
    const t0 = performance.now();
    tick(state, 0.1);
    samples.push(performance.now() - t0);
  }
  const alive = [...state.units.values()].filter(u => u.hp > 0).length;
  return { armySize, ticks, ...summarizeTimings(samples), finalUnitsAlive: alive };
}

/**
 * Heap growth from creating `matchCount` independent sessions (server/session.js), the shape the
 * real match server actually instantiates per match. Forces a GC pass before/after when the
 * process was launched with --expose-gc (not required — `gcForced: false` just means the reading
 * is noisier, not that it's missing).
 * @param {{matchCount?:number}} [opts]
 */
export function benchMemory({ matchCount = 20 } = {}) {
  const gcForced = typeof global.gc === "function";
  if (gcForced) global.gc();
  const before = process.memoryUsage().heapUsed;
  const sessions = [];
  for (let i = 0; i < matchCount; i++) {
    sessions.push(createSession({ planetId: "ferros", seed: i + 1, rng: mulberry32(i + 1) }));
  }
  if (gcForced) global.gc();
  const after = process.memoryUsage().heapUsed;
  const deltaMB = (after - before) / (1024 * 1024);
  return { matchCount, gcForced, totalMB: round(deltaMB), perMatchMB: round(deltaMB / matchCount) };
}

/**
 * The whole scoreboard as one JSON-able object — what /__bench (tools/serve.js) returns verbatim.
 * `quick` shrinks every scenario for a fast smoke run (test/bench.test.js); the real re-run
 * (TASKS.md T-014) calls this with no arguments.
 * @param {{quick?:boolean}} [opts]
 */
export function runBenchSuite({ quick = false } = {}) {
  const natural = quick
    ? [benchNaturalMatch({ seed: 1, maxSeconds: 30 })]
    : [
        benchNaturalMatch({ seed: 1, maxSeconds: 600 }),
        benchNaturalMatch({ seed: 2, maxSeconds: 2400 }),
        benchNaturalMatch({ seed: 3, sizeMult: 4, maxSeconds: 2400 }),
      ];
  const stress = quick
    ? [benchStress({ armySize: 10, ticks: 10 })]
    : [200, 400, 800].map(armySize => benchStress({ armySize }));
  return {
    node: process.version, cpus: cpus().length, at: new Date().toISOString(),
    natural, stress, memory: benchMemory({ matchCount: quick ? 3 : 20 }),
  };
}

	/* ============================================================
	BENCH — the committed, re-runnable form of the two throwaway spikes in
	docs/analysis/00-feasibility-spikes.md (ADR-0003, PRD NFR-2/NFR-4). Those spikes settled that
	server-side simulation is affordable — this is what re-runs the same two measurements on
	whatever hardware it's actually run on, so "affordable on the session container" (where the
	spikes were first run) becomes "affordable on the real Hugging Face Space" (TASKS.md T-014)
	instead of staying an assumption.

	THREE MEASUREMENTS, matching the spike's own method:
	1. NATURAL MATCHES — tools/selfplay.js drives a real AI-vs-AI game; per-tick wall-clock cost
	is measured as it's actually incurred in play, not synthetically.
	2. STRESS — large armies seeded directly (engine/state.js makeUnit) on a Gigantic (4x) map
	and sent at each other (engine/commands.js issueAttackMove), so combat/pathing/targeting/
	separation are all hot at once — the honest worst case a natural 20-40 unit match would
	flatter.
	3. MEMORY PER MATCH — server/session.js createSession N times, measuring heap growth. Neither
	original spike measured this; ADR-0003's "N concurrent matches per box" question needs it
	as much as CPU.

	Deterministic wherever it can be: same seed -> same units, same ticks, same outcome (proven in
	test/bench.test.js) — only the WALL-CLOCK TIMING numbers themselves vary run to run, which is
	the whole point of measuring them fresh rather than trusting the spike's old numbers forever.

	Lives in tools/, like tools/ailab.js and tools/selfplay.js — a dev/ops bench, never imported by
	the shipped client or the (future) match server.
	============================================================ */

	"use strict";

	import { cpus } from "node:os";
	import { createGameState, makeUnit } from "../engine/state.js";
	import { tick } from "../engine/sim.js";
	import { issueAttackMove } from "../engine/commands.js";
	import { mulberry32 } from "../engine/rng.js";
	import { createSelfPlayState, tickSelfPlay, SELFPLAY_DT } from "./selfplay.js";
	import { createSession } from "../server/session.js";

	// p99 by nearest-rank on a copy sorted ascending — same simple method the original spike used
	// (docs/analysis/00's own tables), good enough for a bench, not a statistics paper.
	function percentile(sortedAsc, p) {
	const idx = Math.min(sortedAsc.length - 1, Math.floor(p * sortedAsc.length));
	return sortedAsc[idx];
	}

	/** @param {number[]} samplesMs @returns {{mean:number, p50:number, p99:number, max:number}} */
	export function summarizeTimings(samplesMs) {
	const sorted = [...samplesMs].sort((a, b) => a - b);
	const mean = samplesMs.reduce((a, v) => a + v, 0) / samplesMs.length;
	return {
	mean: round(mean), p50: round(percentile(sorted, 0.5)),
	p99: round(percentile(sorted, 0.99)), max: round(sorted[sorted.length - 1]),
	};
	}

	function round(ms) { return Math.round(ms * 1000) / 1000; } // 3 decimal places — the spike's own tables use 2-3

	/**
	* One natural AI-vs-AI match, timed tick by tick. Runs to victory/timeout (state.over) or
	* `maxSeconds` of SIM time, whichever comes first — same "to victory" framing the original
	* spike's 40-minute row used.
	* @param {{planetId?:string, seed?:number, sizeMult?:number, maxSeconds?:number}} [opts]
	*/
	export function benchNaturalMatch({ planetId = "ferros", seed = 1, sizeMult, maxSeconds = 2400 } = {}) {
	const state = createSelfPlayState({ planetId, seed, sizeMult });
	const samples = [];
	while (!state.over && state.time < maxSeconds) {
	const t0 = performance.now();
	tickSelfPlay(state, SELFPLAY_DT);
	samples.push(performance.now() - t0);
	}
	return {
	scenario: `${planetId}${sizeMult ? ` ${sizeMult}x` : ""}, seed ${seed}, ${Math.round(maxSeconds / 60)} sim-min cap`,
	...summarizeTimings(samples), ticks: samples.length, units: state.units.size,
	over: state.over, winner: state.winner ?? null, time: round(state.time),
	};
	}

	// Two facing lines, `armySize` skiffs each, ALREADY IN CONTACT — a fixed, modest gap between them
	// regardless of map size (a Gigantic map's edges are thousands of pixels apart; anchoring armies
	// there instead of near the centre makes almost every one of `ticks` a pure march with near-zero
	// combat cost, and only the last few ticks after they finally close the distance actually stress
	// combat/pathing/targeting/separation — measured, not assumed: an earlier edge-to-edge version of
	// this file produced a p50 of 0ms for 400+ unit armies for exactly that reason). Spread across
	// rows so units don't all stack on one point (the same reason a real army isn't a single-file
	// line); 40-wide rows keep even an 800-unit army's own footprint modest next to the gap.
	function seedFacingArmies(state, armySize) {
	const { width: w, height: h } = state.map;
	const rowW = 40, spacing = 22, gap = 260; // gap comfortably inside a skiff's attack range + a short close
	const cx = w / 2, cy = h / 2;
	const army = (owner, dir) => {
	const units = [];
	for (let i = 0; i < armySize; i++) {
	const row = Math.floor(i / rowW), col = i % rowW;
	const u = makeUnit("skiff", owner, cx + dir * (gap / 2 + row * spacing), cy + (col - rowW / 2) * spacing);
	state.units.set(u.id, u);
	units.push(u);
	}
	return units;
	};
	return { a: army("player", -1), b: army("ai", 1) };
	}

	/**
	* Two `armySize`-unit armies in contact on a Gigantic map for `ticks` steps — the spike's
	* "honest worst case", combat/pathing/targeting/separation all hot simultaneously.
	* @param {{armySize:number, ticks?:number, seed?:number}} opts
	*/
	export function benchStress({ armySize, ticks = 1200, seed = 1 }) {
	const state = createGameState({ planetId: "ferros", seed, rng: mulberry32(seed), sizeMult: 4 });
	const { width: w, height: h } = state.map;
	const { a, b } = seedFacingArmies(state, armySize);
	// Attack-move THROUGH the opposing line (well past it, not just to its near edge) so units keep
	// engaging instead of arriving, disengaging once nothing is left in range, and going idle.
	issueAttackMove(a, w / 2 + w * 0.1, h / 2);
	issueAttackMove(b, w / 2 - w * 0.1, h / 2);
	const samples = [];
	for (let i = 0; i < ticks; i++) {
	const t0 = performance.now();
	tick(state, 0.1);
	samples.push(performance.now() - t0);
	}
	const alive = [...state.units.values()].filter(u => u.hp > 0).length;
	return { armySize, ticks, ...summarizeTimings(samples), finalUnitsAlive: alive };
	}

	/**
	* Heap growth from creating `matchCount` independent sessions (server/session.js), the shape the
	* real match server actually instantiates per match. Forces a GC pass before/after when the
	* process was launched with --expose-gc (not required — `gcForced: false` just means the reading
	* is noisier, not that it's missing).
	* @param {{matchCount?:number}} [opts]
	*/
	export function benchMemory({ matchCount = 20 } = {}) {
	const gcForced = typeof global.gc === "function";
	if (gcForced) global.gc();
	const before = process.memoryUsage().heapUsed;
	const sessions = [];
	for (let i = 0; i < matchCount; i++) {
	sessions.push(createSession({ planetId: "ferros", seed: i + 1, rng: mulberry32(i + 1) }));
	}
	if (gcForced) global.gc();
	const after = process.memoryUsage().heapUsed;
	const deltaMB = (after - before) / (1024 * 1024);
	return { matchCount, gcForced, totalMB: round(deltaMB), perMatchMB: round(deltaMB / matchCount) };
	}

	/**
	* The whole scoreboard as one JSON-able object — what /__bench (tools/serve.js) returns verbatim.
	* `quick` shrinks every scenario for a fast smoke run (test/bench.test.js); the real re-run
	* (TASKS.md T-014) calls this with no arguments.
	* @param {{quick?:boolean}} [opts]
	*/
	export function runBenchSuite({ quick = false } = {}) {
	const natural = quick
	? [benchNaturalMatch({ seed: 1, maxSeconds: 30 })]
	: [
	benchNaturalMatch({ seed: 1, maxSeconds: 600 }),
	benchNaturalMatch({ seed: 2, maxSeconds: 2400 }),
	benchNaturalMatch({ seed: 3, sizeMult: 4, maxSeconds: 2400 }),
	];
	const stress = quick
	? [benchStress({ armySize: 10, ticks: 10 })]
	: [200, 400, 800].map(armySize => benchStress({ armySize }));
	return {
	node: process.version, cpus: cpus().length, at: new Date().toISOString(),
	natural, stress, memory: benchMemory({ matchCount: quick ? 3 : 20 }),
	};
	}